From 566cc81d258865e3caf26aca625fac5fcea2cca1 Mon Sep 17 00:00:00 2001 From: Jonathan Gray Date: Wed, 25 Apr 2018 01:27:48 +0000 Subject: update ttm and radeondrm(4) to Linux 4.4.129 Compared to the previous Linux 3.8 based port this adds support for KAVERI/KABINI/MULLINS APUs and OLAND/BONAIRE/HAINAN/HAWAII GPUs. Thanks to the OpenBSD Foundation for sponsoring this work. --- sys/dev/pci/drm/drm_cache.c | 10 +- sys/dev/pci/drm/drm_cache.h | 13 +- sys/dev/pci/drm/drm_linux.c | 16 +- sys/dev/pci/drm/drm_linux.h | 426 +- sys/dev/pci/drm/drm_linux_rbtree.h | 42 +- sys/dev/pci/drm/drm_pciids.h | 131 +- sys/dev/pci/drm/files.drm | 62 +- sys/dev/pci/drm/i915/intel_i2c.c | 2 +- sys/dev/pci/drm/i915/intel_sprite.c | 2 +- sys/dev/pci/drm/linux_rcupdate.h | 18 + sys/dev/pci/drm/linux_reservation.c | 495 ++ sys/dev/pci/drm/linux_reservation.h | 140 + sys/dev/pci/drm/radeon/ObjectID.h | 41 +- sys/dev/pci/drm/radeon/atom-bits.h | 1 - sys/dev/pci/drm/radeon/atom-names.h | 1 - sys/dev/pci/drm/radeon/atom-types.h | 1 - sys/dev/pci/drm/radeon/atom.c | 33 +- sys/dev/pci/drm/radeon/atom.h | 3 +- sys/dev/pci/drm/radeon/atombios.h | 1176 ++- sys/dev/pci/drm/radeon/atombios_crtc.c | 465 +- sys/dev/pci/drm/radeon/atombios_dp.c | 46 +- sys/dev/pci/drm/radeon/atombios_encoders.c | 506 +- sys/dev/pci/drm/radeon/atombios_i2c.c | 39 +- sys/dev/pci/drm/radeon/avivod.h | 1 - sys/dev/pci/drm/radeon/btc_dpm.c | 2824 ++++++ sys/dev/pci/drm/radeon/btc_dpm.h | 59 + sys/dev/pci/drm/radeon/btcd.h | 185 + sys/dev/pci/drm/radeon/cayman_blit_shaders.c | 61 +- sys/dev/pci/drm/radeon/cayman_blit_shaders.h | 1 - sys/dev/pci/drm/radeon/ci_dpm.c | 5969 +++++++++++++ sys/dev/pci/drm/radeon/ci_dpm.h | 341 + sys/dev/pci/drm/radeon/ci_smc.c | 294 + sys/dev/pci/drm/radeon/cik.c | 10078 ++++++++++++++++++++++ sys/dev/pci/drm/radeon/cik_blit_shaders.c | 244 + sys/dev/pci/drm/radeon/cik_blit_shaders.h | 32 + sys/dev/pci/drm/radeon/cik_reg.h | 243 + sys/dev/pci/drm/radeon/cik_sdma.c | 995 +++ sys/dev/pci/drm/radeon/cikd.h | 2172 +++++ sys/dev/pci/drm/radeon/clearstate_cayman.h | 1081 +++ sys/dev/pci/drm/radeon/clearstate_ci.h | 944 ++ sys/dev/pci/drm/radeon/clearstate_defs.h | 44 + sys/dev/pci/drm/radeon/clearstate_evergreen.h | 1080 +++ sys/dev/pci/drm/radeon/clearstate_si.h | 941 ++ sys/dev/pci/drm/radeon/cypress_dpm.c | 2165 +++++ sys/dev/pci/drm/radeon/cypress_dpm.h | 160 + sys/dev/pci/drm/radeon/dce3_1_afmt.c | 232 + sys/dev/pci/drm/radeon/dce6_afmt.c | 318 + sys/dev/pci/drm/radeon/drm_buffer.h | 148 + sys/dev/pci/drm/radeon/evergreen.c | 3563 ++++++-- sys/dev/pci/drm/radeon/evergreen_blit_kms.c | 730 -- sys/dev/pci/drm/radeon/evergreen_blit_shaders.c | 61 +- sys/dev/pci/drm/radeon/evergreen_blit_shaders.h | 1 - sys/dev/pci/drm/radeon/evergreen_cs.c | 1462 ++-- sys/dev/pci/drm/radeon/evergreen_dma.c | 182 + sys/dev/pci/drm/radeon/evergreen_hdmi.c | 546 +- sys/dev/pci/drm/radeon/evergreen_reg.h | 79 +- sys/dev/pci/drm/radeon/evergreen_smc.h | 67 + sys/dev/pci/drm/radeon/evergreend.h | 592 +- sys/dev/pci/drm/radeon/kv_dpm.c | 2897 +++++++ sys/dev/pci/drm/radeon/kv_dpm.h | 200 + sys/dev/pci/drm/radeon/kv_smc.c | 215 + sys/dev/pci/drm/radeon/mkregtable.c | 40 +- sys/dev/pci/drm/radeon/ni.c | 1704 ++-- sys/dev/pci/drm/radeon/ni_dma.c | 472 + sys/dev/pci/drm/radeon/ni_dpm.c | 4377 ++++++++++ sys/dev/pci/drm/radeon/ni_dpm.h | 250 + sys/dev/pci/drm/radeon/ni_reg.h | 45 +- sys/dev/pci/drm/radeon/nid.h | 663 +- sys/dev/pci/drm/radeon/nislands_smc.h | 329 + sys/dev/pci/drm/radeon/ppsmc.h | 191 + sys/dev/pci/drm/radeon/pptable.h | 690 ++ sys/dev/pci/drm/radeon/r100.c | 581 +- sys/dev/pci/drm/radeon/r100_track.h | 5 - sys/dev/pci/drm/radeon/r100d.h | 12 - sys/dev/pci/drm/radeon/r200.c | 72 +- sys/dev/pci/drm/radeon/r300.c | 150 +- sys/dev/pci/drm/radeon/r300_reg.h | 1 - sys/dev/pci/drm/radeon/r300d.h | 12 - sys/dev/pci/drm/radeon/r420.c | 17 +- sys/dev/pci/drm/radeon/r420d.h | 1 - sys/dev/pci/drm/radeon/r500_reg.h | 3 +- sys/dev/pci/drm/radeon/r520.c | 4 +- sys/dev/pci/drm/radeon/r520d.h | 1 - sys/dev/pci/drm/radeon/r600.c | 2006 +++-- sys/dev/pci/drm/radeon/r600_audio.c | 255 - sys/dev/pci/drm/radeon/r600_blit_kms.c | 786 -- sys/dev/pci/drm/radeon/r600_blit_shaders.c | 6 +- sys/dev/pci/drm/radeon/r600_blit_shaders.h | 2 - sys/dev/pci/drm/radeon/r600_cs.c | 498 +- sys/dev/pci/drm/radeon/r600_dma.c | 491 ++ sys/dev/pci/drm/radeon/r600_dpm.c | 1368 +++ sys/dev/pci/drm/radeon/r600_dpm.h | 236 + sys/dev/pci/drm/radeon/r600_hdmi.c | 653 +- sys/dev/pci/drm/radeon/r600_reg.h | 7 +- sys/dev/pci/drm/radeon/r600d.h | 490 +- sys/dev/pci/drm/radeon/radeon.h | 1593 +++- sys/dev/pci/drm/radeon/radeon_acpi.c | 151 +- sys/dev/pci/drm/radeon/radeon_acpi.h | 1 - sys/dev/pci/drm/radeon/radeon_agp.c | 7 +- sys/dev/pci/drm/radeon/radeon_asic.c | 1643 +++- sys/dev/pci/drm/radeon/radeon_asic.h | 624 +- sys/dev/pci/drm/radeon/radeon_atombios.c | 1441 +++- sys/dev/pci/drm/radeon/radeon_audio.c | 788 ++ sys/dev/pci/drm/radeon/radeon_audio.h | 84 + sys/dev/pci/drm/radeon/radeon_benchmark.c | 49 +- sys/dev/pci/drm/radeon/radeon_bios.c | 218 +- sys/dev/pci/drm/radeon/radeon_blit_common.h | 45 - sys/dev/pci/drm/radeon/radeon_clocks.c | 1 - sys/dev/pci/drm/radeon/radeon_combios.c | 57 +- sys/dev/pci/drm/radeon/radeon_connectors.c | 834 +- sys/dev/pci/drm/radeon/radeon_cs.c | 848 +- sys/dev/pci/drm/radeon/radeon_cursor.c | 321 +- sys/dev/pci/drm/radeon/radeon_device.c | 804 +- sys/dev/pci/drm/radeon/radeon_devlist.h | 102 +- sys/dev/pci/drm/radeon/radeon_display.c | 952 +- sys/dev/pci/drm/radeon/radeon_dp_mst.c | 794 ++ sys/dev/pci/drm/radeon/radeon_drv.c | 718 ++ sys/dev/pci/drm/radeon/radeon_drv.h | 2165 +++++ sys/dev/pci/drm/radeon/radeon_encoders.c | 89 +- sys/dev/pci/drm/radeon/radeon_family.h | 9 +- sys/dev/pci/drm/radeon/radeon_fb.c | 191 +- sys/dev/pci/drm/radeon/radeon_fence.c | 771 +- sys/dev/pci/drm/radeon/radeon_gart.c | 1014 +-- sys/dev/pci/drm/radeon/radeon_gem.c | 368 +- sys/dev/pci/drm/radeon/radeon_i2c.c | 73 +- sys/dev/pci/drm/radeon/radeon_ib.c | 319 + sys/dev/pci/drm/radeon/radeon_ioc32.c | 423 + sys/dev/pci/drm/radeon/radeon_irq_kms.c | 95 +- sys/dev/pci/drm/radeon/radeon_kfd.c | 859 ++ sys/dev/pci/drm/radeon/radeon_kfd.h | 48 + sys/dev/pci/drm/radeon/radeon_kms.c | 1058 ++- sys/dev/pci/drm/radeon/radeon_legacy_crtc.c | 32 +- sys/dev/pci/drm/radeon/radeon_legacy_encoders.c | 12 +- sys/dev/pci/drm/radeon/radeon_legacy_tv.c | 2 - sys/dev/pci/drm/radeon/radeon_mn.c | 319 + sys/dev/pci/drm/radeon/radeon_mode.h | 324 +- sys/dev/pci/drm/radeon/radeon_object.c | 443 +- sys/dev/pci/drm/radeon/radeon_object.h | 51 +- sys/dev/pci/drm/radeon/radeon_pm.c | 1233 ++- sys/dev/pci/drm/radeon/radeon_prime.c | 197 +- sys/dev/pci/drm/radeon/radeon_reg.h | 19 +- sys/dev/pci/drm/radeon/radeon_ring.c | 515 +- sys/dev/pci/drm/radeon/radeon_sa.c | 39 +- sys/dev/pci/drm/radeon/radeon_semaphore.c | 66 +- sys/dev/pci/drm/radeon/radeon_sync.c | 220 + sys/dev/pci/drm/radeon/radeon_test.c | 226 +- sys/dev/pci/drm/radeon/radeon_trace.h | 126 +- sys/dev/pci/drm/radeon/radeon_trace_points.c | 2 - sys/dev/pci/drm/radeon/radeon_ttm.c | 604 +- sys/dev/pci/drm/radeon/radeon_ucode.c | 174 + sys/dev/pci/drm/radeon/radeon_ucode.h | 227 + sys/dev/pci/drm/radeon/radeon_uvd.c | 1001 +++ sys/dev/pci/drm/radeon/radeon_vce.c | 864 ++ sys/dev/pci/drm/radeon/radeon_vm.c | 1265 +++ sys/dev/pci/drm/radeon/reg_srcs/cayman | 2 +- sys/dev/pci/drm/radeon/reg_srcs/evergreen | 2 +- sys/dev/pci/drm/radeon/rs100d.h | 1 - sys/dev/pci/drm/radeon/rs400.c | 42 +- sys/dev/pci/drm/radeon/rs400d.h | 1 - sys/dev/pci/drm/radeon/rs600.c | 166 +- sys/dev/pci/drm/radeon/rs600d.h | 1 - sys/dev/pci/drm/radeon/rs690.c | 346 +- sys/dev/pci/drm/radeon/rs690d.h | 4 +- sys/dev/pci/drm/radeon/rs780_dpm.c | 1075 +++ sys/dev/pci/drm/radeon/rs780_dpm.h | 109 + sys/dev/pci/drm/radeon/rs780d.h | 171 + sys/dev/pci/drm/radeon/rv200d.h | 1 - sys/dev/pci/drm/radeon/rv250d.h | 1 - sys/dev/pci/drm/radeon/rv350d.h | 1 - sys/dev/pci/drm/radeon/rv515.c | 248 +- sys/dev/pci/drm/radeon/rv515d.h | 12 - sys/dev/pci/drm/radeon/rv6xx_dpm.c | 2158 +++++ sys/dev/pci/drm/radeon/rv6xx_dpm.h | 95 + sys/dev/pci/drm/radeon/rv6xxd.h | 246 + sys/dev/pci/drm/radeon/rv730_dpm.c | 508 ++ sys/dev/pci/drm/radeon/rv730d.h | 165 + sys/dev/pci/drm/radeon/rv740_dpm.c | 416 + sys/dev/pci/drm/radeon/rv740d.h | 117 + sys/dev/pci/drm/radeon/rv770.c | 1036 ++- sys/dev/pci/drm/radeon/rv770_dma.c | 94 + sys/dev/pci/drm/radeon/rv770_dpm.c | 2585 ++++++ sys/dev/pci/drm/radeon/rv770_dpm.h | 284 + sys/dev/pci/drm/radeon/rv770_smc.c | 630 ++ sys/dev/pci/drm/radeon/rv770_smc.h | 207 + sys/dev/pci/drm/radeon/rv770d.h | 353 +- sys/dev/pci/drm/radeon/si.c | 5184 +++++++++-- sys/dev/pci/drm/radeon/si_blit_shaders.c | 5 +- sys/dev/pci/drm/radeon/si_blit_shaders.h | 1 - sys/dev/pci/drm/radeon/si_dma.c | 284 + sys/dev/pci/drm/radeon/si_dpm.c | 7178 +++++++++++++++ sys/dev/pci/drm/radeon/si_dpm.h | 238 + sys/dev/pci/drm/radeon/si_reg.h | 1 - sys/dev/pci/drm/radeon/si_smc.c | 309 + sys/dev/pci/drm/radeon/sid.h | 927 +- sys/dev/pci/drm/radeon/sislands_smc.h | 424 + sys/dev/pci/drm/radeon/smu7.h | 170 + sys/dev/pci/drm/radeon/smu7_discrete.h | 514 ++ sys/dev/pci/drm/radeon/smu7_fusion.h | 300 + sys/dev/pci/drm/radeon/sumo_dpm.c | 1944 +++++ sys/dev/pci/drm/radeon/sumo_dpm.h | 220 + sys/dev/pci/drm/radeon/sumo_smc.c | 221 + sys/dev/pci/drm/radeon/sumod.h | 372 + sys/dev/pci/drm/radeon/trinity_dpm.c | 2103 +++++ sys/dev/pci/drm/radeon/trinity_dpm.h | 134 + sys/dev/pci/drm/radeon/trinity_smc.c | 127 + sys/dev/pci/drm/radeon/trinityd.h | 228 + sys/dev/pci/drm/radeon/uvd_v1_0.c | 534 ++ sys/dev/pci/drm/radeon/uvd_v2_2.c | 197 + sys/dev/pci/drm/radeon/uvd_v3_1.c | 57 + sys/dev/pci/drm/radeon/uvd_v4_2.c | 67 + sys/dev/pci/drm/radeon/vce_v1_0.c | 383 + sys/dev/pci/drm/radeon/vce_v2_0.c | 193 + sys/dev/pci/drm/radeon_drm.h | 47 +- sys/dev/pci/drm/ttm/ttm_agp_backend.c | 2 +- sys/dev/pci/drm/ttm/ttm_bo.c | 686 +- sys/dev/pci/drm/ttm/ttm_bo_api.h | 120 +- sys/dev/pci/drm/ttm/ttm_bo_driver.h | 245 +- sys/dev/pci/drm/ttm/ttm_bo_manager.c | 17 +- sys/dev/pci/drm/ttm/ttm_bo_util.c | 124 +- sys/dev/pci/drm/ttm/ttm_bo_vm.c | 526 +- sys/dev/pci/drm/ttm/ttm_execbuf_util.c | 197 +- sys/dev/pci/drm/ttm/ttm_execbuf_util.h | 41 +- sys/dev/pci/drm/ttm/ttm_lock.c | 342 - sys/dev/pci/drm/ttm/ttm_lock.h | 247 - sys/dev/pci/drm/ttm/ttm_memory.c | 42 +- sys/dev/pci/drm/ttm/ttm_memory.h | 6 +- sys/dev/pci/drm/ttm/ttm_module.c | 100 - sys/dev/pci/drm/ttm/ttm_module.h | 1 - sys/dev/pci/drm/ttm/ttm_object.c | 451 - sys/dev/pci/drm/ttm/ttm_object.h | 275 - sys/dev/pci/drm/ttm/ttm_page_alloc.c | 210 +- sys/dev/pci/drm/ttm/ttm_page_alloc.h | 18 +- sys/dev/pci/drm/ttm/ttm_page_alloc_dma.c | 1160 +++ sys/dev/pci/drm/ttm/ttm_placement.h | 4 +- sys/dev/pci/drm/ttm/ttm_tt.c | 49 +- 235 files changed, 110885 insertions(+), 16229 deletions(-) create mode 100644 sys/dev/pci/drm/linux_rcupdate.h create mode 100644 sys/dev/pci/drm/linux_reservation.c create mode 100644 sys/dev/pci/drm/linux_reservation.h create mode 100644 sys/dev/pci/drm/radeon/btc_dpm.c create mode 100644 sys/dev/pci/drm/radeon/btc_dpm.h create mode 100644 sys/dev/pci/drm/radeon/btcd.h create mode 100644 sys/dev/pci/drm/radeon/ci_dpm.c create mode 100644 sys/dev/pci/drm/radeon/ci_dpm.h create mode 100644 sys/dev/pci/drm/radeon/ci_smc.c create mode 100644 sys/dev/pci/drm/radeon/cik.c create mode 100644 sys/dev/pci/drm/radeon/cik_blit_shaders.c create mode 100644 sys/dev/pci/drm/radeon/cik_blit_shaders.h create mode 100644 sys/dev/pci/drm/radeon/cik_reg.h create mode 100644 sys/dev/pci/drm/radeon/cik_sdma.c create mode 100644 sys/dev/pci/drm/radeon/cikd.h create mode 100644 sys/dev/pci/drm/radeon/clearstate_cayman.h create mode 100644 sys/dev/pci/drm/radeon/clearstate_ci.h create mode 100644 sys/dev/pci/drm/radeon/clearstate_defs.h create mode 100644 sys/dev/pci/drm/radeon/clearstate_evergreen.h create mode 100644 sys/dev/pci/drm/radeon/clearstate_si.h create mode 100644 sys/dev/pci/drm/radeon/cypress_dpm.c create mode 100644 sys/dev/pci/drm/radeon/cypress_dpm.h create mode 100644 sys/dev/pci/drm/radeon/dce3_1_afmt.c create mode 100644 sys/dev/pci/drm/radeon/dce6_afmt.c create mode 100644 sys/dev/pci/drm/radeon/drm_buffer.h delete mode 100644 sys/dev/pci/drm/radeon/evergreen_blit_kms.c create mode 100644 sys/dev/pci/drm/radeon/evergreen_dma.c create mode 100644 sys/dev/pci/drm/radeon/evergreen_smc.h create mode 100644 sys/dev/pci/drm/radeon/kv_dpm.c create mode 100644 sys/dev/pci/drm/radeon/kv_dpm.h create mode 100644 sys/dev/pci/drm/radeon/kv_smc.c create mode 100644 sys/dev/pci/drm/radeon/ni_dma.c create mode 100644 sys/dev/pci/drm/radeon/ni_dpm.c create mode 100644 sys/dev/pci/drm/radeon/ni_dpm.h create mode 100644 sys/dev/pci/drm/radeon/nislands_smc.h create mode 100644 sys/dev/pci/drm/radeon/ppsmc.h create mode 100644 sys/dev/pci/drm/radeon/pptable.h delete mode 100644 sys/dev/pci/drm/radeon/r600_audio.c delete mode 100644 sys/dev/pci/drm/radeon/r600_blit_kms.c create mode 100644 sys/dev/pci/drm/radeon/r600_dma.c create mode 100644 sys/dev/pci/drm/radeon/r600_dpm.c create mode 100644 sys/dev/pci/drm/radeon/r600_dpm.h create mode 100644 sys/dev/pci/drm/radeon/radeon_audio.c create mode 100644 sys/dev/pci/drm/radeon/radeon_audio.h delete mode 100644 sys/dev/pci/drm/radeon/radeon_blit_common.h create mode 100644 sys/dev/pci/drm/radeon/radeon_dp_mst.c create mode 100644 sys/dev/pci/drm/radeon/radeon_drv.c create mode 100644 sys/dev/pci/drm/radeon/radeon_drv.h create mode 100644 sys/dev/pci/drm/radeon/radeon_ib.c create mode 100644 sys/dev/pci/drm/radeon/radeon_ioc32.c create mode 100644 sys/dev/pci/drm/radeon/radeon_kfd.c create mode 100644 sys/dev/pci/drm/radeon/radeon_kfd.h create mode 100644 sys/dev/pci/drm/radeon/radeon_mn.c create mode 100644 sys/dev/pci/drm/radeon/radeon_sync.c create mode 100644 sys/dev/pci/drm/radeon/radeon_ucode.c create mode 100644 sys/dev/pci/drm/radeon/radeon_ucode.h create mode 100644 sys/dev/pci/drm/radeon/radeon_uvd.c create mode 100644 sys/dev/pci/drm/radeon/radeon_vce.c create mode 100644 sys/dev/pci/drm/radeon/radeon_vm.c create mode 100644 sys/dev/pci/drm/radeon/rs780_dpm.c create mode 100644 sys/dev/pci/drm/radeon/rs780_dpm.h create mode 100644 sys/dev/pci/drm/radeon/rs780d.h create mode 100644 sys/dev/pci/drm/radeon/rv6xx_dpm.c create mode 100644 sys/dev/pci/drm/radeon/rv6xx_dpm.h create mode 100644 sys/dev/pci/drm/radeon/rv6xxd.h create mode 100644 sys/dev/pci/drm/radeon/rv730_dpm.c create mode 100644 sys/dev/pci/drm/radeon/rv730d.h create mode 100644 sys/dev/pci/drm/radeon/rv740_dpm.c create mode 100644 sys/dev/pci/drm/radeon/rv740d.h create mode 100644 sys/dev/pci/drm/radeon/rv770_dma.c create mode 100644 sys/dev/pci/drm/radeon/rv770_dpm.c create mode 100644 sys/dev/pci/drm/radeon/rv770_dpm.h create mode 100644 sys/dev/pci/drm/radeon/rv770_smc.c create mode 100644 sys/dev/pci/drm/radeon/rv770_smc.h create mode 100644 sys/dev/pci/drm/radeon/si_dma.c create mode 100644 sys/dev/pci/drm/radeon/si_dpm.c create mode 100644 sys/dev/pci/drm/radeon/si_dpm.h create mode 100644 sys/dev/pci/drm/radeon/si_smc.c create mode 100644 sys/dev/pci/drm/radeon/sislands_smc.h create mode 100644 sys/dev/pci/drm/radeon/smu7.h create mode 100644 sys/dev/pci/drm/radeon/smu7_discrete.h create mode 100644 sys/dev/pci/drm/radeon/smu7_fusion.h create mode 100644 sys/dev/pci/drm/radeon/sumo_dpm.c create mode 100644 sys/dev/pci/drm/radeon/sumo_dpm.h create mode 100644 sys/dev/pci/drm/radeon/sumo_smc.c create mode 100644 sys/dev/pci/drm/radeon/sumod.h create mode 100644 sys/dev/pci/drm/radeon/trinity_dpm.c create mode 100644 sys/dev/pci/drm/radeon/trinity_dpm.h create mode 100644 sys/dev/pci/drm/radeon/trinity_smc.c create mode 100644 sys/dev/pci/drm/radeon/trinityd.h create mode 100644 sys/dev/pci/drm/radeon/uvd_v1_0.c create mode 100644 sys/dev/pci/drm/radeon/uvd_v2_2.c create mode 100644 sys/dev/pci/drm/radeon/uvd_v3_1.c create mode 100644 sys/dev/pci/drm/radeon/uvd_v4_2.c create mode 100644 sys/dev/pci/drm/radeon/vce_v1_0.c create mode 100644 sys/dev/pci/drm/radeon/vce_v2_0.c delete mode 100644 sys/dev/pci/drm/ttm/ttm_lock.c delete mode 100644 sys/dev/pci/drm/ttm/ttm_lock.h delete mode 100644 sys/dev/pci/drm/ttm/ttm_module.c delete mode 100644 sys/dev/pci/drm/ttm/ttm_object.c delete mode 100644 sys/dev/pci/drm/ttm/ttm_object.h create mode 100644 sys/dev/pci/drm/ttm/ttm_page_alloc_dma.c diff --git a/sys/dev/pci/drm/drm_cache.c b/sys/dev/pci/drm/drm_cache.c index 56654378487..01fdcee38aa 100644 --- a/sys/dev/pci/drm/drm_cache.c +++ b/sys/dev/pci/drm/drm_cache.c @@ -1,4 +1,4 @@ -/* $OpenBSD: drm_cache.c,v 1.5 2018/04/20 16:09:36 deraadt Exp $ */ +/* $OpenBSD: drm_cache.c,v 1.6 2018/04/25 01:27:46 jsg Exp $ */ /* * Copyright (c) 2017 Mark Kettenis * @@ -17,6 +17,7 @@ #include +#if defined(__i386__) || defined(__amd64__) static void drm_clflush_page(struct vm_page *page) { @@ -53,3 +54,10 @@ drm_clflush_virt_range(void *addr, unsigned long length) { pmap_flush_cache((vaddr_t)addr, length); } +#else +void +drm_clflush_pages(struct vm_page *pages[], unsigned long num_pages) +{ + STUB(); +} +#endif diff --git a/sys/dev/pci/drm/drm_cache.h b/sys/dev/pci/drm/drm_cache.h index 4642b588170..09c6166410d 100644 --- a/sys/dev/pci/drm/drm_cache.h +++ b/sys/dev/pci/drm/drm_cache.h @@ -1,4 +1,4 @@ -/* $OpenBSD: drm_cache.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ +/* $OpenBSD: drm_cache.h,v 1.4 2018/04/25 01:27:46 jsg Exp $ */ /************************************************************************** * * Copyright 2009 Red Hat Inc. @@ -38,4 +38,15 @@ void drm_clflush_pages(struct page *pages[], unsigned long num_pages); #endif +static inline bool drm_arch_can_wc_memory(void) +{ +#if defined(__powerpc__) + return false; +#elif defined(__mips__) + return false; +#else + return true; +#endif +} + #endif diff --git a/sys/dev/pci/drm/drm_linux.c b/sys/dev/pci/drm/drm_linux.c index 0245a7af73c..0cd7f50aa55 100644 --- a/sys/dev/pci/drm/drm_linux.c +++ b/sys/dev/pci/drm/drm_linux.c @@ -1,4 +1,4 @@ -/* $OpenBSD: drm_linux.c,v 1.22 2018/04/20 16:09:36 deraadt Exp $ */ +/* $OpenBSD: drm_linux.c,v 1.23 2018/04/25 01:27:46 jsg Exp $ */ /* * Copyright (c) 2013 Jonathan Gray * Copyright (c) 2015, 2016 Mark Kettenis @@ -20,6 +20,10 @@ #include #include +struct mutex sch_mtx = MUTEX_INITIALIZER(IPL_SCHED); +void *sch_ident; +int sch_priority; + void flush_barrier(void *arg) { @@ -79,7 +83,7 @@ flush_delayed_work(struct delayed_work *dwork) tsleep(&barrier, PWAIT, "fldwto", 1); task_set(&task, flush_barrier, &barrier); - task_add(dwork->tq, &task); + task_add(dwork->tq ? dwork->tq : systq, &task); while (!barrier) { sleep_setup(&sls, &barrier, PWAIT, "fldwbar"); sleep_finish(&sls, !barrier); @@ -791,3 +795,11 @@ drm_sysfs_hotplug_event(struct drm_device *dev) { KNOTE(&dev->note, NOTE_CHANGE); } + +unsigned int drm_fence_count; + +unsigned int +fence_context_alloc(unsigned int num) +{ + return __sync_add_and_fetch(&drm_fence_count, num) - num; +} diff --git a/sys/dev/pci/drm/drm_linux.h b/sys/dev/pci/drm/drm_linux.h index 5dead9a6040..a7ad53c3f49 100644 --- a/sys/dev/pci/drm/drm_linux.h +++ b/sys/dev/pci/drm/drm_linux.h @@ -1,4 +1,4 @@ -/* $OpenBSD: drm_linux.h,v 1.87 2018/04/20 16:09:36 deraadt Exp $ */ +/* $OpenBSD: drm_linux.h,v 1.88 2018/04/25 01:27:46 jsg Exp $ */ /* * Copyright (c) 2013, 2014, 2015 Mark Kettenis * Copyright (c) 2017 Martin Pieuchot @@ -54,6 +54,8 @@ #pragma GCC diagnostic ignored "-Wformat-zero-length" #endif +#define STUB() do { printf("%s: stub\n", __func__); } while(0) + typedef int irqreturn_t; enum irqreturn { IRQ_NONE = 0, @@ -558,45 +560,72 @@ _spin_unlock_irqrestore(struct mutex *mtxp, __unused unsigned long flags #define free_irq(irq, dev) #define synchronize_irq(x) -#define fence_wait(x, y) -#define fence_put(x) +typedef struct wait_queue wait_queue_t; +struct wait_queue { + unsigned int flags; + void *private; + int (*func)(wait_queue_t *, unsigned, int, void *); +}; + +extern struct mutex sch_mtx; +extern void *sch_ident; +extern int sch_priority; struct wait_queue_head { struct mutex lock; unsigned int count; + struct wait_queue *_wq; }; typedef struct wait_queue_head wait_queue_head_t; +#define MAX_SCHEDULE_TIMEOUT (INT32_MAX) + static inline void init_waitqueue_head(wait_queue_head_t *wq) { mtx_init(&wq->lock, IPL_TTY); wq->count = 0; + wq->_wq = NULL; +} + +static inline void +__add_wait_queue(wait_queue_head_t *head, wait_queue_t *new) +{ + head->_wq = new; +} + +static inline void +__remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old) +{ + head->_wq = NULL; } #define __wait_event_intr_timeout(wq, condition, timo, prio) \ ({ \ long ret = timo; \ - mtx_enter(&(wq).lock); \ do { \ int deadline, __error; \ \ KASSERT(!cold); \ + \ + mtx_enter(&sch_mtx); \ atomic_inc_int(&(wq).count); \ deadline = ticks + ret; \ - __error = msleep(&wq, &(wq).lock, prio, "drmweti", ret); \ + __error = msleep(&wq, &sch_mtx, prio, "drmweti", ret); \ ret = deadline - ticks; \ atomic_dec_int(&(wq).count); \ if (__error == ERESTART || __error == EINTR) { \ ret = -ERESTARTSYS; \ + mtx_leave(&sch_mtx); \ break; \ } \ if (timo && (ret <= 0 || __error == EWOULDBLOCK)) { \ + mtx_leave(&sch_mtx); \ ret = ((condition)) ? 1 : 0; \ break; \ } \ + mtx_leave(&sch_mtx); \ } while (ret > 0 && !(condition)); \ - mtx_leave(&(wq).lock); \ ret; \ }) @@ -609,6 +638,14 @@ do { \ __wait_event_intr_timeout(wq, condition, 0, 0); \ } while (0) +#define wait_event_interruptible_locked(wq, condition) \ +({ \ + int __ret = 0; \ + if (!(condition)) \ + __ret = __wait_event_intr_timeout(wq, condition, 0, PCATCH); \ + __ret; \ +}) + /* * Sleep until `condition' gets true or `timo' expires. * @@ -639,16 +676,43 @@ do { \ __ret; \ }) -#define wake_up(wq) \ +static inline void +_wake_up(wait_queue_head_t *wq LOCK_FL_VARS) +{ + _mtx_enter(&wq->lock LOCK_FL_ARGS); + if (wq->_wq != NULL && wq->_wq->func != NULL) + wq->_wq->func(wq->_wq, 0, wq->_wq->flags, NULL); + else { + mtx_enter(&sch_mtx); + wakeup(wq); + mtx_leave(&sch_mtx); + } + _mtx_leave(&wq->lock LOCK_FL_ARGS); +} + +#define wake_up_process(task) \ do { \ - mtx_enter(&(wq)->lock); \ - wakeup(wq); \ - mtx_leave(&(wq)->lock); \ + mtx_enter(&sch_mtx); \ + wakeup(task); \ + mtx_leave(&sch_mtx); \ } while (0) -#define wake_up_all(wq) wake_up(wq) -#define wake_up_all_locked(wq) wakeup(wq) -#define wake_up_interruptible(wq) wake_up(wq) +#define wake_up(wq) _wake_up(wq LOCK_FILE_LINE) +#define wake_up_all(wq) _wake_up(wq LOCK_FILE_LINE) + +static inline void +wake_up_all_locked(wait_queue_head_t *wq) +{ + if (wq->_wq != NULL && wq->_wq->func != NULL) + wq->_wq->func(wq->_wq, 0, wq->_wq->flags, NULL); + else { + mtx_enter(&sch_mtx); + wakeup(wq); + mtx_leave(&sch_mtx); + } +} + +#define wake_up_interruptible(wq) _wake_up(wq LOCK_FILE_LINE) #define waitqueue_active(wq) ((wq)->count > 0) struct completion { @@ -848,6 +912,7 @@ typedef void *async_cookie_t; #define TASK_UNINTERRUPTIBLE 0 #define TASK_INTERRUPTIBLE PCATCH +#define TASK_RUNNING -1 #define signal_pending_state(x, y) CURSIG(curproc) #define signal_pending(y) CURSIG(curproc) @@ -874,6 +939,7 @@ timespec_sub(struct timespec t1, struct timespec t2) #define time_in_range(x, min, max) ((x) >= (min) && (x) <= (max)) extern volatile unsigned long jiffies; +#define jiffies_64 jiffies /* XXX */ #undef HZ #define HZ hz @@ -1218,27 +1284,300 @@ static inline void prepare_to_wait(wait_queue_head_t *wq, wait_queue_head_t **wait, int state) { if (*wait == NULL) { - mtx_enter(&wq->lock); + mtx_enter(&sch_mtx); *wait = wq; } + MUTEX_ASSERT_LOCKED(&sch_mtx); + sch_ident = wq; + sch_priority = state; } static inline void finish_wait(wait_queue_head_t *wq, wait_queue_head_t **wait) { - if (*wait) - mtx_leave(&wq->lock); + if (*wait) { + MUTEX_ASSERT_LOCKED(&sch_mtx); + sch_ident = NULL; + mtx_leave(&sch_mtx); + } +} + +static inline void +set_current_state(int state) +{ + if (sch_ident != curproc) + mtx_enter(&sch_mtx); + MUTEX_ASSERT_LOCKED(&sch_mtx); + sch_ident = curproc; + sch_priority = state; +} + +static inline void +__set_current_state(int state) +{ + KASSERT(state == TASK_RUNNING); + if (sch_ident == curproc) { + MUTEX_ASSERT_LOCKED(&sch_mtx); + sch_ident = NULL; + mtx_leave(&sch_mtx); + } } static inline long -schedule_timeout(long timeout, wait_queue_head_t **wait) +schedule_timeout(long timeout) { + int err; + long deadline; + if (cold) { delay((timeout * 1000000) / hz); - return -ETIMEDOUT; + return 0; + } + + if (timeout == MAX_SCHEDULE_TIMEOUT) { + err = msleep(sch_ident, &sch_mtx, sch_priority, "schto", 0); + sch_ident = curproc; + return timeout; + } + + deadline = ticks + timeout; + err = msleep(sch_ident, &sch_mtx, sch_priority, "schto", timeout); + timeout = deadline - ticks; + if (timeout < 0) + timeout = 0; + sch_ident = curproc; + return timeout; +} + +struct seq_file; + +static inline void +seq_printf(struct seq_file *m, const char *fmt, ...) {}; + +#define preempt_enable() +#define preempt_disable() + +#define FENCE_TRACE(fence, fmt, args...) do {} while(0) + +struct fence { + struct kref refcount; + const struct fence_ops *ops; + unsigned long flags; + unsigned int context; + unsigned int seqno; + struct mutex *lock; + struct list_head cb_list; +}; + +enum fence_flag_bits { + FENCE_FLAG_SIGNALED_BIT, + FENCE_FLAG_ENABLE_SIGNAL_BIT, + FENCE_FLAG_USER_BITS, +}; + +struct fence_ops { + const char * (*get_driver_name)(struct fence *); + const char * (*get_timeline_name)(struct fence *); + bool (*enable_signaling)(struct fence *); + bool (*signaled)(struct fence *); + long (*wait)(struct fence *, bool, long); + void (*release)(struct fence *); +}; + +struct fence_cb; +typedef void (*fence_func_t)(struct fence *fence, struct fence_cb *cb); + +struct fence_cb { + struct list_head node; + fence_func_t func; +}; + +unsigned int fence_context_alloc(unsigned int); + +static inline struct fence * +fence_get(struct fence *fence) +{ + if (fence) + kref_get(&fence->refcount); + return fence; +} + +static inline struct fence * +fence_get_rcu(struct fence *fence) +{ + if (fence) + kref_get(&fence->refcount); + return fence; +} + +static inline void +fence_release(struct kref *ref) +{ + struct fence *fence = container_of(ref, struct fence, refcount); + if (fence->ops && fence->ops->release) + fence->ops->release(fence); + else + free(fence, M_DRM, 0); +} + +static inline void +fence_put(struct fence *fence) +{ + if (fence) + kref_put(&fence->refcount, fence_release); +} + +static inline int +fence_signal(struct fence *fence) +{ + if (fence == NULL) + return -EINVAL; + + if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return -EINVAL; + + if (test_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) { + struct fence_cb *cur, *tmp; + + mtx_enter(fence->lock); + list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) { + list_del_init(&cur->node); + cur->func(fence, cur); + } + mtx_leave(fence->lock); + } + + return 0; +} + +static inline int +fence_signal_locked(struct fence *fence) +{ + struct fence_cb *cur, *tmp; + + if (fence == NULL) + return -EINVAL; + + if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return -EINVAL; + + list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) { + list_del_init(&cur->node); + cur->func(fence, cur); + } + + return 0; +} + +static inline bool +fence_is_signaled(struct fence *fence) +{ + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return true; + + if (fence->ops->signaled && fence->ops->signaled(fence)) { + fence_signal(fence); + return true; } - return -msleep(*wait, &(*wait)->lock, PZERO, "schto", timeout); + return false; +} + +static inline long +fence_wait_timeout(struct fence *fence, bool intr, signed long timeout) +{ + if (timeout < 0) + return -EINVAL; + + if (timeout == 0) + return fence_is_signaled(fence); + + return fence->ops->wait(fence, intr, timeout); +} + +static inline long +fence_wait(struct fence *fence, bool intr) +{ + return fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT); +} + +static inline void +fence_enable_sw_signaling(struct fence *fence) +{ + if (!test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) && + !test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { + mtx_enter(fence->lock); + if (!fence->ops->enable_signaling(fence)) + fence_signal_locked(fence); + mtx_leave(fence->lock); + } +} + +static inline void +fence_init(struct fence *fence, const struct fence_ops *ops, + struct mutex *lock, unsigned context, unsigned seqno) +{ + fence->ops = ops; + fence->lock = lock; + fence->context = context; + fence->seqno = seqno; + fence->flags = 0; + kref_init(&fence->refcount); + INIT_LIST_HEAD(&fence->cb_list); +} + +static inline int +fence_add_callback(struct fence *fence, struct fence_cb *cb, + fence_func_t func) +{ + int ret = 0; + bool was_set; + + if (WARN_ON(!fence || !func)) + return -EINVAL; + + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { + INIT_LIST_HEAD(&cb->node); + return -ENOENT; + } + + mtx_enter(fence->lock); + + was_set = test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags); + + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + ret = -ENOENT; + else if (!was_set) { + if (!fence->ops->enable_signaling(fence)) { + fence_signal_locked(fence); + ret = -ENOENT; + } + } + + if (!ret) { + cb->func = func; + list_add_tail(&cb->node, &fence->cb_list); + } else + INIT_LIST_HEAD(&cb->node); + mtx_leave(fence->lock); + + return ret; +} + +static inline bool +fence_remove_callback(struct fence *fence, struct fence_cb *cb) +{ + bool ret; + + mtx_enter(fence->lock); + + ret = !list_empty(&cb->node); + if (ret) + list_del_init(&cb->node); + + mtx_leave(fence->lock); + + return ret; } struct idr_entry { @@ -1864,6 +2203,55 @@ cpu_relax(void) #define cpu_has_pat 1 #define cpu_has_clflush 1 +struct lock_class_key { +}; + +typedef struct { + unsigned int sequence; +} seqcount_t; + +static inline void +__seqcount_init(seqcount_t *s, const char *name, + struct lock_class_key *key) +{ + s->sequence = 0; +} + +static inline unsigned int +read_seqcount_begin(const seqcount_t *s) +{ + unsigned int r; + for (;;) { + r = s->sequence; + if ((r & 1) == 0) + break; + cpu_relax(); + } + membar_consumer(); + return r; +} + +static inline int +read_seqcount_retry(const seqcount_t *s, unsigned start) +{ + membar_consumer(); + return (s->sequence != start); +} + +static inline void +write_seqcount_begin(seqcount_t *s) +{ + s->sequence++; + membar_producer(); +} + +static inline void +write_seqcount_end(seqcount_t *s) +{ + membar_producer(); + s->sequence++; +} + static inline uint32_t ror32(uint32_t word, unsigned int shift) { return (word >> shift) | (word << (32 - shift)); diff --git a/sys/dev/pci/drm/drm_linux_rbtree.h b/sys/dev/pci/drm/drm_linux_rbtree.h index 864feb3b0b3..0245dcedccf 100644 --- a/sys/dev/pci/drm/drm_linux_rbtree.h +++ b/sys/dev/pci/drm/drm_linux_rbtree.h @@ -60,8 +60,10 @@ RB_PROTOTYPE(linux_root, rb_node, __entry, panic_cmp); #define rb_set_parent(r, p) rb_parent((r)) = (p) #define rb_set_color(r, c) rb_color((r)) = (c) #define rb_entry(ptr, type, member) container_of(ptr, type, member) +#define rb_entry_safe(ptr, type, member) \ + (ptr ? rb_entry(ptr, type, member) : NULL) -#define RB_EMPTY_ROOT(root) RB_EMPTY((struct linux_root *)root) +#define RB_EMPTY_ROOT(root) ((root)->rb_node == NULL) #define RB_EMPTY_NODE(node) (rb_parent(node) == node) #define RB_CLEAR_NODE(node) (rb_set_parent(node, node)) @@ -73,6 +75,12 @@ RB_PROTOTYPE(linux_root, rb_node, __entry, panic_cmp); #define rb_prev(node) RB_PREV(linux_root, NULL, (node)) #define rb_first(root) RB_MIN(linux_root, (struct linux_root *)(root)) #define rb_last(root) RB_MAX(linux_root, (struct linux_root *)(root)) +#define rbtree_postorder_for_each_entry_safe(x, y, head, member) \ + for ((x) = rb_entry_safe(RB_MIN(linux_root, (struct linux_root *)head), \ + __typeof(*x), member); \ + ((x) != NULL) && ({(y) = \ + rb_entry_safe(linux_root_RB_NEXT(&x->member), typeof(*x), member); 1; }); \ + (x) = (y)) static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, @@ -108,4 +116,36 @@ rb_replace_node(struct rb_node *victim, struct rb_node *new, #undef RB_ROOT #define RB_ROOT (struct rb_root) { NULL } +struct interval_tree_node { + struct rb_node rb; + unsigned long start; + unsigned long last; +}; + +static inline struct interval_tree_node * +interval_tree_iter_first(struct rb_root *root, + unsigned long start, unsigned long last) +{ +#ifdef DRMDEBUG + printf("%s: stub start: 0x%lx last: 0x%lx\n", __func__, start, last); +#endif + return NULL; +} + +static inline void +interval_tree_insert(struct interval_tree_node *node, struct rb_root *root) +{ +#ifdef DRMDEBUG + printf("%s: stub start: 0x%lx last: 0x%lx\n", __func__, node->start, node->last); +#endif +} + +static inline void +interval_tree_remove(struct interval_tree_node *node, struct rb_root *root) +{ +#ifdef DRMDEBUG + printf("%s: stub start: 0x%lx last: 0x%lx\n", __func__, node->start, node->last); +#endif +} + #endif /* _LINUX_RBTREE_H_ */ diff --git a/sys/dev/pci/drm/drm_pciids.h b/sys/dev/pci/drm/drm_pciids.h index ef64ba06202..8bc073d297d 100644 --- a/sys/dev/pci/drm/drm_pciids.h +++ b/sys/dev/pci/drm/drm_pciids.h @@ -1,4 +1,26 @@ #define radeon_PCI_IDS \ + {0x1002, 0x1304, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1305, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1306, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1307, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1309, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x130A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x130B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x130C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x130D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x130E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x130F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1310, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1311, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1312, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1313, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1315, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1316, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1317, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x1318, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x131B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x131C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x131D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ {0x1002, 0x3150, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV380|RADEON_IS_MOBILITY}, \ {0x1002, 0x3151, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV380|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ {0x1002, 0x3152, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV380|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ @@ -138,6 +160,40 @@ {0x1002, 0x5e4c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV410|RADEON_NEW_MEMMAP}, \ {0x1002, 0x5e4d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV410|RADEON_NEW_MEMMAP}, \ {0x1002, 0x5e4f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV410|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6600, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6601, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6602, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6603, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6604, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6605, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6606, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6607, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6608, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6610, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6611, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6613, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6617, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6620, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6621, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6623, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6631, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6640, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6641, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6646, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6647, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6649, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6650, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6651, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6658, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x665c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x665d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x665f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6660, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6663, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6664, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6665, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x6667, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x666F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ {0x1002, 0x6700, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAYMAN|RADEON_NEW_MEMMAP}, \ {0x1002, 0x6701, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAYMAN|RADEON_NEW_MEMMAP}, \ {0x1002, 0x6702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAYMAN|RADEON_NEW_MEMMAP}, \ @@ -212,6 +268,18 @@ {0x1002, 0x679B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI|RADEON_NEW_MEMMAP}, \ {0x1002, 0x679E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI|RADEON_NEW_MEMMAP}, \ {0x1002, 0x679F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67A0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67A1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67A2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67A8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67A9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67AA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67B0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67B1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67B8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67B9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67BA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x67BE, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII|RADEON_NEW_MEMMAP}, \ {0x1002, 0x6800, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ {0x1002, 0x6801, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ {0x1002, 0x6802, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ @@ -236,6 +304,7 @@ {0x1002, 0x6829, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|RADEON_NEW_MEMMAP}, \ {0x1002, 0x682A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ {0x1002, 0x682B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ + {0x1002, 0x682C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|RADEON_NEW_MEMMAP}, \ {0x1002, 0x682D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ {0x1002, 0x682F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ {0x1002, 0x6830, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ @@ -560,6 +629,38 @@ {0x1002, 0x9808, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ {0x1002, 0x9809, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ {0x1002, 0x980A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9830, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9831, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9832, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9833, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9834, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9835, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9836, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9837, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9838, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9839, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x983a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x983b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x983c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x983d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x983e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x983f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9850, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9851, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9852, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9853, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9854, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9855, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9856, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9857, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9858, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x9859, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x985A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x985B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x985C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x985D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x985E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ + {0x1002, 0x985F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ {0x1002, 0x9900, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ARUBA|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ {0x1002, 0x9901, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ARUBA|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ {0x1002, 0x9903, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ARUBA|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ @@ -646,29 +747,6 @@ {0x102b, 0x2527, PCI_ANY_ID, PCI_ANY_ID, 0, 0, MGA_CARD_TYPE_G550}, \ {0, 0, 0} -#define mach64_PCI_IDS \ - {0x1002, 0x4749, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4750, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4751, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4742, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4744, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4c49, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4c50, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4c51, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4c42, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4c44, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x474c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x474f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4752, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4753, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x474d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x474e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4c52, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4c53, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4c4d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0x1002, 0x4c4e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0, 0, 0} - #define sisdrv_PCI_IDS \ {0x1039, 0x0300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ {0x1039, 0x5300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ @@ -708,10 +786,6 @@ {0x8086, 0x1132, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ {0, 0, 0} -#define gamma_PCI_IDS \ - {0x3d3d, 0x0008, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, \ - {0, 0, 0} - #define savage_PCI_IDS \ {0x5333, 0x8a20, PCI_ANY_ID, PCI_ANY_ID, 0, 0, S3_SAVAGE3D}, \ {0x5333, 0x8a21, PCI_ANY_ID, PCI_ANY_ID, 0, 0, S3_SAVAGE3D}, \ @@ -737,6 +811,3 @@ {0x5333, 0x8d03, PCI_ANY_ID, PCI_ANY_ID, 0, 0, S3_PROSAVAGEDDR}, \ {0x5333, 0x8d04, PCI_ANY_ID, PCI_ANY_ID, 0, 0, S3_PROSAVAGEDDR}, \ {0, 0, 0} - -#define ffb_PCI_IDS \ - {0, 0, 0} diff --git a/sys/dev/pci/drm/files.drm b/sys/dev/pci/drm/files.drm index 79e0d686a61..b670817f87d 100644 --- a/sys/dev/pci/drm/files.drm +++ b/sys/dev/pci/drm/files.drm @@ -1,5 +1,5 @@ # $NetBSD: files.drm,v 1.2 2007/03/28 11:29:37 jmcneill Exp $ -# $OpenBSD: files.drm,v 1.40 2018/04/20 16:09:36 deraadt Exp $ +# $OpenBSD: files.drm,v 1.41 2018/04/25 01:27:46 jsg Exp $ # direct rendering modules define drmbase {[console = -1]} @@ -28,12 +28,13 @@ file dev/pci/drm/drm_bridge.c drm file dev/pci/drm/drm_gem.c drm file dev/pci/drm/drm_vma_manager.c drm file dev/pci/drm/drm_linux.c drm -file dev/pci/drm/drm_cache.c drm & inteldrm +file dev/pci/drm/drm_cache.c drm file dev/pci/drm/drm_panel.c drm & inteldrm file dev/pci/drm/drm_dp_mst_topology.c drm file dev/pci/drm/drm_mipi_dsi.c drm & inteldrm file dev/pci/drm/linux_hdmi.c drm file dev/pci/drm/linux_list_sort.c drm +file dev/pci/drm/linux_reservation.c drm define ttm file dev/pci/drm/ttm/ttm_agp_backend.c ttm & agp @@ -42,10 +43,12 @@ file dev/pci/drm/ttm/ttm_bo_manager.c ttm file dev/pci/drm/ttm/ttm_bo_util.c ttm file dev/pci/drm/ttm/ttm_bo_vm.c ttm file dev/pci/drm/ttm/ttm_execbuf_util.c ttm -file dev/pci/drm/ttm/ttm_lock.c ttm +#file dev/pci/drm/ttm/ttm_lock.c ttm file dev/pci/drm/ttm/ttm_memory.c ttm -file dev/pci/drm/ttm/ttm_object.c ttm +#file dev/pci/drm/ttm/ttm_module.c ttm +#file dev/pci/drm/ttm/ttm_object.c ttm file dev/pci/drm/ttm/ttm_page_alloc.c ttm +file dev/pci/drm/ttm/ttm_page_alloc_dma.c ttm file dev/pci/drm/ttm/ttm_tt.c ttm device inteldrm: agpint, drmbase, wsemuldisplaydev, rasops32, rasops_rotation, i2cbus, i2c_bitbang @@ -127,28 +130,42 @@ file dev/pci/drm/radeon/atombios_crtc.c radeondrm file dev/pci/drm/radeon/atombios_dp.c radeondrm file dev/pci/drm/radeon/atombios_encoders.c radeondrm file dev/pci/drm/radeon/atombios_i2c.c radeondrm +file dev/pci/drm/radeon/btc_dpm.c radeondrm file dev/pci/drm/radeon/cayman_blit_shaders.c radeondrm +file dev/pci/drm/radeon/ci_dpm.c radeondrm +file dev/pci/drm/radeon/ci_smc.c radeondrm +file dev/pci/drm/radeon/cik.c radeondrm +file dev/pci/drm/radeon/cik_blit_shaders.c radeondrm +file dev/pci/drm/radeon/cik_sdma.c radeondrm +file dev/pci/drm/radeon/cypress_dpm.c radeondrm +file dev/pci/drm/radeon/dce3_1_afmt.c radeondrm +file dev/pci/drm/radeon/dce6_afmt.c radeondrm file dev/pci/drm/radeon/evergreen.c radeondrm -file dev/pci/drm/radeon/evergreen_blit_kms.c radeondrm file dev/pci/drm/radeon/evergreen_blit_shaders.c radeondrm file dev/pci/drm/radeon/evergreen_cs.c radeondrm +file dev/pci/drm/radeon/evergreen_dma.c radeondrm file dev/pci/drm/radeon/evergreen_hdmi.c radeondrm +file dev/pci/drm/radeon/kv_dpm.c radeondrm +file dev/pci/drm/radeon/kv_smc.c radeondrm file dev/pci/drm/radeon/ni.c radeondrm +file dev/pci/drm/radeon/ni_dma.c radeondrm +file dev/pci/drm/radeon/ni_dpm.c radeondrm file dev/pci/drm/radeon/r100.c radeondrm file dev/pci/drm/radeon/r200.c radeondrm file dev/pci/drm/radeon/r300.c radeondrm file dev/pci/drm/radeon/r420.c radeondrm file dev/pci/drm/radeon/r520.c radeondrm file dev/pci/drm/radeon/r600.c radeondrm -file dev/pci/drm/radeon/r600_audio.c radeondrm -file dev/pci/drm/radeon/r600_blit_kms.c radeondrm file dev/pci/drm/radeon/r600_blit_shaders.c radeondrm file dev/pci/drm/radeon/r600_cs.c radeondrm +file dev/pci/drm/radeon/r600_dma.c radeondrm +file dev/pci/drm/radeon/r600_dpm.c radeondrm file dev/pci/drm/radeon/r600_hdmi.c radeondrm #file dev/pci/drm/radeon/radeon_acpi.c radeondrm file dev/pci/drm/radeon/radeon_agp.c radeondrm file dev/pci/drm/radeon/radeon_asic.c radeondrm file dev/pci/drm/radeon/radeon_atombios.c radeondrm +file dev/pci/drm/radeon/radeon_audio.c radeondrm file dev/pci/drm/radeon/radeon_benchmark.c radeondrm file dev/pci/drm/radeon/radeon_bios.c radeondrm file dev/pci/drm/radeon/radeon_clocks.c radeondrm @@ -159,30 +176,61 @@ file dev/pci/drm/radeon/radeon_cursor.c radeondrm file dev/pci/drm/radeon/radeon_device.c radeondrm file dev/pci/drm/radeon/radeon_display.c radeondrm file dev/pci/drm/radeon/radeon_dp_auxch.c radeondrm +file dev/pci/drm/radeon/radeon_dp_mst.c radeondrm +file dev/pci/drm/radeon/radeon_drv.c radeondrm file dev/pci/drm/radeon/radeon_encoders.c radeondrm file dev/pci/drm/radeon/radeon_fb.c radeondrm file dev/pci/drm/radeon/radeon_fence.c radeondrm file dev/pci/drm/radeon/radeon_gart.c radeondrm file dev/pci/drm/radeon/radeon_gem.c radeondrm file dev/pci/drm/radeon/radeon_i2c.c radeondrm +file dev/pci/drm/radeon/radeon_ib.c radeondrm +#file dev/pci/drm/radeon/radeon_ioc32.c radeondrm COMPAT file dev/pci/drm/radeon/radeon_irq_kms.c radeondrm +#file dev/pci/drm/radeon/radeon_kfd.c radeondrm file dev/pci/drm/radeon/radeon_kms.c radeondrm file dev/pci/drm/radeon/radeon_legacy_crtc.c radeondrm file dev/pci/drm/radeon/radeon_legacy_encoders.c radeondrm file dev/pci/drm/radeon/radeon_legacy_tv.c radeondrm +#file dev/pci/drm/radeon/radeon_mn.c radeondrm not needed? file dev/pci/drm/radeon/radeon_object.c radeondrm file dev/pci/drm/radeon/radeon_pm.c radeondrm #file dev/pci/drm/radeon/radeon_prime.c radeondrm file dev/pci/drm/radeon/radeon_ring.c radeondrm file dev/pci/drm/radeon/radeon_sa.c radeondrm file dev/pci/drm/radeon/radeon_semaphore.c radeondrm +file dev/pci/drm/radeon/radeon_sync.c radeondrm file dev/pci/drm/radeon/radeon_test.c radeondrm #file dev/pci/drm/radeon/radeon_trace_points.c radeondrm file dev/pci/drm/radeon/radeon_ttm.c radeondrm +file dev/pci/drm/radeon/radeon_ucode.c radeondrm +file dev/pci/drm/radeon/radeon_uvd.c radeondrm +file dev/pci/drm/radeon/radeon_vce.c radeondrm +file dev/pci/drm/radeon/radeon_vm.c radeondrm file dev/pci/drm/radeon/rs400.c radeondrm file dev/pci/drm/radeon/rs600.c radeondrm file dev/pci/drm/radeon/rs690.c radeondrm +file dev/pci/drm/radeon/rs780_dpm.c radeondrm file dev/pci/drm/radeon/rv515.c radeondrm +file dev/pci/drm/radeon/rv6xx_dpm.c radeondrm +file dev/pci/drm/radeon/rv730_dpm.c radeondrm +file dev/pci/drm/radeon/rv740_dpm.c radeondrm file dev/pci/drm/radeon/rv770.c radeondrm +file dev/pci/drm/radeon/rv770_dma.c radeondrm +file dev/pci/drm/radeon/rv770_dpm.c radeondrm +file dev/pci/drm/radeon/rv770_smc.c radeondrm file dev/pci/drm/radeon/si.c radeondrm file dev/pci/drm/radeon/si_blit_shaders.c radeondrm +file dev/pci/drm/radeon/si_dma.c radeondrm +file dev/pci/drm/radeon/si_dpm.c radeondrm +file dev/pci/drm/radeon/si_smc.c radeondrm +file dev/pci/drm/radeon/sumo_dpm.c radeondrm +file dev/pci/drm/radeon/sumo_smc.c radeondrm +file dev/pci/drm/radeon/trinity_dpm.c radeondrm +file dev/pci/drm/radeon/trinity_smc.c radeondrm +file dev/pci/drm/radeon/uvd_v1_0.c radeondrm +file dev/pci/drm/radeon/uvd_v2_2.c radeondrm +file dev/pci/drm/radeon/uvd_v3_1.c radeondrm +file dev/pci/drm/radeon/uvd_v4_2.c radeondrm +file dev/pci/drm/radeon/vce_v1_0.c radeondrm +file dev/pci/drm/radeon/vce_v2_0.c radeondrm diff --git a/sys/dev/pci/drm/i915/intel_i2c.c b/sys/dev/pci/drm/i915/intel_i2c.c index 36428ad123b..6fb3bb9f240 100644 --- a/sys/dev/pci/drm/i915/intel_i2c.c +++ b/sys/dev/pci/drm/i915/intel_i2c.c @@ -390,7 +390,7 @@ gmbus_wait_hw_status(struct drm_i915_private *dev_priv, if (gmbus2 & (GMBUS_SATOER | gmbus2_status)) break; - schedule_timeout(1, &wait); + schedule_timeout(1); } finish_wait(&dev_priv->gmbus_wait_queue, &wait); diff --git a/sys/dev/pci/drm/i915/intel_sprite.c b/sys/dev/pci/drm/i915/intel_sprite.c index 9c0394934be..80c0cf968ed 100644 --- a/sys/dev/pci/drm/i915/intel_sprite.c +++ b/sys/dev/pci/drm/i915/intel_sprite.c @@ -128,7 +128,7 @@ void intel_pipe_update_start(struct intel_crtc *crtc) local_irq_enable(); - timeout = schedule_timeout(timeout, &wait); + timeout = schedule_timeout(timeout); local_irq_disable(); } diff --git a/sys/dev/pci/drm/linux_rcupdate.h b/sys/dev/pci/drm/linux_rcupdate.h new file mode 100644 index 00000000000..9fa8291c1f5 --- /dev/null +++ b/sys/dev/pci/drm/linux_rcupdate.h @@ -0,0 +1,18 @@ +/* Public domain. */ + +#ifndef LINUX_RCUPDATE_H +#define LINUX_RCUPDATE_H + +struct rcu_head { +}; + +#define __rcu +#define rcu_dereference(p) (p) +#define rcu_dereference_protected(p, c) (p) +#define RCU_INIT_POINTER(p, v) do { (p) = (v); } while(0) +#define rcu_read_lock() +#define rcu_read_unlock() + +#define kfree_rcu(objp, name) do { free((void *)objp, M_DRM, 0); } while(0) + +#endif diff --git a/sys/dev/pci/drm/linux_reservation.c b/sys/dev/pci/drm/linux_reservation.c new file mode 100644 index 00000000000..82fda18f73d --- /dev/null +++ b/sys/dev/pci/drm/linux_reservation.c @@ -0,0 +1,495 @@ +/* + * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst) + * + * Based on bo.c which bears the following copyright notice, + * but is dual licensed: + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#include +#include +#include + +DEFINE_WW_CLASS(reservation_ww_class); +EXPORT_SYMBOL(reservation_ww_class); + +struct lock_class_key reservation_seqcount_class; +EXPORT_SYMBOL(reservation_seqcount_class); + +const char reservation_seqcount_string[] = "reservation_seqcount"; +EXPORT_SYMBOL(reservation_seqcount_string); +/* + * Reserve space to add a shared fence to a reservation_object, + * must be called with obj->lock held. + */ +int reservation_object_reserve_shared(struct reservation_object *obj) +{ + struct reservation_object_list *fobj, *old; + u32 max; + + old = reservation_object_get_list(obj); + + if (old && old->shared_max) { + if (old->shared_count < old->shared_max) { + /* perform an in-place update */ + kfree(obj->staged); + obj->staged = NULL; + return 0; + } else + max = old->shared_max * 2; + } else + max = 4; + + /* + * resize obj->staged or allocate if it doesn't exist, + * noop if already correct size + */ + fobj = kmalloc(offsetof(typeof(*fobj), shared[max]), GFP_KERNEL); + if (!fobj) + return -ENOMEM; + if (obj->staged != NULL) + memcpy(fobj, obj->staged, offsetof(typeof(*fobj), shared[max])); + kfree(obj->staged); + + obj->staged = fobj; + fobj->shared_max = max; + return 0; +} +EXPORT_SYMBOL(reservation_object_reserve_shared); + +static void +reservation_object_add_shared_inplace(struct reservation_object *obj, + struct reservation_object_list *fobj, + struct fence *fence) +{ + u32 i; + + fence_get(fence); + + preempt_disable(); + write_seqcount_begin(&obj->seq); + + for (i = 0; i < fobj->shared_count; ++i) { + struct fence *old_fence; + + old_fence = rcu_dereference_protected(fobj->shared[i], + reservation_object_held(obj)); + + if (old_fence->context == fence->context) { + /* memory barrier is added by write_seqcount_begin */ + RCU_INIT_POINTER(fobj->shared[i], fence); + write_seqcount_end(&obj->seq); + preempt_enable(); + + fence_put(old_fence); + return; + } + } + + /* + * memory barrier is added by write_seqcount_begin, + * fobj->shared_count is protected by this lock too + */ + RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); + fobj->shared_count++; + + write_seqcount_end(&obj->seq); + preempt_enable(); +} + +static void +reservation_object_add_shared_replace(struct reservation_object *obj, + struct reservation_object_list *old, + struct reservation_object_list *fobj, + struct fence *fence) +{ + unsigned i; + struct fence *old_fence = NULL; + + fence_get(fence); + + if (!old) { + RCU_INIT_POINTER(fobj->shared[0], fence); + fobj->shared_count = 1; + goto done; + } + + /* + * no need to bump fence refcounts, rcu_read access + * requires the use of kref_get_unless_zero, and the + * references from the old struct are carried over to + * the new. + */ + fobj->shared_count = old->shared_count; + + for (i = 0; i < old->shared_count; ++i) { + struct fence *check; + + check = rcu_dereference_protected(old->shared[i], + reservation_object_held(obj)); + + if (!old_fence && check->context == fence->context) { + old_fence = check; + RCU_INIT_POINTER(fobj->shared[i], fence); + } else + RCU_INIT_POINTER(fobj->shared[i], check); + } + if (!old_fence) { + RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); + fobj->shared_count++; + } + +done: + preempt_disable(); + write_seqcount_begin(&obj->seq); + /* + * RCU_INIT_POINTER can be used here, + * seqcount provides the necessary barriers + */ + RCU_INIT_POINTER(obj->fence, fobj); + write_seqcount_end(&obj->seq); + preempt_enable(); + + if (old) + kfree_rcu(old, rcu); + + if (old_fence) + fence_put(old_fence); +} + +/* + * Add a fence to a shared slot, obj->lock must be held, and + * reservation_object_reserve_shared_fence has been called. + */ +void reservation_object_add_shared_fence(struct reservation_object *obj, + struct fence *fence) +{ + struct reservation_object_list *old, *fobj = obj->staged; + + old = reservation_object_get_list(obj); + obj->staged = NULL; + + if (!fobj) { + BUG_ON(old->shared_count >= old->shared_max); + reservation_object_add_shared_inplace(obj, old, fence); + } else + reservation_object_add_shared_replace(obj, old, fobj, fence); +} +EXPORT_SYMBOL(reservation_object_add_shared_fence); + +void reservation_object_add_excl_fence(struct reservation_object *obj, + struct fence *fence) +{ + struct fence *old_fence = reservation_object_get_excl(obj); + struct reservation_object_list *old; + u32 i = 0; + + old = reservation_object_get_list(obj); + if (old) + i = old->shared_count; + + if (fence) + fence_get(fence); + + preempt_disable(); + write_seqcount_begin(&obj->seq); + /* write_seqcount_begin provides the necessary memory barrier */ + RCU_INIT_POINTER(obj->fence_excl, fence); + if (old) + old->shared_count = 0; + write_seqcount_end(&obj->seq); + preempt_enable(); + + /* inplace update, no shared fences */ + while (i--) + fence_put(rcu_dereference_protected(old->shared[i], + reservation_object_held(obj))); + + if (old_fence) + fence_put(old_fence); +} +EXPORT_SYMBOL(reservation_object_add_excl_fence); + +int reservation_object_get_fences_rcu(struct reservation_object *obj, + struct fence **pfence_excl, + unsigned *pshared_count, + struct fence ***pshared) +{ + unsigned shared_count = 0; + unsigned retry = 1; + struct fence **shared = NULL, *fence_excl = NULL; + int ret = 0; + + while (retry) { + struct reservation_object_list *fobj; + unsigned seq; + + seq = read_seqcount_begin(&obj->seq); + + rcu_read_lock(); + + fobj = rcu_dereference(obj->fence); + if (fobj) { + struct fence **nshared; + size_t sz = sizeof(*shared) * fobj->shared_max; + + nshared = kmalloc(sz, GFP_NOWAIT | __GFP_NOWARN); + if (nshared != NULL && shared != NULL) + memcpy(nshared, shared, sz); + kfree(shared); + if (!nshared) { + rcu_read_unlock(); + nshared = kmalloc(sz, GFP_KERNEL); + if (nshared != NULL && shared != NULL) + memcpy(nshared, shared, sz); + kfree(shared); + if (nshared) { + shared = nshared; + continue; + } + + ret = -ENOMEM; + shared_count = 0; + break; + } + shared = nshared; + memcpy(shared, fobj->shared, sz); + shared_count = fobj->shared_count; + } else + shared_count = 0; + fence_excl = rcu_dereference(obj->fence_excl); + + retry = read_seqcount_retry(&obj->seq, seq); + if (retry) + goto unlock; + + if (!fence_excl || fence_get_rcu(fence_excl)) { + unsigned i; + + for (i = 0; i < shared_count; ++i) { + if (fence_get_rcu(shared[i])) + continue; + + /* uh oh, refcount failed, abort and retry */ + while (i--) + fence_put(shared[i]); + + if (fence_excl) { + fence_put(fence_excl); + fence_excl = NULL; + } + + retry = 1; + break; + } + } else + retry = 1; + +unlock: + rcu_read_unlock(); + } + *pshared_count = shared_count; + if (shared_count) + *pshared = shared; + else { + *pshared = NULL; + kfree(shared); + } + *pfence_excl = fence_excl; + + return ret; +} +#ifdef __linux__ +EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu); +#endif + +long reservation_object_wait_timeout_rcu(struct reservation_object *obj, + bool wait_all, bool intr, + unsigned long timeout) +{ + struct fence *fence; + unsigned seq, shared_count, i = 0; + long ret = timeout; + + if (!timeout) + return reservation_object_test_signaled_rcu(obj, wait_all); + +retry: + fence = NULL; + shared_count = 0; + seq = read_seqcount_begin(&obj->seq); + rcu_read_lock(); + + if (wait_all) { + struct reservation_object_list *fobj = + rcu_dereference(obj->fence); + + if (fobj) + shared_count = fobj->shared_count; + + if (read_seqcount_retry(&obj->seq, seq)) + goto unlock_retry; + + for (i = 0; i < shared_count; ++i) { + struct fence *lfence = rcu_dereference(fobj->shared[i]); + + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) + continue; + + if (!fence_get_rcu(lfence)) + goto unlock_retry; + + if (fence_is_signaled(lfence)) { + fence_put(lfence); + continue; + } + + fence = lfence; + break; + } + } + + if (!shared_count) { + struct fence *fence_excl = rcu_dereference(obj->fence_excl); + + if (read_seqcount_retry(&obj->seq, seq)) + goto unlock_retry; + + if (fence_excl && + !test_bit(FENCE_FLAG_SIGNALED_BIT, &fence_excl->flags)) { + if (!fence_get_rcu(fence_excl)) + goto unlock_retry; + + if (fence_is_signaled(fence_excl)) + fence_put(fence_excl); + else + fence = fence_excl; + } + } + + rcu_read_unlock(); + if (fence) { + ret = fence_wait_timeout(fence, intr, ret); + fence_put(fence); + if (ret > 0 && wait_all && (i + 1 < shared_count)) + goto retry; + } + return ret; + +unlock_retry: + rcu_read_unlock(); + goto retry; +} +#ifdef __linux__ +EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu); +#endif + + +static inline int +reservation_object_test_signaled_single(struct fence *passed_fence) +{ + struct fence *fence, *lfence = passed_fence; + int ret = 1; + + if (!test_bit(FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) { + fence = fence_get_rcu(lfence); + if (!fence) + return -1; + + ret = !!fence_is_signaled(fence); + fence_put(fence); + } + return ret; +} + +bool reservation_object_test_signaled_rcu(struct reservation_object *obj, + bool test_all) +{ + unsigned seq, shared_count; + int ret = true; + +retry: + shared_count = 0; + seq = read_seqcount_begin(&obj->seq); + rcu_read_lock(); + + if (test_all) { + unsigned i; + + struct reservation_object_list *fobj = + rcu_dereference(obj->fence); + + if (fobj) + shared_count = fobj->shared_count; + + if (read_seqcount_retry(&obj->seq, seq)) + goto unlock_retry; + + for (i = 0; i < shared_count; ++i) { + struct fence *fence = rcu_dereference(fobj->shared[i]); + + ret = reservation_object_test_signaled_single(fence); + if (ret < 0) + goto unlock_retry; + else if (!ret) + break; + } + + /* + * There could be a read_seqcount_retry here, but nothing cares + * about whether it's the old or newer fence pointers that are + * signaled. That race could still have happened after checking + * read_seqcount_retry. If you care, use ww_mutex_lock. + */ + } + + if (!shared_count) { + struct fence *fence_excl = rcu_dereference(obj->fence_excl); + + if (read_seqcount_retry(&obj->seq, seq)) + goto unlock_retry; + + if (fence_excl) { + ret = reservation_object_test_signaled_single( + fence_excl); + if (ret < 0) + goto unlock_retry; + } + } + + rcu_read_unlock(); + return ret; + +unlock_retry: + rcu_read_unlock(); + goto retry; +} +#ifdef __linux__ +EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu); +#endif diff --git a/sys/dev/pci/drm/linux_reservation.h b/sys/dev/pci/drm/linux_reservation.h new file mode 100644 index 00000000000..9647dfc54bc --- /dev/null +++ b/sys/dev/pci/drm/linux_reservation.h @@ -0,0 +1,140 @@ +/* + * Header file for reservations for dma-buf and ttm + * + * Copyright(C) 2011 Linaro Limited. All rights reserved. + * Copyright (C) 2012-2013 Canonical Ltd + * Copyright (C) 2012 Texas Instruments + * + * Authors: + * Rob Clark + * Maarten Lankhorst + * Thomas Hellstrom + * + * Based on bo.c which bears the following copyright notice, + * but is dual licensed: + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef _LINUX_RESERVATION_H +#define _LINUX_RESERVATION_H + +#include +#include +#include + +extern struct ww_class reservation_ww_class; +extern struct lock_class_key reservation_seqcount_class; +extern const char reservation_seqcount_string[]; + +struct reservation_object_list { + struct rcu_head rcu; + u32 shared_count, shared_max; + struct fence __rcu *shared[]; +}; + +struct reservation_object { + struct ww_mutex lock; + seqcount_t seq; + + struct fence __rcu *fence_excl; + struct reservation_object_list __rcu *fence; + struct reservation_object_list *staged; +}; + +#define reservation_object_held(obj) lockdep_is_held(&(obj)->lock.base) +#define reservation_object_assert_held(obj) \ + lockdep_assert_held(&(obj)->lock.base) + +static inline void +reservation_object_init(struct reservation_object *obj) +{ + ww_mutex_init(&obj->lock, &reservation_ww_class); + + __seqcount_init(&obj->seq, reservation_seqcount_string, &reservation_seqcount_class); + RCU_INIT_POINTER(obj->fence, NULL); + RCU_INIT_POINTER(obj->fence_excl, NULL); + obj->staged = NULL; +} + +static inline void +reservation_object_fini(struct reservation_object *obj) +{ + int i; + struct reservation_object_list *fobj; + struct fence *excl; + + /* + * This object should be dead and all references must have + * been released to it, so no need to be protected with rcu. + */ + excl = rcu_dereference_protected(obj->fence_excl, 1); + if (excl) + fence_put(excl); + + fobj = rcu_dereference_protected(obj->fence, 1); + if (fobj) { + for (i = 0; i < fobj->shared_count; ++i) + fence_put(rcu_dereference_protected(fobj->shared[i], 1)); + + kfree(fobj); + } + kfree(obj->staged); + + ww_mutex_destroy(&obj->lock); +} + +static inline struct reservation_object_list * +reservation_object_get_list(struct reservation_object *obj) +{ + return rcu_dereference_protected(obj->fence, + reservation_object_held(obj)); +} + +static inline struct fence * +reservation_object_get_excl(struct reservation_object *obj) +{ + return rcu_dereference_protected(obj->fence_excl, + reservation_object_held(obj)); +} + +int reservation_object_reserve_shared(struct reservation_object *obj); +void reservation_object_add_shared_fence(struct reservation_object *obj, + struct fence *fence); + +void reservation_object_add_excl_fence(struct reservation_object *obj, + struct fence *fence); + +int reservation_object_get_fences_rcu(struct reservation_object *obj, + struct fence **pfence_excl, + unsigned *pshared_count, + struct fence ***pshared); + +long reservation_object_wait_timeout_rcu(struct reservation_object *obj, + bool wait_all, bool intr, + unsigned long timeout); + +bool reservation_object_test_signaled_rcu(struct reservation_object *obj, + bool test_all); + +#endif /* _LINUX_RESERVATION_H */ diff --git a/sys/dev/pci/drm/radeon/ObjectID.h b/sys/dev/pci/drm/radeon/ObjectID.h index f84a5ae68cf..06192698bd9 100644 --- a/sys/dev/pci/drm/radeon/ObjectID.h +++ b/sys/dev/pci/drm/radeon/ObjectID.h @@ -1,4 +1,3 @@ -/* $OpenBSD: ObjectID.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2006-2007 Advanced Micro Devices, Inc. * @@ -70,6 +69,8 @@ #define ENCODER_OBJECT_ID_ALMOND 0x22 #define ENCODER_OBJECT_ID_TRAVIS 0x23 #define ENCODER_OBJECT_ID_NUTMEG 0x22 +#define ENCODER_OBJECT_ID_HDMI_ANX9805 0x26 + /* Kaleidoscope (KLDSCP) Class Display Hardware (internal) */ #define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1 0x13 #define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1 0x14 @@ -87,6 +88,8 @@ #define ENCODER_OBJECT_ID_INTERNAL_UNIPHY1 0x20 #define ENCODER_OBJECT_ID_INTERNAL_UNIPHY2 0x21 #define ENCODER_OBJECT_ID_INTERNAL_VCE 0x24 +#define ENCODER_OBJECT_ID_INTERNAL_UNIPHY3 0x25 +#define ENCODER_OBJECT_ID_INTERNAL_AMCLK 0x27 #define ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO 0xFF @@ -365,6 +368,14 @@ GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ ENCODER_OBJECT_ID_INTERNAL_UNIPHY2 << OBJECT_ID_SHIFT) +#define ENCODER_INTERNAL_UNIPHY3_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY3 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_UNIPHY3_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY3 << OBJECT_ID_SHIFT) + #define ENCODER_GENERAL_EXTERNAL_DVO_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO << OBJECT_ID_SHIFT) @@ -393,6 +404,10 @@ GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ ENCODER_OBJECT_ID_INTERNAL_VCE << OBJECT_ID_SHIFT) +#define ENCODER_HDMI_ANX9805_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_HDMI_ANX9805 << OBJECT_ID_SHIFT) + /****************************************************/ /* Connector Object ID definition - Shared with BIOS */ /****************************************************/ @@ -462,6 +477,14 @@ GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\ CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) +#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID5 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID5 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID6 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID6 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) + #define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT) @@ -474,6 +497,10 @@ GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\ CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT) +#define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID4 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT) + #define CONNECTOR_VGA_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ CONNECTOR_OBJECT_ID_VGA << OBJECT_ID_SHIFT) @@ -542,6 +569,18 @@ GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\ CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID4 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID5 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID5 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID6 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID6 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) + #define CONNECTOR_HDMI_TYPE_B_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ CONNECTOR_OBJECT_ID_HDMI_TYPE_B << OBJECT_ID_SHIFT) diff --git a/sys/dev/pci/drm/radeon/atom-bits.h b/sys/dev/pci/drm/radeon/atom-bits.h index 3afe16b2d61..e8fae5c7751 100644 --- a/sys/dev/pci/drm/radeon/atom-bits.h +++ b/sys/dev/pci/drm/radeon/atom-bits.h @@ -1,4 +1,3 @@ -/* $OpenBSD: atom-bits.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * diff --git a/sys/dev/pci/drm/radeon/atom-names.h b/sys/dev/pci/drm/radeon/atom-names.h index 974590b93d1..6f907a5ffa5 100644 --- a/sys/dev/pci/drm/radeon/atom-names.h +++ b/sys/dev/pci/drm/radeon/atom-names.h @@ -1,4 +1,3 @@ -/* $OpenBSD: atom-names.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * diff --git a/sys/dev/pci/drm/radeon/atom-types.h b/sys/dev/pci/drm/radeon/atom-types.h index 4641ee47561..1125b866cdb 100644 --- a/sys/dev/pci/drm/radeon/atom-types.h +++ b/sys/dev/pci/drm/radeon/atom-types.h @@ -1,4 +1,3 @@ -/* $OpenBSD: atom-types.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Red Hat Inc. * diff --git a/sys/dev/pci/drm/radeon/atom.c b/sys/dev/pci/drm/radeon/atom.c index 9331697f9f6..1ce19696ee0 100644 --- a/sys/dev/pci/drm/radeon/atom.c +++ b/sys/dev/pci/drm/radeon/atom.c @@ -1,4 +1,3 @@ -/* $OpenBSD: atom.c,v 1.11 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * @@ -92,8 +91,8 @@ static void debug_print_spaces(int n) #undef DEBUG #endif -#define DEBUG(...) do if (atom_debug) { printk(__FILE__ __VA_ARGS__); } while (0) -#define SDEBUG(...) do if (atom_debug) { printk(__FILE__); debug_print_spaces(debug_depth); printf(__VA_ARGS__); } while (0) +#define DEBUG(...) do if (atom_debug) { printk(KERN_DEBUG __VA_ARGS__); } while (0) +#define SDEBUG(...) do if (atom_debug) { printk(KERN_DEBUG); debug_print_spaces(debug_depth); printk(__VA_ARGS__); } while (0) #else #define DEBUG(...) do { } while (0) #define SDEBUG(...) do { } while (0) @@ -222,11 +221,7 @@ static uint32_t atom_get_src_int(atom_exec_context *ctx, uint8_t attr, (*ptr)++; /* get_unaligned_le32 avoids unaligned accesses from atombios * tables, noticed on a DEC Alpha. */ -#ifdef notyet val = get_unaligned_le32((u32 *)&ctx->ps[idx]); -#else - val = le32_to_cpu(ctx->ps[idx]); -#endif if (print) DEBUG("PS[0x%02X,0x%04X]", idx, val); break; @@ -1221,7 +1216,7 @@ free: return ret; } -int atom_execute_table(struct atom_context *ctx, int index, uint32_t * params) +int atom_execute_table_scratch_unlocked(struct atom_context *ctx, int index, uint32_t * params) { int r; @@ -1242,11 +1237,22 @@ int atom_execute_table(struct atom_context *ctx, int index, uint32_t * params) return r; } +int atom_execute_table(struct atom_context *ctx, int index, uint32_t * params) +{ + int r; + mutex_lock(&ctx->scratch_mutex); + r = atom_execute_table_scratch_unlocked(ctx, index, params); + mutex_unlock(&ctx->scratch_mutex); + return r; +} + static int atom_iio_len[] = { 1, 2, 3, 3, 3, 3, 4, 4, 4, 3 }; static void atom_index_iio(struct atom_context *ctx, int base) { ctx->iio = kzalloc(2 * 256, GFP_KERNEL); + if (!ctx->iio) + return; while (CU8(base) == ATOM_IIO_START) { ctx->iio[CU8(base + 1)] = base + 2; base += 2; @@ -1261,11 +1267,9 @@ struct atom_context *atom_parse(struct card_info *card, void *bios) int base; struct atom_context *ctx = kzalloc(sizeof(struct atom_context), GFP_KERNEL); -#ifdef DRMDEBUG char *str; char name[512]; int i; -#endif if (!ctx) return NULL; @@ -1298,8 +1302,11 @@ struct atom_context *atom_parse(struct card_info *card, void *bios) ctx->cmd_table = CU16(base + ATOM_ROM_CMD_PTR); ctx->data_table = CU16(base + ATOM_ROM_DATA_PTR); atom_index_iio(ctx, CU16(ctx->data_table + ATOM_DATA_IIO_PTR) + 4); + if (!ctx->iio) { + atom_destroy(ctx); + return NULL; + } -#ifdef DRMDEBUG str = CSTR(CU16(base + ATOM_ROM_MSG_PTR)); while (*str && ((*str == '\n') || (*str == '\r'))) str++; @@ -1311,6 +1318,7 @@ struct atom_context *atom_parse(struct card_info *card, void *bios) break; } } +#ifdef DRMDEBUG printk(KERN_INFO "ATOM BIOS: %s\n", name); #endif @@ -1348,8 +1356,7 @@ int atom_asic_init(struct atom_context *ctx) void atom_destroy(struct atom_context *ctx) { - if (ctx->iio) - kfree(ctx->iio); + kfree(ctx->iio); kfree(ctx); } diff --git a/sys/dev/pci/drm/radeon/atom.h b/sys/dev/pci/drm/radeon/atom.h index f9b9e031cc5..0d710ef330a 100644 --- a/sys/dev/pci/drm/radeon/atom.h +++ b/sys/dev/pci/drm/radeon/atom.h @@ -1,4 +1,3 @@ -/* $OpenBSD: atom.h,v 1.4 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * @@ -126,6 +125,7 @@ struct card_info { struct atom_context { struct card_info *card; struct rwlock mutex; + struct rwlock scratch_mutex; void *bios; uint32_t cmd_table, data_table; uint16_t *iio; @@ -146,6 +146,7 @@ extern int atom_debug; struct atom_context *atom_parse(struct card_info *, void *); int atom_execute_table(struct atom_context *, int, uint32_t *); +int atom_execute_table_scratch_unlocked(struct atom_context *, int, uint32_t *); int atom_asic_init(struct atom_context *); void atom_destroy(struct atom_context *); bool atom_parse_data_header(struct atom_context *ctx, int index, uint16_t *size, diff --git a/sys/dev/pci/drm/radeon/atombios.h b/sys/dev/pci/drm/radeon/atombios.h index 7655861e641..ab89eed9ddd 100644 --- a/sys/dev/pci/drm/radeon/atombios.h +++ b/sys/dev/pci/drm/radeon/atombios.h @@ -1,4 +1,3 @@ -/* $OpenBSD: atombios.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2006-2007 Advanced Micro Devices, Inc. * @@ -75,6 +74,8 @@ #define ATOM_PPLL2 1 #define ATOM_DCPLL 2 #define ATOM_PPLL0 2 +#define ATOM_PPLL3 3 + #define ATOM_EXT_PLL1 8 #define ATOM_EXT_PLL2 9 #define ATOM_EXT_CLOCK 10 @@ -260,7 +261,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{ USHORT AdjustDisplayPll; //Atomic Table, used by various SW componentes. USHORT AdjustMemoryController; //Atomic Table, indirectly used by various SW components,called from SetMemoryClock USHORT EnableASIC_StaticPwrMgt; //Atomic Table, only used by Bios - USHORT ASIC_StaticPwrMgtStatusChange; //Obsolete , only used by Bios + USHORT SetUniphyInstance; //Atomic Table, only used by Bios USHORT DAC_LoadDetection; //Atomic Table, directly used by various SW components,latest version 1.2 USHORT LVTMAEncoderControl; //Atomic Table,directly used by various SW components,latest version 1.3 USHORT HW_Misc_Operation; //Atomic Table, directly used by various SW components,latest version 1.1 @@ -272,7 +273,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{ USHORT TVEncoderControl; //Function Table,directly used by various SW components,latest version 1.1 USHORT PatchMCSetting; //only used by BIOS USHORT MC_SEQ_Control; //only used by BIOS - USHORT TV1OutputControl; //Atomic Table, Obsolete from Ry6xx, use DAC2 Output instead + USHORT Gfx_Harvesting; //Atomic Table, Obsolete from Ry6xx, Now only used by BIOS for GFX harvesting USHORT EnableScaler; //Atomic Table, used only by Bios USHORT BlankCRTC; //Atomic Table, directly used by various SW components,latest version 1.1 USHORT EnableCRTC; //Atomic Table, directly used by various SW components,latest version 1.1 @@ -329,7 +330,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{ #define UNIPHYTransmitterControl DIG1TransmitterControl #define LVTMATransmitterControl DIG2TransmitterControl #define SetCRTC_DPM_State GetConditionalGoldenSetting -#define SetUniphyInstance ASIC_StaticPwrMgtStatusChange +#define ASIC_StaticPwrMgtStatusChange SetUniphyInstance #define HPDInterruptService ReadHWAssistedI2CStatus #define EnableVGA_Access GetSCLKOverMCLKRatio #define EnableYUV GetDispObjectInfo @@ -339,7 +340,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{ #define TMDSAEncoderControl PatchMCSetting #define LVDSEncoderControl MC_SEQ_Control #define LCD1OutputControl HW_Misc_Operation - +#define TV1OutputControl Gfx_Harvesting typedef struct _ATOM_MASTER_COMMAND_TABLE { @@ -459,6 +460,7 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V3 union { ATOM_COMPUTE_CLOCK_FREQ ulClock; //Input Parameter + ULONG ulClockParams; //ULONG access for BE ATOM_S_MPLL_FB_DIVIDER ulFbDiv; //Output Parameter }; UCHAR ucRefDiv; //Output Parameter @@ -478,11 +480,11 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V3 typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 { #if ATOM_BIG_ENDIAN - ULONG ucPostDiv; //return parameter: post divider which is used to program to register directly + ULONG ucPostDiv:8; //return parameter: post divider which is used to program to register directly ULONG ulClock:24; //Input= target clock, output = actual clock #else ULONG ulClock:24; //Input= target clock, output = actual clock - ULONG ucPostDiv; //return parameter: post divider which is used to program to register directly + ULONG ucPostDiv:8; //return parameter: post divider which is used to program to register directly #endif }COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4; @@ -491,6 +493,7 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5 union { ATOM_COMPUTE_CLOCK_FREQ ulClock; //Input Parameter + ULONG ulClockParams; //ULONG access for BE ATOM_S_MPLL_FB_DIVIDER ulFbDiv; //Output Parameter }; UCHAR ucRefDiv; //Output Parameter @@ -503,6 +506,32 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5 UCHAR ucReserved; }COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5; + +typedef struct _COMPUTE_GPU_CLOCK_INPUT_PARAMETERS_V1_6 +{ + ATOM_COMPUTE_CLOCK_FREQ ulClock; //Input Parameter + ULONG ulReserved[2]; +}COMPUTE_GPU_CLOCK_INPUT_PARAMETERS_V1_6; + +//ATOM_COMPUTE_CLOCK_FREQ.ulComputeClockFlag +#define COMPUTE_GPUCLK_INPUT_FLAG_CLK_TYPE_MASK 0x0f +#define COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK 0x00 +#define COMPUTE_GPUCLK_INPUT_FLAG_SCLK 0x01 + +typedef struct _COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 +{ + COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 ulClock; //Output Parameter: ucPostDiv=DFS divider + ATOM_S_MPLL_FB_DIVIDER ulFbDiv; //Output Parameter: PLL FB divider + UCHAR ucPllRefDiv; //Output Parameter: PLL ref divider + UCHAR ucPllPostDiv; //Output Parameter: PLL post divider + UCHAR ucPllCntlFlag; //Output Flags: control flag + UCHAR ucReserved; +}COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6; + +//ucPllCntlFlag +#define SPLL_CNTL_FLAG_VCO_MODE_MASK 0x03 + + // ucInputFlag #define ATOM_PLL_INPUT_FLAG_PLL_STROBE_MODE_EN 1 // 1-StrobeMode, 0-PerformanceMode @@ -1682,9 +1711,12 @@ typedef struct _PIXEL_CLOCK_PARAMETERS_V6 #define PIXEL_CLOCK_V6_MISC_HDMI_BPP_MASK 0x0c #define PIXEL_CLOCK_V6_MISC_HDMI_24BPP 0x00 #define PIXEL_CLOCK_V6_MISC_HDMI_36BPP 0x04 +#define PIXEL_CLOCK_V6_MISC_HDMI_36BPP_V6 0x08 //for V6, the correct defintion for 36bpp should be 2 for 36bpp(2:1) #define PIXEL_CLOCK_V6_MISC_HDMI_30BPP 0x08 +#define PIXEL_CLOCK_V6_MISC_HDMI_30BPP_V6 0x04 //for V6, the correct defintion for 30bpp should be 1 for 36bpp(5:4) #define PIXEL_CLOCK_V6_MISC_HDMI_48BPP 0x0c #define PIXEL_CLOCK_V6_MISC_REF_DIV_SRC 0x10 +#define PIXEL_CLOCK_V6_MISC_GEN_DPREFCLK 0x40 typedef struct _GET_DISP_PLL_STATUS_INPUT_PARAMETERS_V2 { @@ -2101,6 +2133,17 @@ typedef struct _DVO_ENCODER_CONTROL_PARAMETERS_V3 }DVO_ENCODER_CONTROL_PARAMETERS_V3; #define DVO_ENCODER_CONTROL_PS_ALLOCATION_V3 DVO_ENCODER_CONTROL_PARAMETERS_V3 +typedef struct _DVO_ENCODER_CONTROL_PARAMETERS_V1_4 +{ + USHORT usPixelClock; + UCHAR ucDVOConfig; + UCHAR ucAction; //ATOM_ENABLE/ATOM_DISABLE/ATOM_HPD_INIT + UCHAR ucBitPerColor; //please refer to definition of PANEL_xBIT_PER_COLOR + UCHAR ucReseved[3]; +}DVO_ENCODER_CONTROL_PARAMETERS_V1_4; +#define DVO_ENCODER_CONTROL_PS_ALLOCATION_V1_4 DVO_ENCODER_CONTROL_PARAMETERS_V1_4 + + //ucTableFormatRevision=1 //ucTableContentRevision=3 structure is not changed but usMisc add bit 1 as another input for // bit1=0: non-coherent mode @@ -2164,7 +2207,7 @@ typedef struct _DVO_ENCODER_CONTROL_PARAMETERS_V3 #define SET_ASIC_VOLTAGE_MODE_SOURCE_B 0x4 #define SET_ASIC_VOLTAGE_MODE_SET_VOLTAGE 0x0 -#define SET_ASIC_VOLTAGE_MODE_GET_GPIOVAL 0x1 +#define SET_ASIC_VOLTAGE_MODE_GET_GPIOVAL 0x1 #define SET_ASIC_VOLTAGE_MODE_GET_GPIOMASK 0x2 typedef struct _SET_VOLTAGE_PARAMETERS @@ -2182,7 +2225,7 @@ typedef struct _SET_VOLTAGE_PARAMETERS_V2 USHORT usVoltageLevel; // real voltage level }SET_VOLTAGE_PARAMETERS_V2; - +// used by both SetVoltageTable v1.3 and v1.4 typedef struct _SET_VOLTAGE_PARAMETERS_V1_3 { UCHAR ucVoltageType; // To tell which voltage to set up, VDDC/MVDDC/MVDDQ/VDDCI @@ -2199,15 +2242,20 @@ typedef struct _SET_VOLTAGE_PARAMETERS_V1_3 //SET_VOLTAGE_PARAMETERS_V3.ucVoltageMode #define ATOM_SET_VOLTAGE 0 //Set voltage Level #define ATOM_INIT_VOLTAGE_REGULATOR 3 //Init Regulator -#define ATOM_SET_VOLTAGE_PHASE 4 //Set Vregulator Phase -#define ATOM_GET_MAX_VOLTAGE 6 //Get Max Voltage, not used in SetVoltageTable v1.3 -#define ATOM_GET_VOLTAGE_LEVEL 6 //Get Voltage level from vitual voltage ID +#define ATOM_SET_VOLTAGE_PHASE 4 //Set Vregulator Phase, only for SVID/PVID regulator +#define ATOM_GET_MAX_VOLTAGE 6 //Get Max Voltage, not used from SetVoltageTable v1.3 +#define ATOM_GET_VOLTAGE_LEVEL 6 //Get Voltage level from vitual voltage ID, not used for SetVoltage v1.4 +#define ATOM_GET_LEAKAGE_ID 8 //Get Leakage Voltage Id ( starting from SMU7x IP ), SetVoltage v1.4 // define vitual voltage id in usVoltageLevel #define ATOM_VIRTUAL_VOLTAGE_ID0 0xff01 #define ATOM_VIRTUAL_VOLTAGE_ID1 0xff02 #define ATOM_VIRTUAL_VOLTAGE_ID2 0xff03 #define ATOM_VIRTUAL_VOLTAGE_ID3 0xff04 +#define ATOM_VIRTUAL_VOLTAGE_ID4 0xff05 +#define ATOM_VIRTUAL_VOLTAGE_ID5 0xff06 +#define ATOM_VIRTUAL_VOLTAGE_ID6 0xff07 +#define ATOM_VIRTUAL_VOLTAGE_ID7 0xff08 typedef struct _SET_VOLTAGE_PS_ALLOCATION { @@ -2244,15 +2292,36 @@ typedef struct _GET_LEAKAGE_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_1 #define ATOM_GET_VOLTAGE_VID 0x00 #define ATOM_GET_VOTLAGE_INIT_SEQ 0x03 #define ATOM_GET_VOLTTAGE_PHASE_PHASE_VID 0x04 -// for SI, this state map to 0xff02 voltage state in Power Play table, which is power boost state -#define ATOM_GET_VOLTAGE_STATE0_LEAKAGE_VID 0x10 +#define ATOM_GET_VOLTAGE_SVID2 0x07 //Get SVI2 Regulator Info +// for SI, this state map to 0xff02 voltage state in Power Play table, which is power boost state +#define ATOM_GET_VOLTAGE_STATE0_LEAKAGE_VID 0x10 // for SI, this state map to 0xff01 voltage state in Power Play table, which is performance state #define ATOM_GET_VOLTAGE_STATE1_LEAKAGE_VID 0x11 -// undefined power state + #define ATOM_GET_VOLTAGE_STATE2_LEAKAGE_VID 0x12 #define ATOM_GET_VOLTAGE_STATE3_LEAKAGE_VID 0x13 +// New Added from CI Hawaii for GetVoltageInfoTable, input parameter structure +typedef struct _GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2 +{ + UCHAR ucVoltageType; // Input: To tell which voltage to set up, VDDC/MVDDC/MVDDQ/VDDCI + UCHAR ucVoltageMode; // Input: Indicate action: Get voltage info + USHORT usVoltageLevel; // Input: real voltage level in unit of mv or Voltage Phase (0, 1, 2, .. ) or Leakage Id + ULONG ulSCLKFreq; // Input: when ucVoltageMode= ATOM_GET_VOLTAGE_EVV_VOLTAGE, DPM state SCLK frequency, Define in PPTable SCLK/Voltage dependence table +}GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2; + +// New in GetVoltageInfo v1.2 ucVoltageMode +#define ATOM_GET_VOLTAGE_EVV_VOLTAGE 0x09 + +// New Added from CI Hawaii for EVV feature +typedef struct _GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 +{ + USHORT usVoltageLevel; // real voltage level in unit of mv + USHORT usVoltageId; // Voltage Id programmed in Voltage Regulator + ULONG ulReseved; +}GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2; + /****************************************************************************/ // Structures used by TVEncoderControlTable /****************************************************************************/ @@ -2627,7 +2696,8 @@ typedef struct _ATOM_FIRMWARE_INFO_V2_2 ULONG ulFirmwareRevision; ULONG ulDefaultEngineClock; //In 10Khz unit ULONG ulDefaultMemoryClock; //In 10Khz unit - ULONG ulReserved[2]; + ULONG ulSPLL_OutputFreq; //In 10Khz unit + ULONG ulGPUPLL_OutputFreq; //In 10Khz unit ULONG ulReserved1; //Was ulMaxEngineClockPLL_Output; //In 10Khz unit* ULONG ulReserved2; //Was ulMaxMemoryClockPLL_Output; //In 10Khz unit* ULONG ulMaxPixelClockPLL_Output; //In 10Khz unit @@ -3812,6 +3882,14 @@ typedef struct _ATOM_GPIO_PIN_ASSIGNMENT UCHAR ucGPIO_ID; }ATOM_GPIO_PIN_ASSIGNMENT; +//ucGPIO_ID pre-define id for multiple usage +//from SMU7.x, if ucGPIO_ID=PP_AC_DC_SWITCH_GPIO_PINID in GPIO_LUTTable, AC/DC swithing feature is enable +#define PP_AC_DC_SWITCH_GPIO_PINID 60 +//from SMU7.x, if ucGPIO_ID=VDDC_REGULATOR_VRHOT_GPIO_PINID in GPIO_LUTable, VRHot feature is enable +#define VDDC_VRHOT_GPIO_PINID 61 +//if ucGPIO_ID=VDDC_PCC_GPIO_PINID in GPIO_LUTable, Peak Current Control feature is enabled +#define VDDC_PCC_GPIO_PINID 62 + typedef struct _ATOM_GPIO_PIN_LUT { ATOM_COMMON_TABLE_HEADER sHeader; @@ -4073,17 +4151,19 @@ typedef struct _EXT_DISPLAY_PATH //usCaps #define EXT_DISPLAY_PATH_CAPS__HBR2_DISABLE 0x01 +#define EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN 0x02 typedef struct _ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO { ATOM_COMMON_TABLE_HEADER sHeader; UCHAR ucGuid [NUMBER_OF_UCHAR_FOR_GUID]; // a GUID is a 16 byte long string EXT_DISPLAY_PATH sPath[MAX_NUMBER_OF_EXT_DISPLAY_PATH]; // total of fixed 7 entries. - UCHAR ucChecksum; // a simple Checksum of the sum of whole structure equal to 0x0. + UCHAR ucChecksum; // a simple Checksum of the sum of whole structure equal to 0x0. UCHAR uc3DStereoPinId; // use for eDP panel UCHAR ucRemoteDisplayConfig; UCHAR uceDPToLVDSRxId; - UCHAR Reserved[4]; // for potential expansion + UCHAR ucFixDPVoltageSwing; // usCaps[1]=1, this indicate DP_LANE_SET value + UCHAR Reserved[3]; // for potential expansion }ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO; //Related definitions, all records are different but they have a commond header @@ -4114,10 +4194,10 @@ typedef struct _ATOM_COMMON_RECORD_HEADER #define ATOM_OBJECT_LINK_RECORD_TYPE 18 //Once this record is present under one object, it indicats the oobject is linked to another obj described by the record #define ATOM_CONNECTOR_REMOTE_CAP_RECORD_TYPE 19 #define ATOM_ENCODER_CAP_RECORD_TYPE 20 - +#define ATOM_BRACKET_LAYOUT_RECORD_TYPE 21 //Must be updated when new record type is added,equal to that record definition! -#define ATOM_MAX_OBJECT_RECORD_NUMBER ATOM_ENCODER_CAP_RECORD_TYPE +#define ATOM_MAX_OBJECT_RECORD_NUMBER ATOM_BRACKET_LAYOUT_RECORD_TYPE typedef struct _ATOM_I2C_RECORD { @@ -4342,6 +4422,31 @@ typedef struct _ATOM_CONNECTOR_REMOTE_CAP_RECORD USHORT usReserved; }ATOM_CONNECTOR_REMOTE_CAP_RECORD; +typedef struct _ATOM_CONNECTOR_LAYOUT_INFO +{ + USHORT usConnectorObjectId; + UCHAR ucConnectorType; + UCHAR ucPosition; +}ATOM_CONNECTOR_LAYOUT_INFO; + +// define ATOM_CONNECTOR_LAYOUT_INFO.ucConnectorType to describe the display connector size +#define CONNECTOR_TYPE_DVI_D 1 +#define CONNECTOR_TYPE_DVI_I 2 +#define CONNECTOR_TYPE_VGA 3 +#define CONNECTOR_TYPE_HDMI 4 +#define CONNECTOR_TYPE_DISPLAY_PORT 5 +#define CONNECTOR_TYPE_MINI_DISPLAY_PORT 6 + +typedef struct _ATOM_BRACKET_LAYOUT_RECORD +{ + ATOM_COMMON_RECORD_HEADER sheader; + UCHAR ucLength; + UCHAR ucWidth; + UCHAR ucConnNum; + UCHAR ucReserved; + ATOM_CONNECTOR_LAYOUT_INFO asConnInfo[1]; +}ATOM_BRACKET_LAYOUT_RECORD; + /****************************************************************************/ // ASIC voltage data table /****************************************************************************/ @@ -4415,6 +4520,13 @@ typedef struct _ATOM_VOLTAGE_CONTROL #define VOLTAGE_CONTROL_ID_CHL822x 0x08 #define VOLTAGE_CONTROL_ID_VT1586M 0x09 #define VOLTAGE_CONTROL_ID_UP1637 0x0A +#define VOLTAGE_CONTROL_ID_CHL8214 0x0B +#define VOLTAGE_CONTROL_ID_UP1801 0x0C +#define VOLTAGE_CONTROL_ID_ST6788A 0x0D +#define VOLTAGE_CONTROL_ID_CHLIR3564SVI2 0x0E +#define VOLTAGE_CONTROL_ID_AD527x 0x0F +#define VOLTAGE_CONTROL_ID_NCP81022 0x10 +#define VOLTAGE_CONTROL_ID_LTC2635 0x11 typedef struct _ATOM_VOLTAGE_OBJECT { @@ -4457,6 +4569,16 @@ typedef struct _ATOM_VOLTAGE_OBJECT_HEADER_V3{ USHORT usSize; //Size of Object }ATOM_VOLTAGE_OBJECT_HEADER_V3; +// ATOM_VOLTAGE_OBJECT_HEADER_V3.ucVoltageMode +#define VOLTAGE_OBJ_GPIO_LUT 0 //VOLTAGE and GPIO Lookup table ->ATOM_GPIO_VOLTAGE_OBJECT_V3 +#define VOLTAGE_OBJ_VR_I2C_INIT_SEQ 3 //VOLTAGE REGULATOR INIT sequece through I2C -> ATOM_I2C_VOLTAGE_OBJECT_V3 +#define VOLTAGE_OBJ_PHASE_LUT 4 //Set Vregulator Phase lookup table ->ATOM_GPIO_VOLTAGE_OBJECT_V3 +#define VOLTAGE_OBJ_SVID2 7 //Indicate voltage control by SVID2 ->ATOM_SVID2_VOLTAGE_OBJECT_V3 +#define VOLTAGE_OBJ_EVV 8 +#define VOLTAGE_OBJ_PWRBOOST_LEAKAGE_LUT 0x10 //Powerboost Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 +#define VOLTAGE_OBJ_HIGH_STATE_LEAKAGE_LUT 0x11 //High voltage state Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 +#define VOLTAGE_OBJ_HIGH1_STATE_LEAKAGE_LUT 0x12 //High1 voltage state Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 + typedef struct _VOLTAGE_LUT_ENTRY_V2 { ULONG ulVoltageId; // The Voltage ID which is used to program GPIO register @@ -4472,7 +4594,7 @@ typedef struct _LEAKAGE_VOLTAGE_LUT_ENTRY_V2 typedef struct _ATOM_I2C_VOLTAGE_OBJECT_V3 { - ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader; + ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader; // voltage mode = VOLTAGE_OBJ_VR_I2C_INIT_SEQ UCHAR ucVoltageRegulatorId; //Indicate Voltage Regulator Id UCHAR ucVoltageControlI2cLine; UCHAR ucVoltageControlAddress; @@ -4481,9 +4603,13 @@ typedef struct _ATOM_I2C_VOLTAGE_OBJECT_V3 VOLTAGE_LUT_ENTRY asVolI2cLut[1]; // end with 0xff }ATOM_I2C_VOLTAGE_OBJECT_V3; +// ATOM_I2C_VOLTAGE_OBJECT_V3.ucVoltageControlFlag +#define VOLTAGE_DATA_ONE_BYTE 0 +#define VOLTAGE_DATA_TWO_BYTE 1 + typedef struct _ATOM_GPIO_VOLTAGE_OBJECT_V3 { - ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader; + ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader; // voltage mode = VOLTAGE_OBJ_GPIO_LUT or VOLTAGE_OBJ_PHASE_LUT UCHAR ucVoltageGpioCntlId; // default is 0 which indicate control through CG VID mode UCHAR ucGpioEntryNum; // indiate the entry numbers of Votlage/Gpio value Look up table UCHAR ucPhaseDelay; // phase delay in unit of micro second @@ -4494,7 +4620,7 @@ typedef struct _ATOM_GPIO_VOLTAGE_OBJECT_V3 typedef struct _ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 { - ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader; + ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader; // voltage mode = 0x10/0x11/0x12 UCHAR ucLeakageCntlId; // default is 0 UCHAR ucLeakageEntryNum; // indicate the entry number of LeakageId/Voltage Lut table UCHAR ucReserved[2]; @@ -4502,10 +4628,27 @@ typedef struct _ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 LEAKAGE_VOLTAGE_LUT_ENTRY_V2 asLeakageIdLut[1]; }ATOM_LEAKAGE_VOLTAGE_OBJECT_V3; + +typedef struct _ATOM_SVID2_VOLTAGE_OBJECT_V3 +{ + ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader; // voltage mode = VOLTAGE_OBJ_SVID2 +// 14:7 – PSI0_VID +// 6 – PSI0_EN +// 5 – PSI1 +// 4:2 – load line slope trim. +// 1:0 – offset trim, + USHORT usLoadLine_PSI; +// GPU GPIO pin Id to SVID2 regulator VRHot pin. possible value 0~31. 0 means GPIO0, 31 means GPIO31 + UCHAR ucSVDGpioId; //0~31 indicate GPIO0~31 + UCHAR ucSVCGpioId; //0~31 indicate GPIO0~31 + ULONG ulReserved; +}ATOM_SVID2_VOLTAGE_OBJECT_V3; + typedef union _ATOM_VOLTAGE_OBJECT_V3{ ATOM_GPIO_VOLTAGE_OBJECT_V3 asGpioVoltageObj; ATOM_I2C_VOLTAGE_OBJECT_V3 asI2cVoltageObj; ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 asLeakageObj; + ATOM_SVID2_VOLTAGE_OBJECT_V3 asSVID2Obj; }ATOM_VOLTAGE_OBJECT_V3; typedef struct _ATOM_VOLTAGE_OBJECT_INFO_V3_1 @@ -4535,6 +4678,64 @@ typedef struct _ATOM_ASIC_PROFILING_INFO ATOM_ASIC_PROFILE_VOLTAGE asVoltage; }ATOM_ASIC_PROFILING_INFO; +typedef struct _ATOM_ASIC_PROFILING_INFO_V2_1 +{ + ATOM_COMMON_TABLE_HEADER asHeader; + UCHAR ucLeakageBinNum; // indicate the entry number of LeakageId/Voltage Lut table + USHORT usLeakageBinArrayOffset; // offset of USHORT Leakage Bin list array ( from lower LeakageId to higher) + + UCHAR ucElbVDDC_Num; + USHORT usElbVDDC_IdArrayOffset; // offset of USHORT virtual VDDC voltage id ( 0xff01~0xff08 ) + USHORT usElbVDDC_LevelArrayOffset; // offset of 2 dimension voltage level USHORT array + + UCHAR ucElbVDDCI_Num; + USHORT usElbVDDCI_IdArrayOffset; // offset of USHORT virtual VDDCI voltage id ( 0xff01~0xff08 ) + USHORT usElbVDDCI_LevelArrayOffset; // offset of 2 dimension voltage level USHORT array +}ATOM_ASIC_PROFILING_INFO_V2_1; + +typedef struct _ATOM_ASIC_PROFILING_INFO_V3_1 +{ + ATOM_COMMON_TABLE_HEADER asHeader; + ULONG ulEvvDerateTdp; + ULONG ulEvvDerateTdc; + ULONG ulBoardCoreTemp; + ULONG ulMaxVddc; + ULONG ulMinVddc; + ULONG ulLoadLineSlop; + ULONG ulLeakageTemp; + ULONG ulLeakageVoltage; + ULONG ulCACmEncodeRange; + ULONG ulCACmEncodeAverage; + ULONG ulCACbEncodeRange; + ULONG ulCACbEncodeAverage; + ULONG ulKt_bEncodeRange; + ULONG ulKt_bEncodeAverage; + ULONG ulKv_mEncodeRange; + ULONG ulKv_mEncodeAverage; + ULONG ulKv_bEncodeRange; + ULONG ulKv_bEncodeAverage; + ULONG ulLkgEncodeLn_MaxDivMin; + ULONG ulLkgEncodeMin; + ULONG ulEfuseLogisticAlpha; + USHORT usPowerDpm0; + USHORT usCurrentDpm0; + USHORT usPowerDpm1; + USHORT usCurrentDpm1; + USHORT usPowerDpm2; + USHORT usCurrentDpm2; + USHORT usPowerDpm3; + USHORT usCurrentDpm3; + USHORT usPowerDpm4; + USHORT usCurrentDpm4; + USHORT usPowerDpm5; + USHORT usCurrentDpm5; + USHORT usPowerDpm6; + USHORT usCurrentDpm6; + USHORT usPowerDpm7; + USHORT usCurrentDpm7; +}ATOM_ASIC_PROFILING_INFO_V3_1; + + typedef struct _ATOM_POWER_SOURCE_OBJECT { UCHAR ucPwrSrcId; // Power source @@ -4651,6 +4852,8 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 #define SYS_INFO_LVDSMISC__888_BPC 0x04 #define SYS_INFO_LVDSMISC__OVERRIDE_EN 0x08 #define SYS_INFO_LVDSMISC__BLON_ACTIVE_LOW 0x10 +// new since Trinity +#define SYS_INFO_LVDSMISC__TRAVIS_LVDS_VOL_OVERRIDE_EN 0x20 // not used any more #define SYS_INFO_LVDSMISC__VSYNC_ACTIVE_LOW 0x04 @@ -4751,6 +4954,29 @@ typedef struct _ATOM_FUSION_SYSTEM_INFO_V1 ATOM_INTEGRATED_SYSTEM_INFO_V6 sIntegratedSysInfo; ULONG ulPowerplayTable[128]; }ATOM_FUSION_SYSTEM_INFO_V1; + + +typedef struct _ATOM_TDP_CONFIG_BITS +{ +#if ATOM_BIG_ENDIAN + ULONG uReserved:2; + ULONG uTDP_Value:14; // Original TDP value in tens of milli watts + ULONG uCTDP_Value:14; // Override value in tens of milli watts + ULONG uCTDP_Enable:2; // = (uCTDP_Value > uTDP_Value? 2: (uCTDP_Value < uTDP_Value)) +#else + ULONG uCTDP_Enable:2; // = (uCTDP_Value > uTDP_Value? 2: (uCTDP_Value < uTDP_Value)) + ULONG uCTDP_Value:14; // Override value in tens of milli watts + ULONG uTDP_Value:14; // Original TDP value in tens of milli watts + ULONG uReserved:2; +#endif +}ATOM_TDP_CONFIG_BITS; + +typedef union _ATOM_TDP_CONFIG +{ + ATOM_TDP_CONFIG_BITS TDP_config; + ULONG TDP_config_all; +}ATOM_TDP_CONFIG; + /********************************************************************************************************************** ATOM_FUSION_SYSTEM_INFO_V1 Description sIntegratedSysInfo: refer to ATOM_INTEGRATED_SYSTEM_INFO_V6 definition. @@ -4783,7 +5009,8 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 UCHAR ucMemoryType; UCHAR ucUMAChannelNumber; UCHAR strVBIOSMsg[40]; - ULONG ulReserved[20]; + ATOM_TDP_CONFIG asTdpConfig; + ULONG ulReserved[19]; ATOM_AVAILABLE_SCLK_LIST sAvail_SCLK[5]; ULONG ulGMCRestoreResetTime; ULONG ulMinimumNClk; @@ -4808,7 +5035,7 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 USHORT GnbTdpLimit; USHORT usMaxLVDSPclkFreqInSingleLink; UCHAR ucLvdsMisc; - UCHAR ucLVDSReserved; + UCHAR ucTravisLVDSVolAdjust; UCHAR ucLVDSPwrOnSeqDIGONtoDE_in4Ms; UCHAR ucLVDSPwrOnSeqDEtoVARY_BL_in4Ms; UCHAR ucLVDSPwrOffSeqVARY_BLtoDE_in4Ms; @@ -4816,7 +5043,7 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 UCHAR ucLVDSOffToOnDelay_in4Ms; UCHAR ucLVDSPwrOnSeqVARY_BLtoBLON_in4Ms; UCHAR ucLVDSPwrOffSeqBLONtoVARY_BL_in4Ms; - UCHAR ucLVDSReserved1; + UCHAR ucMinAllowedBL_Level; ULONG ulLCDBitDepthControlVal; ULONG ulNbpStateMemclkFreq[4]; USHORT usNBP2Voltage; @@ -4845,6 +5072,7 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 #define SYS_INFO_GPUCAPS__TMDSHDMI_COHERENT_SINGLEPLL_MODE 0x01 #define SYS_INFO_GPUCAPS__DP_SINGLEPLL_MODE 0x02 #define SYS_INFO_GPUCAPS__DISABLE_AUX_MODE_DETECT 0x08 +#define SYS_INFO_GPUCAPS__ENABEL_DFS_BYPASS 0x10 /********************************************************************************************************************** ATOM_INTEGRATED_SYSTEM_INFO_V1_7 Description @@ -4944,6 +5172,9 @@ ucLVDSMisc: [bit0] LVDS 888bit panel mode =0: LVDS 888 pan [bit2] LVDS 888bit per color mode =0: 666 bit per color =1:888 bit per color [bit3] LVDS parameter override enable =0: ucLvdsMisc parameter are not used =1: ucLvdsMisc parameter should be used [bit4] Polarity of signal sent to digital BLON output pin. =0: not inverted(active high) =1: inverted ( active low ) + [bit5] Travid LVDS output voltage override enable, when =1, use ucTravisLVDSVolAdjust value to overwrite Traivs register LVDS_CTRL_4 +ucTravisLVDSVolAdjust When ucLVDSMisc[5]=1,it means platform SBIOS want to overwrite TravisLVDSVoltage. Then VBIOS will use ucTravisLVDSVolAdjust + value to program Travis register LVDS_CTRL_4 ucLVDSPwrOnSeqDIGONtoDE_in4Ms: LVDS power up sequence time in unit of 4ms, time delay from DIGON signal active to data enable signal active( DE ). =0 mean use VBIOS default which is 8 ( 32ms ). The LVDS power up sequence is as following: DIGON->DE->VARY_BL->BLON. This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. @@ -4963,18 +5194,241 @@ ucLVDSOffToOnDelay_in4Ms: LVDS power down sequence time in unit of 4ms. =0 means to use VBIOS default delay which is 125 ( 500ms ). This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. -ucLVDSPwrOnVARY_BLtoBLON_in4Ms: LVDS power up sequence time in unit of 4ms. Time delay from VARY_BL signal on to DLON signal active. +ucLVDSPwrOnSeqVARY_BLtoBLON_in4Ms: + LVDS power up sequence time in unit of 4ms. Time delay from VARY_BL signal on to DLON signal active. =0 means to use VBIOS default delay which is 0 ( 0ms ). This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. -ucLVDSPwrOffBLONtoVARY_BL_in4Ms: LVDS power down sequence time in unit of 4ms. Time delay from BLON signal off to VARY_BL signal off. +ucLVDSPwrOffSeqBLONtoVARY_BL_in4Ms: + LVDS power down sequence time in unit of 4ms. Time delay from BLON signal off to VARY_BL signal off. =0 means to use VBIOS default delay which is 0 ( 0ms ). This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. +ucMinAllowedBL_Level: Lowest LCD backlight PWM level. This is customer platform specific parameters. By default it is 0. + ulNbpStateMemclkFreq[4]: system memory clock frequncey in unit of 10Khz in different NB pstate. **********************************************************************************************************************/ +// this IntegrateSystemInfoTable is used for Kaveri & Kabini APU +typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_8 +{ + ATOM_COMMON_TABLE_HEADER sHeader; + ULONG ulBootUpEngineClock; + ULONG ulDentistVCOFreq; + ULONG ulBootUpUMAClock; + ATOM_CLK_VOLT_CAPABILITY sDISPCLK_Voltage[4]; + ULONG ulBootUpReqDisplayVector; + ULONG ulVBIOSMisc; + ULONG ulGPUCapInfo; + ULONG ulDISP_CLK2Freq; + USHORT usRequestedPWMFreqInHz; + UCHAR ucHtcTmpLmt; + UCHAR ucHtcHystLmt; + ULONG ulReserved2; + ULONG ulSystemConfig; + ULONG ulCPUCapInfo; + ULONG ulReserved3; + USHORT usGPUReservedSysMemSize; + USHORT usExtDispConnInfoOffset; + USHORT usPanelRefreshRateRange; + UCHAR ucMemoryType; + UCHAR ucUMAChannelNumber; + UCHAR strVBIOSMsg[40]; + ATOM_TDP_CONFIG asTdpConfig; + ULONG ulReserved[19]; + ATOM_AVAILABLE_SCLK_LIST sAvail_SCLK[5]; + ULONG ulGMCRestoreResetTime; + ULONG ulReserved4; + ULONG ulIdleNClk; + ULONG ulDDR_DLL_PowerUpTime; + ULONG ulDDR_PLL_PowerUpTime; + USHORT usPCIEClkSSPercentage; + USHORT usPCIEClkSSType; + USHORT usLvdsSSPercentage; + USHORT usLvdsSSpreadRateIn10Hz; + USHORT usHDMISSPercentage; + USHORT usHDMISSpreadRateIn10Hz; + USHORT usDVISSPercentage; + USHORT usDVISSpreadRateIn10Hz; + ULONG ulGPUReservedSysMemBaseAddrLo; + ULONG ulGPUReservedSysMemBaseAddrHi; + ULONG ulReserved5[3]; + USHORT usMaxLVDSPclkFreqInSingleLink; + UCHAR ucLvdsMisc; + UCHAR ucTravisLVDSVolAdjust; + UCHAR ucLVDSPwrOnSeqDIGONtoDE_in4Ms; + UCHAR ucLVDSPwrOnSeqDEtoVARY_BL_in4Ms; + UCHAR ucLVDSPwrOffSeqVARY_BLtoDE_in4Ms; + UCHAR ucLVDSPwrOffSeqDEtoDIGON_in4Ms; + UCHAR ucLVDSOffToOnDelay_in4Ms; + UCHAR ucLVDSPwrOnSeqVARY_BLtoBLON_in4Ms; + UCHAR ucLVDSPwrOffSeqBLONtoVARY_BL_in4Ms; + UCHAR ucMinAllowedBL_Level; + ULONG ulLCDBitDepthControlVal; + ULONG ulNbpStateMemclkFreq[4]; + ULONG ulReserved6; + ULONG ulNbpStateNClkFreq[4]; + USHORT usNBPStateVoltage[4]; + USHORT usBootUpNBVoltage; + USHORT usReserved2; + ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO sExtDispConnInfo; +}ATOM_INTEGRATED_SYSTEM_INFO_V1_8; + +/********************************************************************************************************************** + ATOM_INTEGRATED_SYSTEM_INFO_V1_8 Description +ulBootUpEngineClock: VBIOS bootup Engine clock frequency, in 10kHz unit. if it is equal 0, then VBIOS use pre-defined bootup engine clock +ulDentistVCOFreq: Dentist VCO clock in 10kHz unit. +ulBootUpUMAClock: System memory boot up clock frequency in 10Khz unit. +sDISPCLK_Voltage: Report Display clock frequency requirement on GNB voltage(up to 4 voltage levels). + +ulBootUpReqDisplayVector: VBIOS boot up display IDs, following are supported devices in Trinity projects: + ATOM_DEVICE_CRT1_SUPPORT 0x0001 + ATOM_DEVICE_DFP1_SUPPORT 0x0008 + ATOM_DEVICE_DFP6_SUPPORT 0x0040 + ATOM_DEVICE_DFP2_SUPPORT 0x0080 + ATOM_DEVICE_DFP3_SUPPORT 0x0200 + ATOM_DEVICE_DFP4_SUPPORT 0x0400 + ATOM_DEVICE_DFP5_SUPPORT 0x0800 + ATOM_DEVICE_LCD1_SUPPORT 0x0002 + +ulVBIOSMisc: Miscellenous flags for VBIOS requirement and interface + bit[0]=0: INT15 callback function Get LCD EDID ( ax=4e08, bl=1b ) is not supported by SBIOS. + =1: INT15 callback function Get LCD EDID ( ax=4e08, bl=1b ) is supported by SBIOS. + bit[1]=0: INT15 callback function Get boot display( ax=4e08, bl=01h) is not supported by SBIOS + =1: INT15 callback function Get boot display( ax=4e08, bl=01h) is supported by SBIOS + bit[2]=0: INT15 callback function Get panel Expansion ( ax=4e08, bl=02h) is not supported by SBIOS + =1: INT15 callback function Get panel Expansion ( ax=4e08, bl=02h) is supported by SBIOS + bit[3]=0: VBIOS fast boot is disable + =1: VBIOS fast boot is enable. ( VBIOS skip display device detection in every set mode if LCD panel is connect and LID is open) + +ulGPUCapInfo: bit[0~2]= Reserved + bit[3]=0: Enable AUX HW mode detection logic + =1: Disable AUX HW mode detection logic + bit[4]=0: Disable DFS bypass feature + =1: Enable DFS bypass feature + +usRequestedPWMFreqInHz: When it's set to 0x0 by SBIOS: the LCD BackLight is not controlled by GPU(SW). + Any attempt to change BL using VBIOS function or enable VariBri from PP table is not effective since ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==0; + + When it's set to a non-zero frequency, the BackLight is controlled by GPU (SW) in one of two ways below: + 1. SW uses the GPU BL PWM output to control the BL, in chis case, this non-zero frequency determines what freq GPU should use; + VBIOS will set up proper PWM frequency and ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==1,as the result, + Changing BL using VBIOS function is functional in both driver and non-driver present environment; + and enabling VariBri under the driver environment from PP table is optional. + + 2. SW uses other means to control BL (like DPCD),this non-zero frequency serves as a flag only indicating + that BL control from GPU is expected. + VBIOS will NOT set up PWM frequency but make ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==1 + Changing BL using VBIOS function could be functional in both driver and non-driver present environment,but + it's per platform + and enabling VariBri under the driver environment from PP table is optional. + +ucHtcTmpLmt: Refer to D18F3x64 bit[22:16], HtcTmpLmt. Threshold on value to enter HTC_active state. +ucHtcHystLmt: Refer to D18F3x64 bit[27:24], HtcHystLmt. + To calculate threshold off value to exit HTC_active state, which is Threshold on vlaue minus ucHtcHystLmt. + +ulSystemConfig: Bit[0]=0: PCIE Power Gating Disabled + =1: PCIE Power Gating Enabled + Bit[1]=0: DDR-DLL shut-down feature disabled. + 1: DDR-DLL shut-down feature enabled. + Bit[2]=0: DDR-PLL Power down feature disabled. + 1: DDR-PLL Power down feature enabled. + Bit[3]=0: GNB DPM is disabled + =1: GNB DPM is enabled +ulCPUCapInfo: TBD + +usExtDispConnInfoOffset: Offset to sExtDispConnInfo inside the structure +usPanelRefreshRateRange: Bit vector for LCD supported refresh rate range. If DRR is requestd by the platform, at least two bits need to be set + to indicate a range. + SUPPORTED_LCD_REFRESHRATE_30Hz 0x0004 + SUPPORTED_LCD_REFRESHRATE_40Hz 0x0008 + SUPPORTED_LCD_REFRESHRATE_50Hz 0x0010 + SUPPORTED_LCD_REFRESHRATE_60Hz 0x0020 + +ucMemoryType: [3:0]=1:DDR1;=2:DDR2;=3:DDR3;=5:GDDR5; [7:4] is reserved. +ucUMAChannelNumber: System memory channel numbers. + +strVBIOSMsg[40]: VBIOS boot up customized message string + +sAvail_SCLK[5]: Arrays to provide availabe list of SLCK and corresponding voltage, order from low to high + +ulGMCRestoreResetTime: GMC power restore and GMC reset time to calculate data reconnection latency. Unit in ns. +ulIdleNClk: NCLK speed while memory runs in self-refresh state, used to calculate self-refresh latency. Unit in 10kHz. +ulDDR_DLL_PowerUpTime: DDR PHY DLL power up time. Unit in ns. +ulDDR_PLL_PowerUpTime: DDR PHY PLL power up time. Unit in ns. + +usPCIEClkSSPercentage: PCIE Clock Spread Spectrum Percentage in unit 0.01%; 100 mean 1%. +usPCIEClkSSType: PCIE Clock Spread Spectrum Type. 0 for Down spread(default); 1 for Center spread. +usLvdsSSPercentage: LVDS panel ( not include eDP ) Spread Spectrum Percentage in unit of 0.01%, =0, use VBIOS default setting. +usLvdsSSpreadRateIn10Hz: LVDS panel ( not include eDP ) Spread Spectrum frequency in unit of 10Hz, =0, use VBIOS default setting. +usHDMISSPercentage: HDMI Spread Spectrum Percentage in unit 0.01%; 100 mean 1%, =0, use VBIOS default setting. +usHDMISSpreadRateIn10Hz: HDMI Spread Spectrum frequency in unit of 10Hz, =0, use VBIOS default setting. +usDVISSPercentage: DVI Spread Spectrum Percentage in unit 0.01%; 100 mean 1%, =0, use VBIOS default setting. +usDVISSpreadRateIn10Hz: DVI Spread Spectrum frequency in unit of 10Hz, =0, use VBIOS default setting. + +usGPUReservedSysMemSize: Reserved system memory size for ACP engine in APU GNB, units in MB. 0/2/4MB based on CMOS options, current default could be 0MB. KV only, not on KB. +ulGPUReservedSysMemBaseAddrLo: Low 32 bits base address to the reserved system memory. +ulGPUReservedSysMemBaseAddrHi: High 32 bits base address to the reserved system memory. + +usMaxLVDSPclkFreqInSingleLink: Max pixel clock LVDS panel single link, if=0 means VBIOS use default threhold, right now it is 85Mhz +ucLVDSMisc: [bit0] LVDS 888bit panel mode =0: LVDS 888 panel in LDI mode, =1: LVDS 888 panel in FPDI mode + [bit1] LVDS panel lower and upper link mapping =0: lower link and upper link not swap, =1: lower link and upper link are swapped + [bit2] LVDS 888bit per color mode =0: 666 bit per color =1:888 bit per color + [bit3] LVDS parameter override enable =0: ucLvdsMisc parameter are not used =1: ucLvdsMisc parameter should be used + [bit4] Polarity of signal sent to digital BLON output pin. =0: not inverted(active high) =1: inverted ( active low ) + [bit5] Travid LVDS output voltage override enable, when =1, use ucTravisLVDSVolAdjust value to overwrite Traivs register LVDS_CTRL_4 +ucTravisLVDSVolAdjust When ucLVDSMisc[5]=1,it means platform SBIOS want to overwrite TravisLVDSVoltage. Then VBIOS will use ucTravisLVDSVolAdjust + value to program Travis register LVDS_CTRL_4 +ucLVDSPwrOnSeqDIGONtoDE_in4Ms: + LVDS power up sequence time in unit of 4ms, time delay from DIGON signal active to data enable signal active( DE ). + =0 mean use VBIOS default which is 8 ( 32ms ). The LVDS power up sequence is as following: DIGON->DE->VARY_BL->BLON. + This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. +ucLVDSPwrOnDEtoVARY_BL_in4Ms: + LVDS power up sequence time in unit of 4ms., time delay from DE( data enable ) active to Vary Brightness enable signal active( VARY_BL ). + =0 mean use VBIOS default which is 90 ( 360ms ). The LVDS power up sequence is as following: DIGON->DE->VARY_BL->BLON. + This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. +ucLVDSPwrOffVARY_BLtoDE_in4Ms: + LVDS power down sequence time in unit of 4ms, time delay from data enable ( DE ) signal off to LCDVCC (DIGON) off. + =0 mean use VBIOS default delay which is 8 ( 32ms ). The LVDS power down sequence is as following: BLON->VARY_BL->DE->DIGON + This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. +ucLVDSPwrOffDEtoDIGON_in4Ms: + LVDS power down sequence time in unit of 4ms, time delay from vary brightness enable signal( VARY_BL) off to data enable ( DE ) signal off. + =0 mean use VBIOS default which is 90 ( 360ms ). The LVDS power down sequence is as following: BLON->VARY_BL->DE->DIGON + This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. +ucLVDSOffToOnDelay_in4Ms: + LVDS power down sequence time in unit of 4ms. Time delay from DIGON signal off to DIGON signal active. + =0 means to use VBIOS default delay which is 125 ( 500ms ). + This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. +ucLVDSPwrOnSeqVARY_BLtoBLON_in4Ms: + LVDS power up sequence time in unit of 4ms. Time delay from VARY_BL signal on to DLON signal active. + =0 means to use VBIOS default delay which is 0 ( 0ms ). + This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. + +ucLVDSPwrOffSeqBLONtoVARY_BL_in4Ms: + LVDS power down sequence time in unit of 4ms. Time delay from BLON signal off to VARY_BL signal off. + =0 means to use VBIOS default delay which is 0 ( 0ms ). + This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable. +ucMinAllowedBL_Level: Lowest LCD backlight PWM level. This is customer platform specific parameters. By default it is 0. + +ulLCDBitDepthControlVal: GPU display control encoder bit dither control setting, used to program register mmFMT_BIT_DEPTH_CONTROL + +ulNbpStateMemclkFreq[4]: system memory clock frequncey in unit of 10Khz in different NB P-State(P0, P1, P2 & P3). +ulNbpStateNClkFreq[4]: NB P-State NClk frequency in different NB P-State +usNBPStateVoltage[4]: NB P-State (P0/P1 & P2/P3) voltage; NBP3 refers to lowes voltage +usBootUpNBVoltage: NB P-State voltage during boot up before driver loaded +sExtDispConnInfo: Display connector information table provided to VBIOS + +**********************************************************************************************************************/ + +// this Table is used for Kaveri/Kabini APU +typedef struct _ATOM_FUSION_SYSTEM_INFO_V2 +{ + ATOM_INTEGRATED_SYSTEM_INFO_V1_8 sIntegratedSysInfo; // refer to ATOM_INTEGRATED_SYSTEM_INFO_V1_8 definition + ULONG ulPowerplayTable[128]; // Update comments here to link new powerplay table definition structure +}ATOM_FUSION_SYSTEM_INFO_V2; + + /**************************************************************************/ // This portion is only used when ext thermal chip or engine/memory clock SS chip is populated on a design //Memory SS Info Table @@ -5025,22 +5479,24 @@ typedef struct _ATOM_ASIC_SS_ASSIGNMENT //Define ucClockIndication, SW uses the IDs below to search if the SS is required/enabled on a clock branch/signal type. //SS is not required or enabled if a match is not found. -#define ASIC_INTERNAL_MEMORY_SS 1 -#define ASIC_INTERNAL_ENGINE_SS 2 -#define ASIC_INTERNAL_UVD_SS 3 -#define ASIC_INTERNAL_SS_ON_TMDS 4 -#define ASIC_INTERNAL_SS_ON_HDMI 5 -#define ASIC_INTERNAL_SS_ON_LVDS 6 -#define ASIC_INTERNAL_SS_ON_DP 7 -#define ASIC_INTERNAL_SS_ON_DCPLL 8 -#define ASIC_EXTERNAL_SS_ON_DP_CLOCK 9 -#define ASIC_INTERNAL_VCE_SS 10 +#define ASIC_INTERNAL_MEMORY_SS 1 +#define ASIC_INTERNAL_ENGINE_SS 2 +#define ASIC_INTERNAL_UVD_SS 3 +#define ASIC_INTERNAL_SS_ON_TMDS 4 +#define ASIC_INTERNAL_SS_ON_HDMI 5 +#define ASIC_INTERNAL_SS_ON_LVDS 6 +#define ASIC_INTERNAL_SS_ON_DP 7 +#define ASIC_INTERNAL_SS_ON_DCPLL 8 +#define ASIC_EXTERNAL_SS_ON_DP_CLOCK 9 +#define ASIC_INTERNAL_VCE_SS 10 +#define ASIC_INTERNAL_GPUPLL_SS 11 + typedef struct _ATOM_ASIC_SS_ASSIGNMENT_V2 { ULONG ulTargetClockRange; //For mem/engine/uvd, Clock Out frequence (VCO ), in unit of 10Khz //For TMDS/HDMI/LVDS, it is pixel clock , for DP, it is link clock ( 27000 or 16200 ) - USHORT usSpreadSpectrumPercentage; //in unit of 0.01% + USHORT usSpreadSpectrumPercentage; //in unit of 0.01% or 0.001%, decided by ucSpreadSpectrumMode bit4 USHORT usSpreadRateIn10Hz; //in unit of 10Hz, modulation freq UCHAR ucClockIndication; //Indicate which clock source needs SS UCHAR ucSpreadSpectrumMode; //Bit0=0 Down Spread,=1 Center Spread, bit1=0: internal SS bit1=1: external SS @@ -5078,6 +5534,11 @@ typedef struct _ATOM_ASIC_SS_ASSIGNMENT_V3 UCHAR ucReserved[2]; }ATOM_ASIC_SS_ASSIGNMENT_V3; +//ATOM_ASIC_SS_ASSIGNMENT_V3.ucSpreadSpectrumMode +#define SS_MODE_V3_CENTRE_SPREAD_MASK 0x01 +#define SS_MODE_V3_EXTERNAL_SS_MASK 0x02 +#define SS_MODE_V3_PERCENTAGE_DIV_BY_1000_MASK 0x10 + typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3 { ATOM_COMMON_TABLE_HEADER sHeader; @@ -5446,6 +5907,8 @@ typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3 #define ATOM_S7_DOS_MODE_PIXEL_DEPTHb0 0x0C #define ATOM_S7_DOS_MODE_PIXEL_FORMATb0 0xF0 #define ATOM_S7_DOS_8BIT_DAC_ENb1 0x01 +#define ATOM_S7_ASIC_INIT_COMPLETEb1 0x02 +#define ATOM_S7_ASIC_INIT_COMPLETE_MASK 0x00000200 #define ATOM_S7_DOS_MODE_NUMBERw1 0x0FFFF #define ATOM_S7_DOS_8BIT_DAC_EN_SHIFT 8 @@ -5718,6 +6181,7 @@ typedef struct _INDIRECT_IO_ACCESS #define INDIRECT_IO_PCIE 3 #define INDIRECT_IO_PCIEP 4 #define INDIRECT_IO_NBMISC 5 +#define INDIRECT_IO_SMU 5 #define INDIRECT_IO_PLL_READ INDIRECT_IO_PLL | INDIRECT_READ #define INDIRECT_IO_PLL_WRITE INDIRECT_IO_PLL | INDIRECT_WRITE @@ -5729,6 +6193,8 @@ typedef struct _INDIRECT_IO_ACCESS #define INDIRECT_IO_PCIEP_WRITE INDIRECT_IO_PCIEP | INDIRECT_WRITE #define INDIRECT_IO_NBMISC_READ INDIRECT_IO_NBMISC | INDIRECT_READ #define INDIRECT_IO_NBMISC_WRITE INDIRECT_IO_NBMISC | INDIRECT_WRITE +#define INDIRECT_IO_SMU_READ INDIRECT_IO_SMU | INDIRECT_READ +#define INDIRECT_IO_SMU_WRITE INDIRECT_IO_SMU | INDIRECT_WRITE typedef struct _ATOM_OEM_INFO { @@ -5874,8 +6340,10 @@ typedef struct _ATOM_MC_INIT_PARAM_TABLE #define _64Mx32 0x43 #define _128Mx8 0x51 #define _128Mx16 0x52 +#define _128Mx32 0x53 #define _256Mx8 0x61 #define _256Mx16 0x62 +#define _512Mx8 0x71 #define SAMSUNG 0x1 #define INFINEON 0x2 @@ -5892,6 +6360,8 @@ typedef struct _ATOM_MC_INIT_PARAM_TABLE #define PROMOS MOSEL #define KRETON INFINEON #define ELIXIR NANYA +#define MEZZA ELPIDA + /////////////Support for GDDR5 MC uCode to reside in upper 64K of ROM///////////// @@ -6619,11 +7089,16 @@ typedef struct _ATOM_DISP_OUT_INFO_V3 UCHAR ucMaxDispEngineNum; UCHAR ucMaxActiveDispEngineNum; UCHAR ucMaxPPLLNum; - UCHAR ucCoreRefClkSource; // value of CORE_REF_CLK_SOURCE - UCHAR ucReserved[3]; - ASIC_TRANSMITTER_INFO_V2 asTransmitterInfo[1]; // for alligment only + UCHAR ucCoreRefClkSource; // value of CORE_REF_CLK_SOURCE + UCHAR ucDispCaps; + UCHAR ucReserved[2]; + ASIC_TRANSMITTER_INFO_V2 asTransmitterInfo[1]; // for alligment only }ATOM_DISP_OUT_INFO_V3; +//ucDispCaps +#define DISPLAY_CAPS__DP_PCLK_FROM_PPLL 0x01 +#define DISPLAY_CAPS__FORCE_DISPDEV_CONNECTED 0x02 + typedef enum CORE_REF_CLK_SOURCE{ CLOCK_SRC_XTALIN=0, CLOCK_SRC_XO_IN=1, @@ -6828,6 +7303,17 @@ typedef struct _DIG_TRANSMITTER_INFO_HEADER_V3_1{ USHORT usPhyPllSettingOffset; // offset of CLOCK_CONDITION_SETTING_ENTRY* with Phy Pll Settings }DIG_TRANSMITTER_INFO_HEADER_V3_1; +typedef struct _DIG_TRANSMITTER_INFO_HEADER_V3_2{ + ATOM_COMMON_TABLE_HEADER sHeader; + USHORT usDPVsPreEmphSettingOffset; // offset of PHY_ANALOG_SETTING_INFO * with DP Voltage Swing and Pre-Emphasis for each Link clock + USHORT usPhyAnalogRegListOffset; // offset of CLOCK_CONDITION_REGESTER_INFO* with None-DP mode Analog Setting's register Info + USHORT usPhyAnalogSettingOffset; // offset of CLOCK_CONDITION_SETTING_ENTRY* with None-DP mode Analog Setting for each link clock range + USHORT usPhyPllRegListOffset; // offset of CLOCK_CONDITION_REGESTER_INFO* with Phy Pll register Info + USHORT usPhyPllSettingOffset; // offset of CLOCK_CONDITION_SETTING_ENTRY* with Phy Pll Settings + USHORT usDPSSRegListOffset; // offset of CLOCK_CONDITION_REGESTER_INFO* with Phy SS Pll register Info + USHORT usDPSSSettingOffset; // offset of CLOCK_CONDITION_SETTING_ENTRY* with Phy SS Pll Settings +}DIG_TRANSMITTER_INFO_HEADER_V3_2; + typedef struct _CLOCK_CONDITION_REGESTER_INFO{ USHORT usRegisterIndex; UCHAR ucStartBit; @@ -6851,12 +7337,24 @@ typedef struct _PHY_CONDITION_REG_VAL{ ULONG ulRegVal; }PHY_CONDITION_REG_VAL; +typedef struct _PHY_CONDITION_REG_VAL_V2{ + ULONG ulCondition; + UCHAR ucCondition2; + ULONG ulRegVal; +}PHY_CONDITION_REG_VAL_V2; + typedef struct _PHY_CONDITION_REG_INFO{ USHORT usRegIndex; USHORT usSize; PHY_CONDITION_REG_VAL asRegVal[1]; }PHY_CONDITION_REG_INFO; +typedef struct _PHY_CONDITION_REG_INFO_V2{ + USHORT usRegIndex; + USHORT usSize; + PHY_CONDITION_REG_VAL_V2 asRegVal[1]; +}PHY_CONDITION_REG_INFO_V2; + typedef struct _PHY_ANALOG_SETTING_INFO{ UCHAR ucEncodeMode; UCHAR ucPhySel; @@ -6864,6 +7362,25 @@ typedef struct _PHY_ANALOG_SETTING_INFO{ PHY_CONDITION_REG_INFO asAnalogSetting[1]; }PHY_ANALOG_SETTING_INFO; +typedef struct _PHY_ANALOG_SETTING_INFO_V2{ + UCHAR ucEncodeMode; + UCHAR ucPhySel; + USHORT usSize; + PHY_CONDITION_REG_INFO_V2 asAnalogSetting[1]; +}PHY_ANALOG_SETTING_INFO_V2; + +typedef struct _GFX_HAVESTING_PARAMETERS { + UCHAR ucGfxBlkId; //GFX blk id to be harvested, like CU, RB or PRIM + UCHAR ucReserved; //reserved + UCHAR ucActiveUnitNumPerSH; //requested active CU/RB/PRIM number per shader array + UCHAR ucMaxUnitNumPerSH; //max CU/RB/PRIM number per shader array +} GFX_HAVESTING_PARAMETERS; + +//ucGfxBlkId +#define GFX_HARVESTING_CU_ID 0 +#define GFX_HARVESTING_RB_ID 1 +#define GFX_HARVESTING_PRIM_ID 2 + /****************************************************************************/ //Portion VI: Definitinos for vbios MC scratch registers that driver used /****************************************************************************/ @@ -6874,8 +7391,17 @@ typedef struct _PHY_ANALOG_SETTING_INFO{ #define MC_MISC0__MEMORY_TYPE__GDDR3 0x30000000 #define MC_MISC0__MEMORY_TYPE__GDDR4 0x40000000 #define MC_MISC0__MEMORY_TYPE__GDDR5 0x50000000 +#define MC_MISC0__MEMORY_TYPE__HBM 0x60000000 #define MC_MISC0__MEMORY_TYPE__DDR3 0xB0000000 +#define ATOM_MEM_TYPE_DDR_STRING "DDR" +#define ATOM_MEM_TYPE_DDR2_STRING "DDR2" +#define ATOM_MEM_TYPE_GDDR3_STRING "GDDR3" +#define ATOM_MEM_TYPE_GDDR4_STRING "GDDR4" +#define ATOM_MEM_TYPE_GDDR5_STRING "GDDR5" +#define ATOM_MEM_TYPE_HBM_STRING "HBM" +#define ATOM_MEM_TYPE_DDR3_STRING "DDR3" + /****************************************************************************/ //Portion VI: Definitinos being oboselete /****************************************************************************/ @@ -7238,565 +7764,6 @@ typedef struct _ATOM_POWERPLAY_INFO_V3 ATOM_POWERMODE_INFO_V3 asPowerPlayInfo[ATOM_MAX_NUMBEROF_POWER_BLOCK]; }ATOM_POWERPLAY_INFO_V3; -/* New PPlib */ -/**************************************************************************/ -typedef struct _ATOM_PPLIB_THERMALCONTROLLER - -{ - UCHAR ucType; // one of ATOM_PP_THERMALCONTROLLER_* - UCHAR ucI2cLine; // as interpreted by DAL I2C - UCHAR ucI2cAddress; - UCHAR ucFanParameters; // Fan Control Parameters. - UCHAR ucFanMinRPM; // Fan Minimum RPM (hundreds) -- for display purposes only. - UCHAR ucFanMaxRPM; // Fan Maximum RPM (hundreds) -- for display purposes only. - UCHAR ucReserved; // ---- - UCHAR ucFlags; // to be defined -} ATOM_PPLIB_THERMALCONTROLLER; - -#define ATOM_PP_FANPARAMETERS_TACHOMETER_PULSES_PER_REVOLUTION_MASK 0x0f -#define ATOM_PP_FANPARAMETERS_NOFAN 0x80 // No fan is connected to this controller. - -#define ATOM_PP_THERMALCONTROLLER_NONE 0 -#define ATOM_PP_THERMALCONTROLLER_LM63 1 // Not used by PPLib -#define ATOM_PP_THERMALCONTROLLER_ADM1032 2 // Not used by PPLib -#define ATOM_PP_THERMALCONTROLLER_ADM1030 3 // Not used by PPLib -#define ATOM_PP_THERMALCONTROLLER_MUA6649 4 // Not used by PPLib -#define ATOM_PP_THERMALCONTROLLER_LM64 5 -#define ATOM_PP_THERMALCONTROLLER_F75375 6 // Not used by PPLib -#define ATOM_PP_THERMALCONTROLLER_RV6xx 7 -#define ATOM_PP_THERMALCONTROLLER_RV770 8 -#define ATOM_PP_THERMALCONTROLLER_ADT7473 9 -#define ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO 11 -#define ATOM_PP_THERMALCONTROLLER_EVERGREEN 12 -#define ATOM_PP_THERMALCONTROLLER_EMC2103 13 /* 0x0D */ // Only fan control will be implemented, do NOT show this in PPGen. -#define ATOM_PP_THERMALCONTROLLER_SUMO 14 /* 0x0E */ // Sumo type, used internally -#define ATOM_PP_THERMALCONTROLLER_NISLANDS 15 -#define ATOM_PP_THERMALCONTROLLER_SISLANDS 16 -#define ATOM_PP_THERMALCONTROLLER_LM96163 17 - -// Thermal controller 'combo type' to use an external controller for Fan control and an internal controller for thermal. -// We probably should reserve the bit 0x80 for this use. -// To keep the number of these types low we should also use the same code for all ASICs (i.e. do not distinguish RV6xx and RV7xx Internal here). -// The driver can pick the correct internal controller based on the ASIC. - -#define ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL 0x89 // ADT7473 Fan Control + Internal Thermal Controller -#define ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL 0x8D // EMC2103 Fan Control + Internal Thermal Controller - -typedef struct _ATOM_PPLIB_STATE -{ - UCHAR ucNonClockStateIndex; - UCHAR ucClockStateIndices[1]; // variable-sized -} ATOM_PPLIB_STATE; - - -typedef struct _ATOM_PPLIB_FANTABLE -{ - UCHAR ucFanTableFormat; // Change this if the table format changes or version changes so that the other fields are not the same. - UCHAR ucTHyst; // Temperature hysteresis. Integer. - USHORT usTMin; // The temperature, in 0.01 centigrades, below which we just run at a minimal PWM. - USHORT usTMed; // The middle temperature where we change slopes. - USHORT usTHigh; // The high point above TMed for adjusting the second slope. - USHORT usPWMMin; // The minimum PWM value in percent (0.01% increments). - USHORT usPWMMed; // The PWM value (in percent) at TMed. - USHORT usPWMHigh; // The PWM value at THigh. -} ATOM_PPLIB_FANTABLE; - -typedef struct _ATOM_PPLIB_FANTABLE2 -{ - ATOM_PPLIB_FANTABLE basicTable; - USHORT usTMax; // The max temperature -} ATOM_PPLIB_FANTABLE2; - -typedef struct _ATOM_PPLIB_EXTENDEDHEADER -{ - USHORT usSize; - ULONG ulMaxEngineClock; // For Overdrive. - ULONG ulMaxMemoryClock; // For Overdrive. - // Add extra system parameters here, always adjust size to include all fields. - USHORT usVCETableOffset; //points to ATOM_PPLIB_VCE_Table - USHORT usUVDTableOffset; //points to ATOM_PPLIB_UVD_Table -} ATOM_PPLIB_EXTENDEDHEADER; - -//// ATOM_PPLIB_POWERPLAYTABLE::ulPlatformCaps -#define ATOM_PP_PLATFORM_CAP_BACKBIAS 1 -#define ATOM_PP_PLATFORM_CAP_POWERPLAY 2 -#define ATOM_PP_PLATFORM_CAP_SBIOSPOWERSOURCE 4 -#define ATOM_PP_PLATFORM_CAP_ASPM_L0s 8 -#define ATOM_PP_PLATFORM_CAP_ASPM_L1 16 -#define ATOM_PP_PLATFORM_CAP_HARDWAREDC 32 -#define ATOM_PP_PLATFORM_CAP_GEMINIPRIMARY 64 -#define ATOM_PP_PLATFORM_CAP_STEPVDDC 128 -#define ATOM_PP_PLATFORM_CAP_VOLTAGECONTROL 256 -#define ATOM_PP_PLATFORM_CAP_SIDEPORTCONTROL 512 -#define ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1 1024 -#define ATOM_PP_PLATFORM_CAP_HTLINKCONTROL 2048 -#define ATOM_PP_PLATFORM_CAP_MVDDCONTROL 4096 -#define ATOM_PP_PLATFORM_CAP_GOTO_BOOT_ON_ALERT 0x2000 // Go to boot state on alerts, e.g. on an AC->DC transition. -#define ATOM_PP_PLATFORM_CAP_DONT_WAIT_FOR_VBLANK_ON_ALERT 0x4000 // Do NOT wait for VBLANK during an alert (e.g. AC->DC transition). -#define ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL 0x8000 // Does the driver control VDDCI independently from VDDC. -#define ATOM_PP_PLATFORM_CAP_REGULATOR_HOT 0x00010000 // Enable the 'regulator hot' feature. -#define ATOM_PP_PLATFORM_CAP_BACO 0x00020000 // Does the driver supports BACO state. - - -typedef struct _ATOM_PPLIB_POWERPLAYTABLE -{ - ATOM_COMMON_TABLE_HEADER sHeader; - - UCHAR ucDataRevision; - - UCHAR ucNumStates; - UCHAR ucStateEntrySize; - UCHAR ucClockInfoSize; - UCHAR ucNonClockSize; - - // offset from start of this table to array of ucNumStates ATOM_PPLIB_STATE structures - USHORT usStateArrayOffset; - - // offset from start of this table to array of ASIC-specific structures, - // currently ATOM_PPLIB_CLOCK_INFO. - USHORT usClockInfoArrayOffset; - - // offset from start of this table to array of ATOM_PPLIB_NONCLOCK_INFO - USHORT usNonClockInfoArrayOffset; - - USHORT usBackbiasTime; // in microseconds - USHORT usVoltageTime; // in microseconds - USHORT usTableSize; //the size of this structure, or the extended structure - - ULONG ulPlatformCaps; // See ATOM_PPLIB_CAPS_* - - ATOM_PPLIB_THERMALCONTROLLER sThermalController; - - USHORT usBootClockInfoOffset; - USHORT usBootNonClockInfoOffset; - -} ATOM_PPLIB_POWERPLAYTABLE; - -typedef struct _ATOM_PPLIB_POWERPLAYTABLE2 -{ - ATOM_PPLIB_POWERPLAYTABLE basicTable; - UCHAR ucNumCustomThermalPolicy; - USHORT usCustomThermalPolicyArrayOffset; -}ATOM_PPLIB_POWERPLAYTABLE2, *LPATOM_PPLIB_POWERPLAYTABLE2; - -typedef struct _ATOM_PPLIB_POWERPLAYTABLE3 -{ - ATOM_PPLIB_POWERPLAYTABLE2 basicTable2; - USHORT usFormatID; // To be used ONLY by PPGen. - USHORT usFanTableOffset; - USHORT usExtendendedHeaderOffset; -} ATOM_PPLIB_POWERPLAYTABLE3, *LPATOM_PPLIB_POWERPLAYTABLE3; - -typedef struct _ATOM_PPLIB_POWERPLAYTABLE4 -{ - ATOM_PPLIB_POWERPLAYTABLE3 basicTable3; - ULONG ulGoldenPPID; // PPGen use only - ULONG ulGoldenRevision; // PPGen use only - USHORT usVddcDependencyOnSCLKOffset; - USHORT usVddciDependencyOnMCLKOffset; - USHORT usVddcDependencyOnMCLKOffset; - USHORT usMaxClockVoltageOnDCOffset; - USHORT usVddcPhaseShedLimitsTableOffset; // Points to ATOM_PPLIB_PhaseSheddingLimits_Table - USHORT usReserved; -} ATOM_PPLIB_POWERPLAYTABLE4, *LPATOM_PPLIB_POWERPLAYTABLE4; - -typedef struct _ATOM_PPLIB_POWERPLAYTABLE5 -{ - ATOM_PPLIB_POWERPLAYTABLE4 basicTable4; - ULONG ulTDPLimit; - ULONG ulNearTDPLimit; - ULONG ulSQRampingThreshold; - USHORT usCACLeakageTableOffset; // Points to ATOM_PPLIB_CAC_Leakage_Table - ULONG ulCACLeakage; // The iLeakage for driver calculated CAC leakage table - USHORT usTDPODLimit; - USHORT usLoadLineSlope; // in milliOhms * 100 -} ATOM_PPLIB_POWERPLAYTABLE5, *LPATOM_PPLIB_POWERPLAYTABLE5; - -//// ATOM_PPLIB_NONCLOCK_INFO::usClassification -#define ATOM_PPLIB_CLASSIFICATION_UI_MASK 0x0007 -#define ATOM_PPLIB_CLASSIFICATION_UI_SHIFT 0 -#define ATOM_PPLIB_CLASSIFICATION_UI_NONE 0 -#define ATOM_PPLIB_CLASSIFICATION_UI_BATTERY 1 -#define ATOM_PPLIB_CLASSIFICATION_UI_BALANCED 3 -#define ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE 5 -// 2, 4, 6, 7 are reserved - -#define ATOM_PPLIB_CLASSIFICATION_BOOT 0x0008 -#define ATOM_PPLIB_CLASSIFICATION_THERMAL 0x0010 -#define ATOM_PPLIB_CLASSIFICATION_LIMITEDPOWERSOURCE 0x0020 -#define ATOM_PPLIB_CLASSIFICATION_REST 0x0040 -#define ATOM_PPLIB_CLASSIFICATION_FORCED 0x0080 -#define ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE 0x0100 -#define ATOM_PPLIB_CLASSIFICATION_OVERDRIVETEMPLATE 0x0200 -#define ATOM_PPLIB_CLASSIFICATION_UVDSTATE 0x0400 -#define ATOM_PPLIB_CLASSIFICATION_3DLOW 0x0800 -#define ATOM_PPLIB_CLASSIFICATION_ACPI 0x1000 -#define ATOM_PPLIB_CLASSIFICATION_HD2STATE 0x2000 -#define ATOM_PPLIB_CLASSIFICATION_HDSTATE 0x4000 -#define ATOM_PPLIB_CLASSIFICATION_SDSTATE 0x8000 - -//// ATOM_PPLIB_NONCLOCK_INFO::usClassification2 -#define ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2 0x0001 -#define ATOM_PPLIB_CLASSIFICATION2_ULV 0x0002 -#define ATOM_PPLIB_CLASSIFICATION2_MVC 0x0004 //Multi-View Codec (BD-3D) - -//// ATOM_PPLIB_NONCLOCK_INFO::ulCapsAndSettings -#define ATOM_PPLIB_SINGLE_DISPLAY_ONLY 0x00000001 -#define ATOM_PPLIB_SUPPORTS_VIDEO_PLAYBACK 0x00000002 - -// 0 is 2.5Gb/s, 1 is 5Gb/s -#define ATOM_PPLIB_PCIE_LINK_SPEED_MASK 0x00000004 -#define ATOM_PPLIB_PCIE_LINK_SPEED_SHIFT 2 - -// lanes - 1: 1, 2, 4, 8, 12, 16 permitted by PCIE spec -#define ATOM_PPLIB_PCIE_LINK_WIDTH_MASK 0x000000F8 -#define ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT 3 - -// lookup into reduced refresh-rate table -#define ATOM_PPLIB_LIMITED_REFRESHRATE_VALUE_MASK 0x00000F00 -#define ATOM_PPLIB_LIMITED_REFRESHRATE_VALUE_SHIFT 8 - -#define ATOM_PPLIB_LIMITED_REFRESHRATE_UNLIMITED 0 -#define ATOM_PPLIB_LIMITED_REFRESHRATE_50HZ 1 -// 2-15 TBD as needed. - -#define ATOM_PPLIB_SOFTWARE_DISABLE_LOADBALANCING 0x00001000 -#define ATOM_PPLIB_SOFTWARE_ENABLE_SLEEP_FOR_TIMESTAMPS 0x00002000 - -#define ATOM_PPLIB_DISALLOW_ON_DC 0x00004000 - -#define ATOM_PPLIB_ENABLE_VARIBRIGHT 0x00008000 - -//memory related flags -#define ATOM_PPLIB_SWSTATE_MEMORY_DLL_OFF 0x000010000 - -//M3 Arb //2bits, current 3 sets of parameters in total -#define ATOM_PPLIB_M3ARB_MASK 0x00060000 -#define ATOM_PPLIB_M3ARB_SHIFT 17 - -#define ATOM_PPLIB_ENABLE_DRR 0x00080000 - -// remaining 16 bits are reserved -typedef struct _ATOM_PPLIB_THERMAL_STATE -{ - UCHAR ucMinTemperature; - UCHAR ucMaxTemperature; - UCHAR ucThermalAction; -}ATOM_PPLIB_THERMAL_STATE, *LPATOM_PPLIB_THERMAL_STATE; - -// Contained in an array starting at the offset -// in ATOM_PPLIB_POWERPLAYTABLE::usNonClockInfoArrayOffset. -// referenced from ATOM_PPLIB_STATE_INFO::ucNonClockStateIndex -#define ATOM_PPLIB_NONCLOCKINFO_VER1 12 -#define ATOM_PPLIB_NONCLOCKINFO_VER2 24 -typedef struct _ATOM_PPLIB_NONCLOCK_INFO -{ - USHORT usClassification; - UCHAR ucMinTemperature; - UCHAR ucMaxTemperature; - ULONG ulCapsAndSettings; - UCHAR ucRequiredPower; - USHORT usClassification2; - ULONG ulVCLK; - ULONG ulDCLK; - UCHAR ucUnused[5]; -} ATOM_PPLIB_NONCLOCK_INFO; - -// Contained in an array starting at the offset -// in ATOM_PPLIB_POWERPLAYTABLE::usClockInfoArrayOffset. -// referenced from ATOM_PPLIB_STATE::ucClockStateIndices -typedef struct _ATOM_PPLIB_R600_CLOCK_INFO -{ - USHORT usEngineClockLow; - UCHAR ucEngineClockHigh; - - USHORT usMemoryClockLow; - UCHAR ucMemoryClockHigh; - - USHORT usVDDC; - USHORT usUnused1; - USHORT usUnused2; - - ULONG ulFlags; // ATOM_PPLIB_R600_FLAGS_* - -} ATOM_PPLIB_R600_CLOCK_INFO; - -// ulFlags in ATOM_PPLIB_R600_CLOCK_INFO -#define ATOM_PPLIB_R600_FLAGS_PCIEGEN2 1 -#define ATOM_PPLIB_R600_FLAGS_UVDSAFE 2 -#define ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE 4 -#define ATOM_PPLIB_R600_FLAGS_MEMORY_ODT_OFF 8 -#define ATOM_PPLIB_R600_FLAGS_MEMORY_DLL_OFF 16 -#define ATOM_PPLIB_R600_FLAGS_LOWPOWER 32 // On the RV770 use 'low power' setting (sequencer S0). - -typedef struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO -{ - USHORT usEngineClockLow; - UCHAR ucEngineClockHigh; - - USHORT usMemoryClockLow; - UCHAR ucMemoryClockHigh; - - USHORT usVDDC; - USHORT usVDDCI; - USHORT usUnused; - - ULONG ulFlags; // ATOM_PPLIB_R600_FLAGS_* - -} ATOM_PPLIB_EVERGREEN_CLOCK_INFO; - -typedef struct _ATOM_PPLIB_SI_CLOCK_INFO -{ - USHORT usEngineClockLow; - UCHAR ucEngineClockHigh; - - USHORT usMemoryClockLow; - UCHAR ucMemoryClockHigh; - - USHORT usVDDC; - USHORT usVDDCI; - UCHAR ucPCIEGen; - UCHAR ucUnused1; - - ULONG ulFlags; // ATOM_PPLIB_SI_FLAGS_*, no flag is necessary for now - -} ATOM_PPLIB_SI_CLOCK_INFO; - - -typedef struct _ATOM_PPLIB_RS780_CLOCK_INFO - -{ - USHORT usLowEngineClockLow; // Low Engine clock in MHz (the same way as on the R600). - UCHAR ucLowEngineClockHigh; - USHORT usHighEngineClockLow; // High Engine clock in MHz. - UCHAR ucHighEngineClockHigh; - USHORT usMemoryClockLow; // For now one of the ATOM_PPLIB_RS780_SPMCLK_XXXX constants. - UCHAR ucMemoryClockHigh; // Currentyl unused. - UCHAR ucPadding; // For proper alignment and size. - USHORT usVDDC; // For the 780, use: None, Low, High, Variable - UCHAR ucMaxHTLinkWidth; // From SBIOS - {2, 4, 8, 16} - UCHAR ucMinHTLinkWidth; // From SBIOS - {2, 4, 8, 16}. Effective only if CDLW enabled. Minimum down stream width could be bigger as display BW requriement. - USHORT usHTLinkFreq; // See definition ATOM_PPLIB_RS780_HTLINKFREQ_xxx or in MHz(>=200). - ULONG ulFlags; -} ATOM_PPLIB_RS780_CLOCK_INFO; - -#define ATOM_PPLIB_RS780_VOLTAGE_NONE 0 -#define ATOM_PPLIB_RS780_VOLTAGE_LOW 1 -#define ATOM_PPLIB_RS780_VOLTAGE_HIGH 2 -#define ATOM_PPLIB_RS780_VOLTAGE_VARIABLE 3 - -#define ATOM_PPLIB_RS780_SPMCLK_NONE 0 // We cannot change the side port memory clock, leave it as it is. -#define ATOM_PPLIB_RS780_SPMCLK_LOW 1 -#define ATOM_PPLIB_RS780_SPMCLK_HIGH 2 - -#define ATOM_PPLIB_RS780_HTLINKFREQ_NONE 0 -#define ATOM_PPLIB_RS780_HTLINKFREQ_LOW 1 -#define ATOM_PPLIB_RS780_HTLINKFREQ_HIGH 2 - -typedef struct _ATOM_PPLIB_SUMO_CLOCK_INFO{ - USHORT usEngineClockLow; //clockfrequency & 0xFFFF. The unit is in 10khz - UCHAR ucEngineClockHigh; //clockfrequency >> 16. - UCHAR vddcIndex; //2-bit vddc index; - USHORT tdpLimit; - //please initalize to 0 - USHORT rsv1; - //please initialize to 0s - ULONG rsv2[2]; -}ATOM_PPLIB_SUMO_CLOCK_INFO; - - - -typedef struct _ATOM_PPLIB_STATE_V2 -{ - //number of valid dpm levels in this state; Driver uses it to calculate the whole - //size of the state: sizeof(ATOM_PPLIB_STATE_V2) + (ucNumDPMLevels - 1) * sizeof(UCHAR) - UCHAR ucNumDPMLevels; - - //a index to the array of nonClockInfos - UCHAR nonClockInfoIndex; - /** - * Driver will read the first ucNumDPMLevels in this array - */ - UCHAR clockInfoIndex[1]; -} ATOM_PPLIB_STATE_V2; - -typedef struct _StateArray{ - //how many states we have - UCHAR ucNumEntries; - - ATOM_PPLIB_STATE_V2 states[1]; -}StateArray; - - -typedef struct _ClockInfoArray{ - //how many clock levels we have - UCHAR ucNumEntries; - - //sizeof(ATOM_PPLIB_CLOCK_INFO) - UCHAR ucEntrySize; - - UCHAR clockInfo[1]; -}ClockInfoArray; - -typedef struct _NonClockInfoArray{ - - //how many non-clock levels we have. normally should be same as number of states - UCHAR ucNumEntries; - //sizeof(ATOM_PPLIB_NONCLOCK_INFO) - UCHAR ucEntrySize; - - ATOM_PPLIB_NONCLOCK_INFO nonClockInfo[1]; -}NonClockInfoArray; - -typedef struct _ATOM_PPLIB_Clock_Voltage_Dependency_Record -{ - USHORT usClockLow; - UCHAR ucClockHigh; - USHORT usVoltage; -}ATOM_PPLIB_Clock_Voltage_Dependency_Record; - -typedef struct _ATOM_PPLIB_Clock_Voltage_Dependency_Table -{ - UCHAR ucNumEntries; // Number of entries. - ATOM_PPLIB_Clock_Voltage_Dependency_Record entries[1]; // Dynamically allocate entries. -}ATOM_PPLIB_Clock_Voltage_Dependency_Table; - -typedef struct _ATOM_PPLIB_Clock_Voltage_Limit_Record -{ - USHORT usSclkLow; - UCHAR ucSclkHigh; - USHORT usMclkLow; - UCHAR ucMclkHigh; - USHORT usVddc; - USHORT usVddci; -}ATOM_PPLIB_Clock_Voltage_Limit_Record; - -typedef struct _ATOM_PPLIB_Clock_Voltage_Limit_Table -{ - UCHAR ucNumEntries; // Number of entries. - ATOM_PPLIB_Clock_Voltage_Limit_Record entries[1]; // Dynamically allocate entries. -}ATOM_PPLIB_Clock_Voltage_Limit_Table; - -typedef struct _ATOM_PPLIB_CAC_Leakage_Record -{ - USHORT usVddc; // We use this field for the "fake" standardized VDDC for power calculations - ULONG ulLeakageValue; -}ATOM_PPLIB_CAC_Leakage_Record; - -typedef struct _ATOM_PPLIB_CAC_Leakage_Table -{ - UCHAR ucNumEntries; // Number of entries. - ATOM_PPLIB_CAC_Leakage_Record entries[1]; // Dynamically allocate entries. -}ATOM_PPLIB_CAC_Leakage_Table; - -typedef struct _ATOM_PPLIB_PhaseSheddingLimits_Record -{ - USHORT usVoltage; - USHORT usSclkLow; - UCHAR ucSclkHigh; - USHORT usMclkLow; - UCHAR ucMclkHigh; -}ATOM_PPLIB_PhaseSheddingLimits_Record; - -typedef struct _ATOM_PPLIB_PhaseSheddingLimits_Table -{ - UCHAR ucNumEntries; // Number of entries. - ATOM_PPLIB_PhaseSheddingLimits_Record entries[1]; // Dynamically allocate entries. -}ATOM_PPLIB_PhaseSheddingLimits_Table; - -typedef struct _VCEClockInfo{ - USHORT usEVClkLow; - UCHAR ucEVClkHigh; - USHORT usECClkLow; - UCHAR ucECClkHigh; -}VCEClockInfo; - -typedef struct _VCEClockInfoArray{ - UCHAR ucNumEntries; - VCEClockInfo entries[1]; -}VCEClockInfoArray; - -typedef struct _ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record -{ - USHORT usVoltage; - UCHAR ucVCEClockInfoIndex; -}ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record; - -typedef struct _ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table -{ - UCHAR numEntries; - ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record entries[1]; -}ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table; - -typedef struct _ATOM_PPLIB_VCE_State_Record -{ - UCHAR ucVCEClockInfoIndex; - UCHAR ucClockInfoIndex; //highest 2 bits indicates memory p-states, lower 6bits indicates index to ClockInfoArrary -}ATOM_PPLIB_VCE_State_Record; - -typedef struct _ATOM_PPLIB_VCE_State_Table -{ - UCHAR numEntries; - ATOM_PPLIB_VCE_State_Record entries[1]; -}ATOM_PPLIB_VCE_State_Table; - - -typedef struct _ATOM_PPLIB_VCE_Table -{ - UCHAR revid; -// VCEClockInfoArray array; -// ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table limits; -// ATOM_PPLIB_VCE_State_Table states; -}ATOM_PPLIB_VCE_Table; - - -typedef struct _UVDClockInfo{ - USHORT usVClkLow; - UCHAR ucVClkHigh; - USHORT usDClkLow; - UCHAR ucDClkHigh; -}UVDClockInfo; - -typedef struct _UVDClockInfoArray{ - UCHAR ucNumEntries; - UVDClockInfo entries[1]; -}UVDClockInfoArray; - -typedef struct _ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record -{ - USHORT usVoltage; - UCHAR ucUVDClockInfoIndex; -}ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record; - -typedef struct _ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table -{ - UCHAR numEntries; - ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record entries[1]; -}ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table; - -typedef struct _ATOM_PPLIB_UVD_State_Record -{ - UCHAR ucUVDClockInfoIndex; - UCHAR ucClockInfoIndex; //highest 2 bits indicates memory p-states, lower 6bits indicates index to ClockInfoArrary -}ATOM_PPLIB_UVD_State_Record; - -typedef struct _ATOM_PPLIB_UVD_State_Table -{ - UCHAR numEntries; - ATOM_PPLIB_UVD_State_Record entries[1]; -}ATOM_PPLIB_UVD_State_Table; - - -typedef struct _ATOM_PPLIB_UVD_Table -{ - UCHAR revid; -// UVDClockInfoArray array; -// ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table limits; -// ATOM_PPLIB_UVD_State_Table states; -}ATOM_PPLIB_UVD_Table; - -/**************************************************************************/ - // Following definitions are for compatibility issue in different SW components. #define ATOM_MASTER_DATA_TABLE_REVISION 0x01 @@ -7977,8 +7944,8 @@ typedef struct { typedef struct { AMD_ACPI_DESCRIPTION_HEADER SHeader; UCHAR TableUUID[16]; //0x24 - ULONG VBIOSImageOffset; //0x34. Offset to the first GOP_VBIOS_CONTENT block from the beginning of the stucture. - ULONG Lib1ImageOffset; //0x38. Offset to the first GOP_LIB1_CONTENT block from the beginning of the stucture. + ULONG VBIOSImageOffset; //0x34. Offset to the first GOP_VBIOS_CONTENT block from the beginning of the structure. + ULONG Lib1ImageOffset; //0x38. Offset to the first GOP_LIB1_CONTENT block from the beginning of the structure. ULONG Reserved[4]; //0x3C }UEFI_ACPI_VFCT; @@ -8009,3 +7976,6 @@ typedef struct { #endif /* _ATOMBIOS_H */ + +#include "pptable.h" + diff --git a/sys/dev/pci/drm/radeon/atombios_crtc.c b/sys/dev/pci/drm/radeon/atombios_crtc.c index ae788fa7369..8f3826560ba 100644 --- a/sys/dev/pci/drm/radeon/atombios_crtc.c +++ b/sys/dev/pci/drm/radeon/atombios_crtc.c @@ -1,4 +1,3 @@ -/* $OpenBSD: atombios_crtc.c,v 1.9 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2007-8 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -210,6 +209,16 @@ static void atombios_enable_crtc_memreq(struct drm_crtc *crtc, int state) atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); } +static const u32 vga_control_regs[6] = +{ + AVIVO_D1VGA_CONTROL, + AVIVO_D2VGA_CONTROL, + EVERGREEN_D3VGA_CONTROL, + EVERGREEN_D4VGA_CONTROL, + EVERGREEN_D5VGA_CONTROL, + EVERGREEN_D6VGA_CONTROL, +}; + static void atombios_blank_crtc(struct drm_crtc *crtc, int state) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); @@ -217,13 +226,23 @@ static void atombios_blank_crtc(struct drm_crtc *crtc, int state) struct radeon_device *rdev = dev->dev_private; int index = GetIndexIntoMasterTable(COMMAND, BlankCRTC); BLANK_CRTC_PS_ALLOCATION args; + u32 vga_control = 0; memset(&args, 0, sizeof(args)); + if (ASIC_IS_DCE8(rdev)) { + vga_control = RREG32(vga_control_regs[radeon_crtc->crtc_id]); + WREG32(vga_control_regs[radeon_crtc->crtc_id], vga_control | 1); + } + args.ucCRTC = radeon_crtc->crtc_id; args.ucBlanking = state; atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + if (ASIC_IS_DCE8(rdev)) { + WREG32(vga_control_regs[radeon_crtc->crtc_id], vga_control); + } } static void atombios_powergate_crtc(struct drm_crtc *crtc, int state) @@ -251,13 +270,13 @@ void atombios_crtc_dpms(struct drm_crtc *crtc, int mode) switch (mode) { case DRM_MODE_DPMS_ON: radeon_crtc->enabled = true; - /* adjust pm to dpms changes BEFORE enabling crtcs */ - radeon_pm_compute_clocks(rdev); atombios_enable_crtc(crtc, ATOM_ENABLE); if (ASIC_IS_DCE3(rdev) && !ASIC_IS_DCE6(rdev)) atombios_enable_crtc_memreq(crtc, ATOM_ENABLE); atombios_blank_crtc(crtc, ATOM_DISABLE); drm_vblank_post_modeset(dev, radeon_crtc->crtc_id); + /* Make sure vblank interrupt is still enabled if needed */ + radeon_irq_set(rdev); radeon_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: @@ -270,10 +289,10 @@ void atombios_crtc_dpms(struct drm_crtc *crtc, int mode) atombios_enable_crtc_memreq(crtc, ATOM_DISABLE); atombios_enable_crtc(crtc, ATOM_DISABLE); radeon_crtc->enabled = false; - /* adjust pm to dpms changes AFTER disabling crtcs */ - radeon_pm_compute_clocks(rdev); break; } + /* adjust pm to dpms */ + radeon_pm_compute_clocks(rdev); } static void @@ -313,8 +332,10 @@ atombios_set_crtc_dtd_timing(struct drm_crtc *crtc, misc |= ATOM_COMPOSITESYNC; if (mode->flags & DRM_MODE_FLAG_INTERLACE) misc |= ATOM_INTERLACE; - if (mode->flags & DRM_MODE_FLAG_DBLSCAN) + if (mode->flags & DRM_MODE_FLAG_DBLCLK) misc |= ATOM_DOUBLE_CLOCK_MODE; + if (mode->flags & DRM_MODE_FLAG_DBLSCAN) + misc |= ATOM_H_REPLICATIONBY2 | ATOM_V_REPLICATIONBY2; args.susModeMiscInfo.usAccess = cpu_to_le16(misc); args.ucCRTC = radeon_crtc->crtc_id; @@ -357,8 +378,10 @@ static void atombios_crtc_set_timing(struct drm_crtc *crtc, misc |= ATOM_COMPOSITESYNC; if (mode->flags & DRM_MODE_FLAG_INTERLACE) misc |= ATOM_INTERLACE; - if (mode->flags & DRM_MODE_FLAG_DBLSCAN) + if (mode->flags & DRM_MODE_FLAG_DBLCLK) misc |= ATOM_DOUBLE_CLOCK_MODE; + if (mode->flags & DRM_MODE_FLAG_DBLSCAN) + misc |= ATOM_H_REPLICATIONBY2 | ATOM_V_REPLICATIONBY2; args.susModeMiscInfo.usAccess = cpu_to_le16(misc); args.ucCRTC = radeon_crtc->crtc_id; @@ -424,7 +447,17 @@ static void atombios_crtc_program_ss(struct radeon_device *rdev, int index = GetIndexIntoMasterTable(COMMAND, EnableSpreadSpectrumOnPPLL); union atom_enable_ss args; - if (!enable) { + if (enable) { + /* Don't mess with SS if percentage is 0 or external ss. + * SS is already disabled previously, and disabling it + * again can cause display problems if the pll is already + * programmed. + */ + if (ss->percentage == 0) + return; + if (ss->type & ATOM_EXTERNAL_SS_MASK) + return; + } else { for (i = 0; i < rdev->num_crtc; i++) { if (rdev->mode_info.crtcs[i] && rdev->mode_info.crtcs[i]->enabled && @@ -460,8 +493,6 @@ static void atombios_crtc_program_ss(struct radeon_device *rdev, args.v3.usSpreadSpectrumAmount = cpu_to_le16(ss->amount); args.v3.usSpreadSpectrumStep = cpu_to_le16(ss->step); args.v3.ucEnable = enable; - if ((ss->percentage == 0) || (ss->type & ATOM_EXTERNAL_SS_MASK) || ASIC_IS_DCE61(rdev)) - args.v3.ucEnable = ATOM_DISABLE; } else if (ASIC_IS_DCE4(rdev)) { args.v2.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage); args.v2.ucSpreadSpectrumType = ss->type & ATOM_SS_CENTRE_SPREAD_MODE_MASK; @@ -481,8 +512,6 @@ static void atombios_crtc_program_ss(struct radeon_device *rdev, args.v2.usSpreadSpectrumAmount = cpu_to_le16(ss->amount); args.v2.usSpreadSpectrumStep = cpu_to_le16(ss->step); args.v2.ucEnable = enable; - if ((ss->percentage == 0) || (ss->type & ATOM_EXTERNAL_SS_MASK) || ASIC_IS_DCE41(rdev)) - args.v2.ucEnable = ATOM_DISABLE; } else if (ASIC_IS_DCE3(rdev)) { args.v1.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage); args.v1.ucSpreadSpectrumType = ss->type & ATOM_SS_CENTRE_SPREAD_MODE_MASK; @@ -504,8 +533,7 @@ static void atombios_crtc_program_ss(struct radeon_device *rdev, args.lvds_ss_2.ucSpreadSpectrumRange = ss->range; args.lvds_ss_2.ucEnable = enable; } else { - if ((enable == ATOM_DISABLE) || (ss->percentage == 0) || - (ss->type & ATOM_EXTERNAL_SS_MASK)) { + if (enable == ATOM_DISABLE) { atombios_disable_ss(rdev, pll_id); return; } @@ -535,7 +563,8 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc, u32 adjusted_clock = mode->clock; int encoder_mode = atombios_get_encoder_mode(encoder); u32 dp_clock = mode->clock; - int bpc = radeon_get_monitor_bpc(connector); + u32 clock = mode->clock; + int bpc = radeon_crtc->bpc; bool is_duallink = radeon_dig_monitor_is_duallink(encoder, mode->clock); /* reset the pll flags */ @@ -556,7 +585,7 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc, if (rdev->family < CHIP_RV770) radeon_crtc->pll_flags |= RADEON_PLL_PREFER_MINM_OVER_MAXP; /* use frac fb div on APUs */ - if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE61(rdev)) + if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE61(rdev) || ASIC_IS_DCE8(rdev)) radeon_crtc->pll_flags |= RADEON_PLL_USE_FRAC_FB_DIV; /* use frac fb div on RS780/RS880 */ if ((rdev->family == CHIP_RS780) || (rdev->family == CHIP_RS880)) @@ -583,6 +612,13 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc, } } + if (radeon_encoder->is_mst_encoder) { + struct radeon_encoder_mst *mst_enc = radeon_encoder->enc_priv; + struct radeon_connector_atom_dig *dig_connector = mst_enc->connector->con_priv; + + dp_clock = dig_connector->dp_clock; + } + /* use recommended ref_div for ss */ if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { if (radeon_crtc->ss_enabled) { @@ -610,6 +646,24 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc, radeon_crtc->pll_flags |= RADEON_PLL_USE_REF_DIV; } + /* adjust pll for deep color modes */ + if (encoder_mode == ATOM_ENCODER_MODE_HDMI) { + switch (bpc) { + case 8: + default: + break; + case 10: + clock = (clock * 5) / 4; + break; + case 12: + clock = (clock * 3) / 2; + break; + case 16: + clock = clock * 2; + break; + } + } + /* DCE3+ has an AdjustDisplayPll that will adjust the pixel clock * accordingly based on the encoder/transmitter to work around * special hw requirements. @@ -631,7 +685,7 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc, switch (crev) { case 1: case 2: - args.v1.usPixelClock = cpu_to_le16(mode->clock / 10); + args.v1.usPixelClock = cpu_to_le16(clock / 10); args.v1.ucTransmitterID = radeon_encoder->encoder_id; args.v1.ucEncodeMode = encoder_mode; if (radeon_crtc->ss_enabled && radeon_crtc->ss.percentage) @@ -643,7 +697,7 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc, adjusted_clock = le16_to_cpu(args.v1.usPixelClock) * 10; break; case 3: - args.v3.sInput.usPixelClock = cpu_to_le16(mode->clock / 10); + args.v3.sInput.usPixelClock = cpu_to_le16(clock / 10); args.v3.sInput.ucTransmitterID = radeon_encoder->encoder_id; args.v3.sInput.ucEncodeMode = encoder_mode; args.v3.sInput.ucDispPllConfig = 0; @@ -657,10 +711,6 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc, args.v3.sInput.usPixelClock = cpu_to_le16(dp_clock / 10); } else if (radeon_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) { struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - if (encoder_mode == ATOM_ENCODER_MODE_HDMI) - /* deep color support */ - args.v3.sInput.usPixelClock = - cpu_to_le16((mode->clock * bpc / 8) / 10); if (dig->coherent_mode) args.v3.sInput.ucDispPllConfig |= DISPPLL_CONFIG_COHERENT_MODE; @@ -744,7 +794,7 @@ static void atombios_crtc_set_disp_eng_pll(struct radeon_device *rdev, * SetPixelClock provides the dividers */ args.v6.ulDispEngClkFreq = cpu_to_le32(dispclk); - if (ASIC_IS_DCE61(rdev)) + if (ASIC_IS_DCE61(rdev) || ASIC_IS_DCE8(rdev)) args.v6.ucPpll = ATOM_EXT_PLL1; else if (ASIC_IS_DCE6(rdev)) args.v6.ucPpll = ATOM_PPLL0; @@ -847,6 +897,11 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc, args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_24BPP; break; case 10: + /* yes this is correct, the atom define is wrong */ + args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_32BPP; + break; + case 12: + /* yes this is correct, the atom define is wrong */ args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_30BPP; break; } @@ -871,10 +926,10 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc, args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_24BPP; break; case 10: - args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_30BPP; + args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_30BPP_V6; break; case 12: - args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_36BPP; + args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_36BPP_V6; break; case 16: args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_48BPP; @@ -910,7 +965,9 @@ static bool atombios_crtc_prepare_pll(struct drm_crtc *crtc, struct drm_display_ radeon_crtc->bpc = 8; radeon_crtc->ss_enabled = false; - if ((radeon_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) || + if (radeon_encoder->is_mst_encoder) { + radeon_dp_mst_prepare_pll(crtc, mode); + } else if ((radeon_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) || (radeon_encoder_get_dp_bridge_encoder_id(radeon_crtc->encoder) != ENCODER_OBJECT_ID_NONE)) { struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; struct drm_connector *connector = @@ -920,6 +977,9 @@ static bool atombios_crtc_prepare_pll(struct drm_crtc *crtc, struct drm_display_ struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv; int dp_clock; + + /* Assign mode clock for hdmi deep color max clock limit check */ + radeon_connector->pixelclock_for_modeset = mode->clock; radeon_crtc->bpc = radeon_get_monitor_bpc(connector); switch (encoder_mode) { @@ -1001,10 +1061,17 @@ static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode struct radeon_encoder *radeon_encoder = to_radeon_encoder(radeon_crtc->encoder); u32 pll_clock = mode->clock; + u32 clock = mode->clock; u32 ref_div = 0, fb_div = 0, frac_fb_div = 0, post_div = 0; struct radeon_pll *pll; int encoder_mode = atombios_get_encoder_mode(radeon_crtc->encoder); + /* pass the actual clock to atombios_crtc_program_pll for DCE5,6 for HDMI */ + if (ASIC_IS_DCE5(rdev) && + (encoder_mode == ATOM_ENCODER_MODE_HDMI) && + (radeon_crtc->bpc > 8)) + clock = radeon_crtc->adjusted_clock; + switch (radeon_crtc->pll_id) { case ATOM_PPLL1: pll = &rdev->clock.p1pll; @@ -1039,7 +1106,7 @@ static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode radeon_crtc->crtc_id, &radeon_crtc->ss); atombios_crtc_program_pll(crtc, radeon_crtc->crtc_id, radeon_crtc->pll_id, - encoder_mode, radeon_encoder->encoder_id, mode->clock, + encoder_mode, radeon_encoder->encoder_id, clock, ref_div, fb_div, frac_fb_div, post_div, radeon_crtc->bpc, radeon_crtc->ss_enabled, &radeon_crtc->ss); @@ -1047,15 +1114,17 @@ static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode /* calculate ss amount and step size */ if (ASIC_IS_DCE4(rdev)) { u32 step_size; - u32 amount = (((fb_div * 10) + frac_fb_div) * radeon_crtc->ss.percentage) / 10000; + u32 amount = (((fb_div * 10) + frac_fb_div) * + (u32)radeon_crtc->ss.percentage) / + (100 * (u32)radeon_crtc->ss.percentage_divider); radeon_crtc->ss.amount = (amount / 10) & ATOM_PPLL_SS_AMOUNT_V2_FBDIV_MASK; radeon_crtc->ss.amount |= ((amount - (amount / 10)) << ATOM_PPLL_SS_AMOUNT_V2_NFRAC_SHIFT) & ATOM_PPLL_SS_AMOUNT_V2_NFRAC_MASK; if (radeon_crtc->ss.type & ATOM_PPLL_SS_TYPE_V2_CENTRE_SPREAD) - step_size = (4 * amount * ref_div * (radeon_crtc->ss.rate * 2048)) / + step_size = (4 * amount * ref_div * ((u32)radeon_crtc->ss.rate * 2048)) / (125 * 25 * pll->reference_freq / 100); else - step_size = (2 * amount * ref_div * (radeon_crtc->ss.rate * 2048)) / + step_size = (2 * amount * ref_div * ((u32)radeon_crtc->ss.rate * 2048)) / (125 * 25 * pll->reference_freq / 100); radeon_crtc->ss.step = step_size; } @@ -1082,6 +1151,7 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc, u32 fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_NONE); u32 tmp, viewport_w, viewport_h; int r; + bool bypass_lut = false; /* no fb bound */ if (!atomic && !crtc->primary->fb) { @@ -1120,71 +1190,165 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc, radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL); radeon_bo_unreserve(rbo); - switch (target_fb->bits_per_pixel) { - case 8: + switch (target_fb->pixel_format) { + case DRM_FORMAT_C8: fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_8BPP) | EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_INDEXED)); break; - case 15: + case DRM_FORMAT_XRGB4444: + case DRM_FORMAT_ARGB4444: + fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) | + EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB4444)); +#ifdef __BIG_ENDIAN + fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16); +#endif + break; + case DRM_FORMAT_XRGB1555: + case DRM_FORMAT_ARGB1555: fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) | EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB1555)); +#ifdef __BIG_ENDIAN + fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16); +#endif + break; + case DRM_FORMAT_BGRX5551: + case DRM_FORMAT_BGRA5551: + fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) | + EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_BGRA5551)); +#ifdef __BIG_ENDIAN + fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16); +#endif break; - case 16: + case DRM_FORMAT_RGB565: fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) | EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB565)); #ifdef __BIG_ENDIAN fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16); #endif break; - case 24: - case 32: + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ARGB8888: fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) | EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB8888)); #ifdef __BIG_ENDIAN fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32); #endif break; + case DRM_FORMAT_XRGB2101010: + case DRM_FORMAT_ARGB2101010: + fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) | + EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB2101010)); +#ifdef __BIG_ENDIAN + fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32); +#endif + /* Greater 8 bpc fb needs to bypass hw-lut to retain precision */ + bypass_lut = true; + break; + case DRM_FORMAT_BGRX1010102: + case DRM_FORMAT_BGRA1010102: + fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) | + EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_BGRA1010102)); +#ifdef __BIG_ENDIAN + fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32); +#endif + /* Greater 8 bpc fb needs to bypass hw-lut to retain precision */ + bypass_lut = true; + break; default: - DRM_ERROR("Unsupported screen depth %d\n", - target_fb->bits_per_pixel); + DRM_ERROR("Unsupported screen format %s\n", + drm_get_format_name(target_fb->pixel_format)); return -EINVAL; } if (tiling_flags & RADEON_TILING_MACRO) { - if (rdev->family >= CHIP_TAHITI) - tmp = rdev->config.si.tile_config; - else if (rdev->family >= CHIP_CAYMAN) - tmp = rdev->config.cayman.tile_config; - else - tmp = rdev->config.evergreen.tile_config; + evergreen_tiling_fields(tiling_flags, &bankw, &bankh, &mtaspect, &tile_split); - switch ((tmp & 0xf0) >> 4) { - case 0: /* 4 banks */ - fb_format |= EVERGREEN_GRPH_NUM_BANKS(EVERGREEN_ADDR_SURF_4_BANK); - break; - case 1: /* 8 banks */ - default: - fb_format |= EVERGREEN_GRPH_NUM_BANKS(EVERGREEN_ADDR_SURF_8_BANK); - break; - case 2: /* 16 banks */ - fb_format |= EVERGREEN_GRPH_NUM_BANKS(EVERGREEN_ADDR_SURF_16_BANK); - break; + /* Set NUM_BANKS. */ + if (rdev->family >= CHIP_TAHITI) { + unsigned index, num_banks; + + if (rdev->family >= CHIP_BONAIRE) { + unsigned tileb, tile_split_bytes; + + /* Calculate the macrotile mode index. */ + tile_split_bytes = 64 << tile_split; + tileb = 8 * 8 * target_fb->bits_per_pixel / 8; + tileb = min(tile_split_bytes, tileb); + + for (index = 0; tileb > 64; index++) + tileb >>= 1; + + if (index >= 16) { + DRM_ERROR("Wrong screen bpp (%u) or tile split (%u)\n", + target_fb->bits_per_pixel, tile_split); + return -EINVAL; + } + + num_banks = (rdev->config.cik.macrotile_mode_array[index] >> 6) & 0x3; + } else { + switch (target_fb->bits_per_pixel) { + case 8: + index = 10; + break; + case 16: + index = SI_TILE_MODE_COLOR_2D_SCANOUT_16BPP; + break; + default: + case 32: + index = SI_TILE_MODE_COLOR_2D_SCANOUT_32BPP; + break; + } + + num_banks = (rdev->config.si.tile_mode_array[index] >> 20) & 0x3; + } + + fb_format |= EVERGREEN_GRPH_NUM_BANKS(num_banks); + } else { + /* NI and older. */ + if (rdev->family >= CHIP_CAYMAN) + tmp = rdev->config.cayman.tile_config; + else + tmp = rdev->config.evergreen.tile_config; + + switch ((tmp & 0xf0) >> 4) { + case 0: /* 4 banks */ + fb_format |= EVERGREEN_GRPH_NUM_BANKS(EVERGREEN_ADDR_SURF_4_BANK); + break; + case 1: /* 8 banks */ + default: + fb_format |= EVERGREEN_GRPH_NUM_BANKS(EVERGREEN_ADDR_SURF_8_BANK); + break; + case 2: /* 16 banks */ + fb_format |= EVERGREEN_GRPH_NUM_BANKS(EVERGREEN_ADDR_SURF_16_BANK); + break; + } } fb_format |= EVERGREEN_GRPH_ARRAY_MODE(EVERGREEN_GRPH_ARRAY_2D_TILED_THIN1); - - evergreen_tiling_fields(tiling_flags, &bankw, &bankh, &mtaspect, &tile_split); fb_format |= EVERGREEN_GRPH_TILE_SPLIT(tile_split); fb_format |= EVERGREEN_GRPH_BANK_WIDTH(bankw); fb_format |= EVERGREEN_GRPH_BANK_HEIGHT(bankh); fb_format |= EVERGREEN_GRPH_MACRO_TILE_ASPECT(mtaspect); + if (rdev->family >= CHIP_BONAIRE) { + /* XXX need to know more about the surface tiling mode */ + fb_format |= CIK_GRPH_MICRO_TILE_MODE(CIK_DISPLAY_MICRO_TILING); + } } else if (tiling_flags & RADEON_TILING_MICRO) fb_format |= EVERGREEN_GRPH_ARRAY_MODE(EVERGREEN_GRPH_ARRAY_1D_TILED_THIN1); - if ((rdev->family == CHIP_TAHITI) || - (rdev->family == CHIP_PITCAIRN)) + if (rdev->family >= CHIP_BONAIRE) { + /* Read the pipe config from the 2D TILED SCANOUT mode. + * It should be the same for the other modes too, but not all + * modes set the pipe config field. */ + u32 pipe_config = (rdev->config.cik.tile_mode_array[10] >> 6) & 0x1f; + + fb_format |= CIK_GRPH_PIPE_CONFIG(pipe_config); + } else if ((rdev->family == CHIP_TAHITI) || + (rdev->family == CHIP_PITCAIRN)) fb_format |= SI_GRPH_PIPE_CONFIG(SI_ADDR_SURF_P8_32x32_8x16); - else if (rdev->family == CHIP_VERDE) + else if ((rdev->family == CHIP_VERDE) || + (rdev->family == CHIP_OLAND) || + (rdev->family == CHIP_HAINAN)) /* for completeness. HAINAN has no display hw */ fb_format |= SI_GRPH_PIPE_CONFIG(SI_ADDR_SURF_P4_8x16); switch (radeon_crtc->crtc_id) { @@ -1221,6 +1385,18 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc, WREG32(EVERGREEN_GRPH_CONTROL + radeon_crtc->crtc_offset, fb_format); WREG32(EVERGREEN_GRPH_SWAP_CONTROL + radeon_crtc->crtc_offset, fb_swap); + /* + * The LUT only has 256 slots for indexing by a 8 bpc fb. Bypass the LUT + * for > 8 bpc scanout to avoid truncation of fb indices to 8 msb's, to + * retain the full precision throughout the pipeline. + */ + WREG32_P(EVERGREEN_GRPH_LUT_10BIT_BYPASS_CONTROL + radeon_crtc->crtc_offset, + (bypass_lut ? EVERGREEN_LUT_10BIT_BYPASS_EN : 0), + ~EVERGREEN_LUT_10BIT_BYPASS_EN); + + if (bypass_lut) + DRM_DEBUG_KMS("Bypassing hardware LUT due to 10 bit fb scanout.\n"); + WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_X + radeon_crtc->crtc_offset, 0); WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_Y + radeon_crtc->crtc_offset, 0); WREG32(EVERGREEN_GRPH_X_START + radeon_crtc->crtc_offset, 0); @@ -1232,14 +1408,21 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc, WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset, fb_pitch_pixels); WREG32(EVERGREEN_GRPH_ENABLE + radeon_crtc->crtc_offset, 1); - WREG32(EVERGREEN_DESKTOP_HEIGHT + radeon_crtc->crtc_offset, - target_fb->height); + if (rdev->family >= CHIP_BONAIRE) + WREG32(CIK_LB_DESKTOP_HEIGHT + radeon_crtc->crtc_offset, + target_fb->height); + else + WREG32(EVERGREEN_DESKTOP_HEIGHT + radeon_crtc->crtc_offset, + target_fb->height); x &= ~3; y &= ~1; WREG32(EVERGREEN_VIEWPORT_START + radeon_crtc->crtc_offset, (x << 16) | y); viewport_w = crtc->mode.hdisplay; viewport_h = (crtc->mode.vdisplay + 1) & ~1; + if ((rdev->family >= CHIP_BONAIRE) && + (crtc->mode.flags & DRM_MODE_FLAG_INTERLACE)) + viewport_h *= 2; WREG32(EVERGREEN_VIEWPORT_SIZE + radeon_crtc->crtc_offset, (viewport_w << 16) | viewport_h); @@ -1249,8 +1432,8 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc, tmp &= ~EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN; WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, tmp); - /* set pageflip to happen anywhere in vblank interval */ - WREG32(EVERGREEN_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0); + /* set pageflip to happen only at start of vblank interval (front porch) */ + WREG32(EVERGREEN_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 3); if (!atomic && fb && fb != crtc->primary->fb) { radeon_fb = to_radeon_framebuffer(fb); @@ -1284,6 +1467,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc, u32 fb_swap = R600_D1GRPH_SWAP_ENDIAN_NONE; u32 tmp, viewport_w, viewport_h; int r; + bool bypass_lut = false; /* no fb bound */ if (!atomic && !crtc->primary->fb) { @@ -1321,18 +1505,30 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc, radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL); radeon_bo_unreserve(rbo); - switch (target_fb->bits_per_pixel) { - case 8: + switch (target_fb->pixel_format) { + case DRM_FORMAT_C8: fb_format = AVIVO_D1GRPH_CONTROL_DEPTH_8BPP | AVIVO_D1GRPH_CONTROL_8BPP_INDEXED; break; - case 15: + case DRM_FORMAT_XRGB4444: + case DRM_FORMAT_ARGB4444: + fb_format = + AVIVO_D1GRPH_CONTROL_DEPTH_16BPP | + AVIVO_D1GRPH_CONTROL_16BPP_ARGB4444; +#ifdef __BIG_ENDIAN + fb_swap = R600_D1GRPH_SWAP_ENDIAN_16BIT; +#endif + break; + case DRM_FORMAT_XRGB1555: fb_format = AVIVO_D1GRPH_CONTROL_DEPTH_16BPP | AVIVO_D1GRPH_CONTROL_16BPP_ARGB1555; +#ifdef __BIG_ENDIAN + fb_swap = R600_D1GRPH_SWAP_ENDIAN_16BIT; +#endif break; - case 16: + case DRM_FORMAT_RGB565: fb_format = AVIVO_D1GRPH_CONTROL_DEPTH_16BPP | AVIVO_D1GRPH_CONTROL_16BPP_RGB565; @@ -1340,8 +1536,8 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc, fb_swap = R600_D1GRPH_SWAP_ENDIAN_16BIT; #endif break; - case 24: - case 32: + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ARGB8888: fb_format = AVIVO_D1GRPH_CONTROL_DEPTH_32BPP | AVIVO_D1GRPH_CONTROL_32BPP_ARGB8888; @@ -1349,9 +1545,20 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc, fb_swap = R600_D1GRPH_SWAP_ENDIAN_32BIT; #endif break; + case DRM_FORMAT_XRGB2101010: + case DRM_FORMAT_ARGB2101010: + fb_format = + AVIVO_D1GRPH_CONTROL_DEPTH_32BPP | + AVIVO_D1GRPH_CONTROL_32BPP_ARGB2101010; +#ifdef __BIG_ENDIAN + fb_swap = R600_D1GRPH_SWAP_ENDIAN_32BIT; +#endif + /* Greater 8 bpc fb needs to bypass hw-lut to retain precision */ + bypass_lut = true; + break; default: - DRM_ERROR("Unsupported screen depth %d\n", - target_fb->bits_per_pixel); + DRM_ERROR("Unsupported screen format %s\n", + drm_get_format_name(target_fb->pixel_format)); return -EINVAL; } @@ -1390,6 +1597,13 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc, if (rdev->family >= CHIP_R600) WREG32(R600_D1GRPH_SWAP_CONTROL + radeon_crtc->crtc_offset, fb_swap); + /* LUT only has 256 slots for 8 bpc fb. Bypass for > 8 bpc scanout for precision */ + WREG32_P(AVIVO_D1GRPH_LUT_SEL + radeon_crtc->crtc_offset, + (bypass_lut ? AVIVO_LUT_10BIT_BYPASS_EN : 0), ~AVIVO_LUT_10BIT_BYPASS_EN); + + if (bypass_lut) + DRM_DEBUG_KMS("Bypassing hardware LUT due to 10 bit fb scanout.\n"); + WREG32(AVIVO_D1GRPH_SURFACE_OFFSET_X + radeon_crtc->crtc_offset, 0); WREG32(AVIVO_D1GRPH_SURFACE_OFFSET_Y + radeon_crtc->crtc_offset, 0); WREG32(AVIVO_D1GRPH_X_START + radeon_crtc->crtc_offset, 0); @@ -1418,8 +1632,8 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc, tmp &= ~AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN; WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, tmp); - /* set pageflip to happen anywhere in vblank interval */ - WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0); + /* set pageflip to happen only at start of vblank interval (front porch) */ + WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 3); if (!atomic && fb && fb != crtc->primary->fb) { radeon_fb = to_radeon_framebuffer(fb); @@ -1527,6 +1741,7 @@ static u32 radeon_get_pll_use_mask(struct drm_crtc *crtc) static int radeon_get_shared_dp_ppll(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; + struct radeon_device *rdev = dev->dev_private; struct drm_crtc *test_crtc; struct radeon_crtc *test_radeon_crtc; @@ -1536,6 +1751,10 @@ static int radeon_get_shared_dp_ppll(struct drm_crtc *crtc) test_radeon_crtc = to_radeon_crtc(test_crtc); if (test_radeon_crtc->encoder && ENCODER_MODE_IS_DP(atombios_get_encoder_mode(test_radeon_crtc->encoder))) { + /* PPLL2 is exclusive to UNIPHYA on DCE61 */ + if (ASIC_IS_DCE61(rdev) && !ASIC_IS_DCE8(rdev) && + test_radeon_crtc->pll_id == ATOM_PPLL2) + continue; /* for DP use the same PLL for all */ if (test_radeon_crtc->pll_id != ATOM_PPLL_INVALID) return test_radeon_crtc->pll_id; @@ -1557,6 +1776,7 @@ static int radeon_get_shared_nondp_ppll(struct drm_crtc *crtc) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct drm_device *dev = crtc->dev; + struct radeon_device *rdev = dev->dev_private; struct drm_crtc *test_crtc; struct radeon_crtc *test_radeon_crtc; u32 adjusted_clock, test_adjusted_clock; @@ -1572,6 +1792,10 @@ static int radeon_get_shared_nondp_ppll(struct drm_crtc *crtc) test_radeon_crtc = to_radeon_crtc(test_crtc); if (test_radeon_crtc->encoder && !ENCODER_MODE_IS_DP(atombios_get_encoder_mode(test_radeon_crtc->encoder))) { + /* PPLL2 is exclusive to UNIPHYA on DCE61 */ + if (ASIC_IS_DCE61(rdev) && !ASIC_IS_DCE8(rdev) && + test_radeon_crtc->pll_id == ATOM_PPLL2) + continue; /* check if we are already driving this connector with another crtc */ if (test_radeon_crtc->connector == radeon_crtc->connector) { /* if we are, return that pll */ @@ -1605,6 +1829,12 @@ static int radeon_get_shared_nondp_ppll(struct drm_crtc *crtc) * * Asic specific PLL information * + * DCE 8.x + * KB/KV + * - PPLL1, PPLL2 are available for all UNIPHY (both DP and non-DP) + * CI + * - PPLL0, PPLL1, PPLL2 are available for all UNIPHY (both DP and non-DP) and DAC + * * DCE 6.1 * - PPLL2 is only available to UNIPHYA (both DP and non-DP) * - PPLL0, PPLL1 are available for UNIPHYB/C/D/E/F (both DP and non-DP) @@ -1631,7 +1861,47 @@ static int radeon_atom_pick_pll(struct drm_crtc *crtc) u32 pll_in_use; int pll; - if (ASIC_IS_DCE61(rdev)) { + if (ASIC_IS_DCE8(rdev)) { + if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(radeon_crtc->encoder))) { + if (rdev->clock.dp_extclk) + /* skip PPLL programming if using ext clock */ + return ATOM_PPLL_INVALID; + else { + /* use the same PPLL for all DP monitors */ + pll = radeon_get_shared_dp_ppll(crtc); + if (pll != ATOM_PPLL_INVALID) + return pll; + } + } else { + /* use the same PPLL for all monitors with the same clock */ + pll = radeon_get_shared_nondp_ppll(crtc); + if (pll != ATOM_PPLL_INVALID) + return pll; + } + /* otherwise, pick one of the plls */ + if ((rdev->family == CHIP_KABINI) || + (rdev->family == CHIP_MULLINS)) { + /* KB/ML has PPLL1 and PPLL2 */ + pll_in_use = radeon_get_pll_use_mask(crtc); + if (!(pll_in_use & (1 << ATOM_PPLL2))) + return ATOM_PPLL2; + if (!(pll_in_use & (1 << ATOM_PPLL1))) + return ATOM_PPLL1; + DRM_ERROR("unable to allocate a PPLL\n"); + return ATOM_PPLL_INVALID; + } else { + /* CI/KV has PPLL0, PPLL1, and PPLL2 */ + pll_in_use = radeon_get_pll_use_mask(crtc); + if (!(pll_in_use & (1 << ATOM_PPLL2))) + return ATOM_PPLL2; + if (!(pll_in_use & (1 << ATOM_PPLL1))) + return ATOM_PPLL1; + if (!(pll_in_use & (1 << ATOM_PPLL0))) + return ATOM_PPLL0; + DRM_ERROR("unable to allocate a PPLL\n"); + return ATOM_PPLL_INVALID; + } + } else if (ASIC_IS_DCE61(rdev)) { struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; @@ -1775,6 +2045,9 @@ int atombios_crtc_mode_set(struct drm_crtc *crtc, (ATOM_DEVICE_TV_SUPPORT | ATOM_DEVICE_CV_SUPPORT)) is_tvcv = true; + if (!radeon_crtc->adjusted_clock) + return -EINVAL; + atombios_crtc_set_pll(crtc, adjusted_mode); if (ASIC_IS_DCE4(rdev)) @@ -1793,6 +2066,10 @@ int atombios_crtc_mode_set(struct drm_crtc *crtc, atombios_crtc_set_base(crtc, x, y, old_fb); atombios_overscan_setup(crtc, mode, adjusted_mode); atombios_scaler_setup(crtc); + radeon_cursor_reset(crtc); + /* update the hw version fpr dpm */ + radeon_crtc->hw_mode = *adjusted_mode; + return 0; } @@ -1817,6 +2094,12 @@ static bool atombios_crtc_mode_fixup(struct drm_crtc *crtc, radeon_crtc->connector = NULL; return false; } + if (radeon_crtc->encoder) { + struct radeon_encoder *radeon_encoder = + to_radeon_encoder(radeon_crtc->encoder); + + radeon_crtc->output_csc = radeon_encoder->output_csc; + } if (!radeon_crtc_scaling_mode_fixup(crtc, mode, adjusted_mode)) return false; if (!atombios_crtc_prepare_pll(crtc, adjusted_mode)) @@ -1833,12 +2116,9 @@ static bool atombios_crtc_mode_fixup(struct drm_crtc *crtc, static void atombios_crtc_prepare(struct drm_crtc *crtc) { - struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct drm_device *dev = crtc->dev; struct radeon_device *rdev = dev->dev_private; - radeon_crtc->in_mode_set = true; - /* disable crtc pair power gating before programming */ if (ASIC_IS_DCE6(rdev)) atombios_powergate_crtc(crtc, ATOM_DISABLE); @@ -1849,11 +2129,8 @@ static void atombios_crtc_prepare(struct drm_crtc *crtc) static void atombios_crtc_commit(struct drm_crtc *crtc) { - struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); - atombios_crtc_dpms(crtc, DRM_MODE_DPMS_ON); atombios_lock_crtc(crtc, ATOM_DISABLE); - radeon_crtc->in_mode_set = false; } static void atombios_crtc_disable(struct drm_crtc *crtc) @@ -1865,6 +2142,27 @@ static void atombios_crtc_disable(struct drm_crtc *crtc) int i; atombios_crtc_dpms(crtc, DRM_MODE_DPMS_OFF); + if (crtc->primary->fb) { + int r; + struct radeon_framebuffer *radeon_fb; + struct radeon_bo *rbo; + + radeon_fb = to_radeon_framebuffer(crtc->primary->fb); + rbo = gem_to_radeon_bo(radeon_fb->obj); + r = radeon_bo_reserve(rbo, false); + if (unlikely(r)) + DRM_ERROR("failed to reserve rbo before unpin\n"); + else { + radeon_bo_unpin(rbo); + radeon_bo_unreserve(rbo); + } + } + /* disable the GRPH */ + if (ASIC_IS_DCE4(rdev)) + WREG32(EVERGREEN_GRPH_ENABLE + radeon_crtc->crtc_offset, 0); + else if (ASIC_IS_AVIVO(rdev)) + WREG32(AVIVO_D1GRPH_ENABLE + radeon_crtc->crtc_offset, 0); + if (ASIC_IS_DCE6(rdev)) atombios_powergate_crtc(crtc, ATOM_ENABLE); @@ -1889,7 +2187,10 @@ static void atombios_crtc_disable(struct drm_crtc *crtc) break; case ATOM_PPLL0: /* disable the ppll */ - if (ASIC_IS_DCE61(rdev)) + if ((rdev->family == CHIP_ARUBA) || + (rdev->family == CHIP_KAVERI) || + (rdev->family == CHIP_BONAIRE) || + (rdev->family == CHIP_HAWAII)) atombios_crtc_program_pll(crtc, radeon_crtc->crtc_id, radeon_crtc->pll_id, 0, 0, ATOM_DISABLE, 0, 0, 0, 0, 0, false, &ss); break; diff --git a/sys/dev/pci/drm/radeon/atombios_dp.c b/sys/dev/pci/drm/radeon/atombios_dp.c index 5e1ed107e26..b590be0b5a0 100644 --- a/sys/dev/pci/drm/radeon/atombios_dp.c +++ b/sys/dev/pci/drm/radeon/atombios_dp.c @@ -47,34 +47,32 @@ static char *pre_emph_names[] = { /***** radeon AUX functions *****/ -/* Atom needs data in little endian format - * so swap as appropriate when copying data to - * or from atom. Note that atom operates on - * dw units. +/* Atom needs data in little endian format so swap as appropriate when copying + * data to or from atom. Note that atom operates on dw units. + * + * Use to_le=true when sending data to atom and provide at least + * ALIGN(num_bytes,4) bytes in the dst buffer. + * + * Use to_le=false when receiving data from atom and provide ALIGN(num_bytes,4) + * byes in the src buffer. */ void radeon_atom_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le) { #ifdef __BIG_ENDIAN - u8 src_tmp[20], dst_tmp[20]; /* used for byteswapping */ - u32 *dst32, *src32; + u32 src_tmp[5], dst_tmp[5]; int i; + u8 align_num_bytes = roundup2(num_bytes, 4); - memcpy(src_tmp, src, num_bytes); - src32 = (u32 *)src_tmp; - dst32 = (u32 *)dst_tmp; if (to_le) { - for (i = 0; i < ((num_bytes + 3) / 4); i++) - dst32[i] = cpu_to_le32(src32[i]); - memcpy(dst, dst_tmp, num_bytes); + memcpy(src_tmp, src, num_bytes); + for (i = 0; i < align_num_bytes / 4; i++) + dst_tmp[i] = cpu_to_le32(src_tmp[i]); + memcpy(dst, dst_tmp, align_num_bytes); } else { - u8 dws = num_bytes & ~3; - for (i = 0; i < ((num_bytes + 3) / 4); i++) - dst32[i] = le32_to_cpu(src32[i]); - memcpy(dst, dst_tmp, dws); - if (num_bytes % 4) { - for (i = 0; i < (num_bytes % 4); i++) - dst[dws+i] = dst_tmp[dws+i]; - } + memcpy(src_tmp, src, align_num_bytes); + for (i = 0; i < align_num_bytes / 4; i++) + dst_tmp[i] = le32_to_cpu(src_tmp[i]); + memcpy(dst, dst_tmp, num_bytes); } #else memcpy(dst, src, num_bytes); @@ -101,6 +99,9 @@ static int radeon_process_aux_ch(struct radeon_i2c_chan *chan, memset(&args, 0, sizeof(args)); + mutex_lock(&chan->mutex); + mutex_lock(&rdev->mode_info.atom_context->scratch_mutex); + base = (unsigned char *)(rdev->mode_info.atom_context->scratch + 1); radeon_atom_copy_swap(base, send, send_bytes, true); @@ -113,7 +114,7 @@ static int radeon_process_aux_ch(struct radeon_i2c_chan *chan, if (ASIC_IS_DCE4(rdev)) args.v2.ucHPD_ID = chan->rec.hpd; - atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + atom_execute_table_scratch_unlocked(rdev->mode_info.atom_context, index, (uint32_t *)&args); *ack = args.v1.ucReplyStatus; @@ -147,6 +148,9 @@ static int radeon_process_aux_ch(struct radeon_i2c_chan *chan, r = recv_bytes; done: + mutex_unlock(&rdev->mode_info.atom_context->scratch_mutex); + mutex_unlock(&chan->mutex); + return r; } diff --git a/sys/dev/pci/drm/radeon/atombios_encoders.c b/sys/dev/pci/drm/radeon/atombios_encoders.c index b710c1d6ea9..65ca0364b3b 100644 --- a/sys/dev/pci/drm/radeon/atombios_encoders.c +++ b/sys/dev/pci/drm/radeon/atombios_encoders.c @@ -1,4 +1,3 @@ -/* $OpenBSD: atombios_encoders.c,v 1.12 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2007-11 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -28,6 +27,7 @@ #include #include #include "radeon.h" +#include "radeon_audio.h" #include "atom.h" extern int atom_debug; @@ -119,6 +119,7 @@ atombios_set_backlight_level(struct radeon_encoder *radeon_encoder, u8 level) case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: if (dig->backlight_level == 0) atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0); else { @@ -183,9 +184,15 @@ void radeon_atom_backlight_init(struct radeon_encoder *radeon_encoder, struct backlight_properties props; struct radeon_backlight_privdata *pdata; struct radeon_encoder_atom_dig *dig; - u8 backlight_level; char bl_name[16]; + /* Mac laptops with multiple GPUs use the gmux driver for backlight + * so don't register a backlight device + */ + if ((rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) && + (rdev->pdev->device == 0x6741)) + return; + if (!radeon_encoder->enc_priv) return; @@ -204,11 +211,14 @@ void radeon_atom_backlight_init(struct radeon_encoder *radeon_encoder, memset(&props, 0, sizeof(props)); props.max_brightness = RADEON_MAX_BL_LEVEL; props.type = BACKLIGHT_RAW; -#ifdef __linux__ +#ifdef notyet snprintf(bl_name, sizeof(bl_name), "radeon_bl%d", dev->primary->index); +#else + snprintf(bl_name, sizeof(bl_name), + "radeon_bl%d", 0); #endif - bd = backlight_device_register(bl_name, &drm_connector->kdev, + bd = backlight_device_register(bl_name, drm_connector->kdev, pdata, &radeon_atom_backlight_ops, &props); if (IS_ERR(bd)) { DRM_ERROR("Backlight registration failed\n"); @@ -217,16 +227,22 @@ void radeon_atom_backlight_init(struct radeon_encoder *radeon_encoder, pdata->encoder = radeon_encoder; - backlight_level = radeon_atom_get_backlight_level_from_reg(rdev); - dig = radeon_encoder->enc_priv; dig->bl_dev = bd; bd->props.brightness = radeon_atom_backlight_get_brightness(bd); + /* Set a reasonable default here if the level is 0 otherwise + * fbdev will attempt to turn the backlight on after console + * unblanking and it will try and restore 0 which turns the backlight + * off again. + */ + if (bd->props.brightness == 0) + bd->props.brightness = RADEON_MAX_BL_LEVEL; bd->props.power = FB_BLANK_UNBLANK; backlight_update_status(bd); DRM_INFO("radeon atom DIG backlight initialized\n"); + rdev->mode_info.bl_encoder = radeon_encoder; return; @@ -282,28 +298,6 @@ static void radeon_atom_backlight_exit(struct radeon_encoder *encoder) bool radeon_atom_get_tv_timings(struct radeon_device *rdev, int index, struct drm_display_mode *mode); - -static inline bool radeon_encoder_is_digital(struct drm_encoder *encoder) -{ - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - switch (radeon_encoder->encoder_id) { - case ENCODER_OBJECT_ID_INTERNAL_LVDS: - case ENCODER_OBJECT_ID_INTERNAL_TMDS1: - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1: - case ENCODER_OBJECT_ID_INTERNAL_LVTM1: - case ENCODER_OBJECT_ID_INTERNAL_DVO1: - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: - case ENCODER_OBJECT_ID_INTERNAL_DDI: - case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: - case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: - case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: - return true; - default: - return false; - } -} - static bool radeon_atom_mode_fixup(struct drm_encoder *encoder, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode) @@ -321,12 +315,14 @@ static bool radeon_atom_mode_fixup(struct drm_encoder *encoder, && (mode->crtc_vsync_start < (mode->crtc_vdisplay + 2))) adjusted_mode->crtc_vsync_start = adjusted_mode->crtc_vdisplay + 2; - /* get the native mode for LVDS */ - if (radeon_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT)) - radeon_panel_mode_fixup(encoder, adjusted_mode); + /* vertical FP must be at least 1 */ + if (mode->crtc_vsync_start == mode->crtc_vdisplay) + adjusted_mode->crtc_vsync_start++; - /* get the native mode for TV */ - if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) { + /* get the native mode for scaling */ + if (radeon_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT)) { + radeon_panel_mode_fixup(encoder, adjusted_mode); + } else if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) { struct radeon_encoder_atom_dac *tv_dac = radeon_encoder->enc_priv; if (tv_dac) { if (tv_dac->tv_std == TV_STD_NTSC || @@ -336,6 +332,8 @@ static bool radeon_atom_mode_fixup(struct drm_encoder *encoder, else radeon_atom_get_tv_timings(rdev, 1, adjusted_mode); } + } else if (radeon_encoder->rmx_type != RMX_OFF) { + radeon_panel_mode_fixup(encoder, adjusted_mode); } if (ASIC_IS_DCE3(rdev) && @@ -458,11 +456,12 @@ atombios_tv_setup(struct drm_encoder *encoder, int action) static u8 radeon_atom_get_bpc(struct drm_encoder *encoder) { - struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); int bpc = 8; - if (connector) - bpc = radeon_get_monitor_bpc(connector); + if (encoder->crtc) { + struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); + bpc = radeon_crtc->bpc; + } switch (bpc) { case 0: @@ -481,11 +480,11 @@ static u8 radeon_atom_get_bpc(struct drm_encoder *encoder) } } - union dvo_encoder_control { ENABLE_EXTERNAL_TMDS_ENCODER_PS_ALLOCATION ext_tmds; DVO_ENCODER_CONTROL_PS_ALLOCATION dvo; DVO_ENCODER_CONTROL_PS_ALLOCATION_V3 dvo_v3; + DVO_ENCODER_CONTROL_PS_ALLOCATION_V1_4 dvo_v4; }; void @@ -535,6 +534,13 @@ atombios_dvo_setup(struct drm_encoder *encoder, int action) args.dvo_v3.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10); args.dvo_v3.ucDVOConfig = 0; /* XXX */ break; + case 4: + /* DCE8 */ + args.dvo_v4.ucAction = action; + args.dvo_v4.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10); + args.dvo_v4.ucDVOConfig = 0; /* XXX */ + args.dvo_v4.ucBitPerColor = radeon_atom_get_bpc(encoder); + break; default: DRM_ERROR("Unknown table version %d, %d\n", frev, crev); break; @@ -675,7 +681,15 @@ atombios_get_encoder_mode(struct drm_encoder *encoder) struct drm_connector *connector; struct radeon_connector *radeon_connector; struct radeon_connector_atom_dig *dig_connector; + struct radeon_encoder_atom_dig *dig_enc; + if (radeon_encoder_is_digital(encoder)) { + dig_enc = radeon_encoder->enc_priv; + if (dig_enc->active_mst_links) + return ATOM_ENCODER_MODE_DP_MST; + } + if (radeon_encoder->is_mst_encoder || radeon_encoder->offset) + return ATOM_ENCODER_MODE_DP_MST; /* dp bridges are always DP */ if (radeon_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE) return ATOM_ENCODER_MODE_DP; @@ -696,24 +710,37 @@ atombios_get_encoder_mode(struct drm_encoder *encoder) switch (connector->connector_type) { case DRM_MODE_CONNECTOR_DVII: case DRM_MODE_CONNECTOR_HDMIB: /* HDMI-B is basically DL-DVI; analog works fine */ - if (drm_detect_hdmi_monitor(radeon_connector->edid) && - radeon_audio && - !ASIC_IS_DCE6(rdev)) /* remove once we support DCE6 */ - return ATOM_ENCODER_MODE_HDMI; - else if (radeon_connector->use_digital) + if (radeon_audio != 0) { + if (radeon_connector->use_digital && + (radeon_connector->audio == RADEON_AUDIO_ENABLE)) + return ATOM_ENCODER_MODE_HDMI; + else if (drm_detect_hdmi_monitor(radeon_connector_edid(connector)) && + (radeon_connector->audio == RADEON_AUDIO_AUTO)) + return ATOM_ENCODER_MODE_HDMI; + else if (radeon_connector->use_digital) + return ATOM_ENCODER_MODE_DVI; + else + return ATOM_ENCODER_MODE_CRT; + } else if (radeon_connector->use_digital) { return ATOM_ENCODER_MODE_DVI; - else + } else { return ATOM_ENCODER_MODE_CRT; + } break; case DRM_MODE_CONNECTOR_DVID: case DRM_MODE_CONNECTOR_HDMIA: default: - if (drm_detect_hdmi_monitor(radeon_connector->edid) && - radeon_audio && - !ASIC_IS_DCE6(rdev)) /* remove once we support DCE6 */ - return ATOM_ENCODER_MODE_HDMI; - else + if (radeon_audio != 0) { + if (radeon_connector->audio == RADEON_AUDIO_ENABLE) + return ATOM_ENCODER_MODE_HDMI; + else if (drm_detect_hdmi_monitor(radeon_connector_edid(connector)) && + (radeon_connector->audio == RADEON_AUDIO_AUTO)) + return ATOM_ENCODER_MODE_HDMI; + else + return ATOM_ENCODER_MODE_DVI; + } else { return ATOM_ENCODER_MODE_DVI; + } break; case DRM_MODE_CONNECTOR_LVDS: return ATOM_ENCODER_MODE_LVDS; @@ -721,16 +748,29 @@ atombios_get_encoder_mode(struct drm_encoder *encoder) case DRM_MODE_CONNECTOR_DisplayPort: dig_connector = radeon_connector->con_priv; if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || - (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) + (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) { + if (radeon_audio != 0 && + drm_detect_monitor_audio(radeon_connector_edid(connector)) && + ASIC_IS_DCE4(rdev) && !ASIC_IS_DCE5(rdev)) + return ATOM_ENCODER_MODE_DP_AUDIO; return ATOM_ENCODER_MODE_DP; - else if (drm_detect_hdmi_monitor(radeon_connector->edid) && - radeon_audio && - !ASIC_IS_DCE6(rdev)) /* remove once we support DCE6 */ - return ATOM_ENCODER_MODE_HDMI; - else + } else if (radeon_audio != 0) { + if (radeon_connector->audio == RADEON_AUDIO_ENABLE) + return ATOM_ENCODER_MODE_HDMI; + else if (drm_detect_hdmi_monitor(radeon_connector_edid(connector)) && + (radeon_connector->audio == RADEON_AUDIO_AUTO)) + return ATOM_ENCODER_MODE_HDMI; + else + return ATOM_ENCODER_MODE_DVI; + } else { return ATOM_ENCODER_MODE_DVI; + } break; case DRM_MODE_CONNECTOR_eDP: + if (radeon_audio != 0 && + drm_detect_monitor_audio(radeon_connector_edid(connector)) && + ASIC_IS_DCE4(rdev) && !ASIC_IS_DCE5(rdev)) + return ATOM_ENCODER_MODE_DP_AUDIO; return ATOM_ENCODER_MODE_DP; case DRM_MODE_CONNECTOR_DVIA: case DRM_MODE_CONNECTOR_VGA: @@ -801,7 +841,7 @@ union dig_encoder_control { }; void -atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mode) +atombios_dig_encoder_setup2(struct drm_encoder *encoder, int action, int panel_mode, int enc_override) { struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; @@ -861,8 +901,6 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo else args.v1.ucLaneNum = 4; - if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode) && (dp_clock == 270000)) - args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ; switch (radeon_encoder->encoder_id) { case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER1; @@ -879,6 +917,10 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKB; else args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKA; + + if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode) && (dp_clock == 270000)) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ; + break; case 2: case 3: @@ -898,7 +940,10 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo if (ENCODER_MODE_IS_DP(args.v3.ucEncoderMode) && (dp_clock == 270000)) args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ; - args.v3.acConfig.ucDigSel = dig->dig_encoder; + if (enc_override != -1) + args.v3.acConfig.ucDigSel = enc_override; + else + args.v3.acConfig.ucDigSel = dig->dig_encoder; args.v3.ucBitPerColor = radeon_atom_get_bpc(encoder); break; case 4: @@ -917,12 +962,20 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo args.v4.ucLaneNum = 4; if (ENCODER_MODE_IS_DP(args.v4.ucEncoderMode)) { - if (dp_clock == 270000) - args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ; - else if (dp_clock == 540000) + if (dp_clock == 540000) args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_5_40GHZ; + else if (dp_clock == 324000) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_3_24GHZ; + else if (dp_clock == 270000) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ; + else + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_1_62GHZ; } - args.v4.acConfig.ucDigSel = dig->dig_encoder; + + if (enc_override != -1) + args.v4.acConfig.ucDigSel = enc_override; + else + args.v4.acConfig.ucDigSel = dig->dig_encoder; args.v4.ucBitPerColor = radeon_atom_get_bpc(encoder); if (hpd_id == RADEON_HPD_NONE) args.v4.ucHPD_ID = 0; @@ -943,6 +996,12 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo } +void +atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mode) +{ + atombios_dig_encoder_setup2(encoder, action, panel_mode, -1); +} + union dig_transmitter_control { DIG_TRANSMITTER_CONTROL_PS_ALLOCATION v1; DIG_TRANSMITTER_CONTROL_PARAMETERS_V2 v2; @@ -952,7 +1011,7 @@ union dig_transmitter_control { }; void -atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t lane_num, uint8_t lane_set) +atombios_dig_transmitter_setup2(struct drm_encoder *encoder, int action, uint8_t lane_num, uint8_t lane_set, int fe) { struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; @@ -1014,6 +1073,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: index = GetIndexIntoMasterTable(COMMAND, UNIPHYTransmitterControl); break; case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: @@ -1273,6 +1333,9 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t else args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYE; break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYG; + break; } if (is_dp) args.v5.ucLaneNum = dp_lane_count; @@ -1298,7 +1361,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t args.v5.asConfig.ucHPDSel = 0; else args.v5.asConfig.ucHPDSel = hpd_id + 1; - args.v5.ucDigEncoderSel = 1 << dig_encoder; + args.v5.ucDigEncoderSel = (fe != -1) ? (1 << fe) : (1 << dig_encoder); args.v5.ucDPLaneSet = lane_set; break; default: @@ -1314,6 +1377,12 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); } +void +atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t lane_num, uint8_t lane_set) +{ + atombios_dig_transmitter_setup2(encoder, action, lane_num, lane_set, -1); +} + bool atombios_set_edp_panel_power(struct drm_connector *connector, int action) { @@ -1567,8 +1636,14 @@ radeon_atom_encoder_dpms_avivo(struct drm_encoder *encoder, int mode) } else atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { - args.ucAction = ATOM_LCD_BLON; - atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + if (rdev->mode_info.bl_encoder) { + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + + atombios_set_backlight_level(radeon_encoder, dig->backlight_level); + } else { + args.ucAction = ATOM_LCD_BLON; + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + } } break; case DRM_MODE_DPMS_STANDBY: @@ -1595,10 +1670,16 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode) struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); struct radeon_connector *radeon_connector = NULL; struct radeon_connector_atom_dig *radeon_dig_connector = NULL; + bool travis_quirk = false; if (connector) { radeon_connector = to_radeon_connector(connector); radeon_dig_connector = radeon_connector->con_priv; + if ((radeon_connector_encoder_get_dp_bridge_encoder_id(connector) == + ENCODER_OBJECT_ID_TRAVIS) && + (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) && + !ASIC_IS_DCE5(rdev)) + travis_quirk = true; } switch (mode) { @@ -1619,25 +1700,13 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode) atombios_external_encoder_setup(encoder, ext_encoder, EXTERNAL_ENCODER_ACTION_V3_ENCODER_SETUP); } - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0); } else if (ASIC_IS_DCE4(rdev)) { /* setup and enable the encoder */ atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_SETUP, 0); - /* enable the transmitter */ - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0); - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE_OUTPUT, 0, 0); } else { /* setup and enable the encoder and transmitter */ atombios_dig_encoder_setup(encoder, ATOM_ENABLE, 0); atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_SETUP, 0, 0); - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0); - /* some dce3.x boards have a bug in their transmitter control table. - * ACTION_ENABLE_OUTPUT can probably be dropped since ACTION_ENABLE - * does the same thing and more. - */ - if ((rdev->family != CHIP_RV710) && (rdev->family != CHIP_RV730) && - (rdev->family != CHIP_RS780) && (rdev->family != CHIP_RS880)) - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE_OUTPUT, 0, 0); } if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector) { if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { @@ -1645,73 +1714,65 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode) ATOM_TRANSMITTER_ACTION_POWER_ON); radeon_dig_connector->edp_on = true; } + } + /* enable the transmitter */ + atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0); + if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector) { + /* DP_SET_POWER_D0 is set in radeon_dp_link_train */ radeon_dp_link_train(encoder, connector); if (ASIC_IS_DCE4(rdev)) atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_ON, 0); } - if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLON, 0, 0); + if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { + if (rdev->mode_info.bl_encoder) + atombios_set_backlight_level(radeon_encoder, dig->backlight_level); + else + atombios_dig_transmitter_setup(encoder, + ATOM_TRANSMITTER_ACTION_LCD_BLON, 0, 0); + } + if (ext_encoder) + atombios_external_encoder_setup(encoder, ext_encoder, ATOM_ENABLE); break; case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: case DRM_MODE_DPMS_OFF: - if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) { - /* disable the transmitter */ - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0); - } else if (ASIC_IS_DCE4(rdev)) { + + /* don't power off encoders with active MST links */ + if (dig->active_mst_links) + return; + + if (ASIC_IS_DCE4(rdev)) { + if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector) + atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0); + } + if (ext_encoder) + atombios_external_encoder_setup(encoder, ext_encoder, ATOM_DISABLE); + if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) + atombios_dig_transmitter_setup(encoder, + ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0); + + if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && + connector && !travis_quirk) + radeon_dp_set_rx_power_state(connector, DP_SET_POWER_D3); + if (ASIC_IS_DCE4(rdev)) { /* disable the transmitter */ - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE_OUTPUT, 0, 0); - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0); + atombios_dig_transmitter_setup(encoder, + ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0); } else { /* disable the encoder and transmitter */ - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE_OUTPUT, 0, 0); - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0); + atombios_dig_transmitter_setup(encoder, + ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0); atombios_dig_encoder_setup(encoder, ATOM_DISABLE, 0); } if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector) { - if (ASIC_IS_DCE4(rdev)) - atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0); + if (travis_quirk) + radeon_dp_set_rx_power_state(connector, DP_SET_POWER_D3); if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { atombios_set_edp_panel_power(connector, ATOM_TRANSMITTER_ACTION_POWER_OFF); radeon_dig_connector->edp_on = false; } } - if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) - atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0); - break; - } -} - -static void -radeon_atom_encoder_dpms_ext(struct drm_encoder *encoder, - struct drm_encoder *ext_encoder, - int mode) -{ - struct drm_device *dev = encoder->dev; - struct radeon_device *rdev = dev->dev_private; - - switch (mode) { - case DRM_MODE_DPMS_ON: - default: - if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE61(rdev)) { - atombios_external_encoder_setup(encoder, ext_encoder, - EXTERNAL_ENCODER_ACTION_V3_ENABLE_OUTPUT); - atombios_external_encoder_setup(encoder, ext_encoder, - EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING_OFF); - } else - atombios_external_encoder_setup(encoder, ext_encoder, ATOM_ENABLE); - break; - case DRM_MODE_DPMS_STANDBY: - case DRM_MODE_DPMS_SUSPEND: - case DRM_MODE_DPMS_OFF: - if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE61(rdev)) { - atombios_external_encoder_setup(encoder, ext_encoder, - EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING); - atombios_external_encoder_setup(encoder, ext_encoder, - EXTERNAL_ENCODER_ACTION_V3_DISABLE_OUTPUT); - } else - atombios_external_encoder_setup(encoder, ext_encoder, ATOM_DISABLE); break; } } @@ -1722,11 +1783,17 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode) struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - struct drm_encoder *ext_encoder = radeon_get_external_encoder(encoder); + int encoder_mode = atombios_get_encoder_mode(encoder); DRM_DEBUG_KMS("encoder dpms %d to mode %d, devices %08x, active_devices %08x\n", radeon_encoder->encoder_id, mode, radeon_encoder->devices, radeon_encoder->active_device); + + if ((radeon_audio != 0) && + ((encoder_mode == ATOM_ENCODER_MODE_HDMI) || + ENCODER_MODE_IS_DP(encoder_mode))) + radeon_audio_dpms(encoder, mode); + switch (radeon_encoder->encoder_id) { case ENCODER_OBJECT_ID_INTERNAL_TMDS1: case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1: @@ -1741,6 +1808,7 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode) case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: radeon_atom_encoder_dpms_dig(encoder, mode); break; @@ -1781,9 +1849,6 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode) return; } - if (ext_encoder) - radeon_atom_encoder_dpms_ext(encoder, ext_encoder, mode); - radeon_atombios_encoder_dpms_scratch_regs(encoder, (mode == DRM_MODE_DPMS_ON) ? true : false); } @@ -1881,6 +1946,7 @@ atombios_set_encoder_crtc_source(struct drm_encoder *encoder) case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: dig = radeon_encoder->enc_priv; switch (dig->dig_encoder) { @@ -1902,6 +1968,9 @@ atombios_set_encoder_crtc_source(struct drm_encoder *encoder) case 5: args.v2.ucEncoderID = ASIC_INT_DIG6_ENCODER_ID; break; + case 6: + args.v2.ucEncoderID = ASIC_INT_DIG7_ENCODER_ID; + break; } break; case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: @@ -1938,6 +2007,53 @@ atombios_set_encoder_crtc_source(struct drm_encoder *encoder) radeon_atombios_encoder_crtc_scratch_regs(encoder, radeon_crtc->crtc_id); } +void +atombios_set_mst_encoder_crtc_source(struct drm_encoder *encoder, int fe) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); + int index = GetIndexIntoMasterTable(COMMAND, SelectCRTC_Source); + uint8_t frev, crev; + union crtc_source_param args; + + memset(&args, 0, sizeof(args)); + + if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev)) + return; + + if (frev != 1 && crev != 2) + DRM_ERROR("Unknown table for MST %d, %d\n", frev, crev); + + args.v2.ucCRTC = radeon_crtc->crtc_id; + args.v2.ucEncodeMode = ATOM_ENCODER_MODE_DP_MST; + + switch (fe) { + case 0: + args.v2.ucEncoderID = ASIC_INT_DIG1_ENCODER_ID; + break; + case 1: + args.v2.ucEncoderID = ASIC_INT_DIG2_ENCODER_ID; + break; + case 2: + args.v2.ucEncoderID = ASIC_INT_DIG3_ENCODER_ID; + break; + case 3: + args.v2.ucEncoderID = ASIC_INT_DIG4_ENCODER_ID; + break; + case 4: + args.v2.ucEncoderID = ASIC_INT_DIG5_ENCODER_ID; + break; + case 5: + args.v2.ucEncoderID = ASIC_INT_DIG6_ENCODER_ID; + break; + case 6: + args.v2.ucEncoderID = ASIC_INT_DIG7_ENCODER_ID; + break; + } + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); +} + static void atombios_apply_encoder_quirks(struct drm_encoder *encoder, struct drm_display_mode *mode) @@ -1964,7 +2080,13 @@ atombios_apply_encoder_quirks(struct drm_encoder *encoder, /* set scaler clears this on some chips */ if (ASIC_IS_AVIVO(rdev) && (!(radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)))) { - if (ASIC_IS_DCE4(rdev)) { + if (ASIC_IS_DCE8(rdev)) { + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + WREG32(CIK_LB_DATA_FORMAT + radeon_crtc->crtc_offset, + CIK_INTERLEAVE_EN); + else + WREG32(CIK_LB_DATA_FORMAT + radeon_crtc->crtc_offset, 0); + } else if (ASIC_IS_DCE4(rdev)) { if (mode->flags & DRM_MODE_FLAG_INTERLACE) WREG32(EVERGREEN_DATA_FORMAT + radeon_crtc->crtc_offset, EVERGREEN_INTERLEAVE_EN); @@ -1980,7 +2102,14 @@ atombios_apply_encoder_quirks(struct drm_encoder *encoder, } } -static int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder) +void radeon_atom_release_dig_encoder(struct radeon_device *rdev, int enc_idx) +{ + if (enc_idx < 0) + return; + rdev->mode_info.active_encoders &= ~(1 << enc_idx); +} + +int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder, int fe_idx) { struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; @@ -1989,68 +2118,87 @@ static int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder) struct drm_encoder *test_encoder; struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; uint32_t dig_enc_in_use = 0; + int enc_idx = -1; + if (fe_idx >= 0) { + enc_idx = fe_idx; + goto assigned; + } if (ASIC_IS_DCE6(rdev)) { /* DCE6 */ switch (radeon_encoder->encoder_id) { case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: if (dig->linkb) - return 1; + enc_idx = 1; else - return 0; + enc_idx = 0; break; case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: if (dig->linkb) - return 3; + enc_idx = 3; else - return 2; + enc_idx = 2; break; case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: if (dig->linkb) - return 5; + enc_idx = 5; else - return 4; + enc_idx = 4; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + enc_idx = 6; break; } + goto assigned; } else if (ASIC_IS_DCE4(rdev)) { /* DCE4/5 */ if (ASIC_IS_DCE41(rdev) && !ASIC_IS_DCE61(rdev)) { /* ontario follows DCE4 */ if (rdev->family == CHIP_PALM) { if (dig->linkb) - return 1; + enc_idx = 1; else - return 0; + enc_idx = 0; } else /* llano follows DCE3.2 */ - return radeon_crtc->crtc_id; + enc_idx = radeon_crtc->crtc_id; } else { switch (radeon_encoder->encoder_id) { case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: if (dig->linkb) - return 1; + enc_idx = 1; else - return 0; + enc_idx = 0; break; case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: if (dig->linkb) - return 3; + enc_idx = 3; else - return 2; + enc_idx = 2; break; case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: if (dig->linkb) - return 5; + enc_idx = 5; else - return 4; + enc_idx = 4; break; } } + goto assigned; } - /* on DCE32 and encoder can driver any block so just crtc id */ + /* + * On DCE32 any encoder can drive any block so usually just use crtc id, + * but Apple thinks different at least on iMac10,1, so there use linkb, + * otherwise the internal eDP panel will stay dark. + */ if (ASIC_IS_DCE32(rdev)) { - return radeon_crtc->crtc_id; + if (dmi_match(DMI_PRODUCT_NAME, "iMac10,1")) + enc_idx = (dig->linkb) ? 1 : 0; + else + enc_idx = radeon_crtc->crtc_id; + + goto assigned; } /* on DCE3 - LVTMA can only be driven by DIGB */ @@ -2078,6 +2226,17 @@ static int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder) if (!(dig_enc_in_use & 1)) return 0; return 1; + +assigned: + if (enc_idx == -1) { + DRM_ERROR("Got encoder index incorrect - returning 0\n"); + return 0; + } + if (rdev->mode_info.active_encoders & (1 << enc_idx)) { + DRM_ERROR("chosen encoder in use %d\n", enc_idx); + } + rdev->mode_info.active_encoders |= (1 << enc_idx); + return enc_idx; } /* This only needs to be called once at startup */ @@ -2095,6 +2254,7 @@ radeon_atom_encoder_init(struct radeon_device *rdev) case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_INIT, 0, 0); break; @@ -2116,6 +2276,8 @@ radeon_atom_encoder_mode_set(struct drm_encoder *encoder, struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + int encoder_mode; radeon_encoder->pixel_clock = adjusted_mode->clock; @@ -2139,6 +2301,7 @@ radeon_atom_encoder_mode_set(struct drm_encoder *encoder, case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: /* handled in dpms */ break; @@ -2163,15 +2326,11 @@ radeon_atom_encoder_mode_set(struct drm_encoder *encoder, atombios_apply_encoder_quirks(encoder, adjusted_mode); - if (atombios_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_HDMI) { - r600_hdmi_enable(encoder); - if (ASIC_IS_DCE6(rdev)) - ; /* TODO (use pointers instead of if-s?) */ - else if (ASIC_IS_DCE4(rdev)) - evergreen_hdmi_setmode(encoder, adjusted_mode); - else - r600_hdmi_setmode(encoder, adjusted_mode); - } + encoder_mode = atombios_get_encoder_mode(encoder); + if (connector && (radeon_audio != 0) && + ((encoder_mode == ATOM_ENCODER_MODE_HDMI) || + ENCODER_MODE_IS_DP(encoder_mode))) + radeon_audio_mode_set(encoder, adjusted_mode); } static bool @@ -2339,7 +2498,9 @@ static void radeon_atom_encoder_prepare(struct drm_encoder *encoder) ENCODER_OBJECT_ID_NONE)) { struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; if (dig) { - dig->dig_encoder = radeon_atom_pick_dig_encoder(encoder); + if (dig->dig_encoder >= 0) + radeon_atom_release_dig_encoder(rdev, dig->dig_encoder); + dig->dig_encoder = radeon_atom_pick_dig_encoder(encoder, -1); if (radeon_encoder->active_device & ATOM_DEVICE_DFP_SUPPORT) { if (rdev->family >= CHIP_R600) dig->afmt = rdev->mode_info.afmt[dig->dig_encoder]; @@ -2367,6 +2528,15 @@ static void radeon_atom_encoder_prepare(struct drm_encoder *encoder) /* this is needed for the pll/ss setup to work correctly in some cases */ atombios_set_encoder_crtc_source(encoder); + /* set up the FMT blocks */ + if (ASIC_IS_DCE8(rdev)) + dce8_program_fmt(encoder); + else if (ASIC_IS_DCE4(rdev)) + dce4_program_fmt(encoder); + else if (ASIC_IS_DCE3(rdev)) + dce3_program_fmt(encoder); + else if (ASIC_IS_AVIVO(rdev)) + avivo_program_fmt(encoder); } static void radeon_atom_encoder_commit(struct drm_encoder *encoder) @@ -2411,6 +2581,7 @@ static void radeon_atom_encoder_disable(struct drm_encoder *encoder) case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: /* handled in dpms */ break; @@ -2431,12 +2602,18 @@ static void radeon_atom_encoder_disable(struct drm_encoder *encoder) disable_done: if (radeon_encoder_is_digital(encoder)) { - if (atombios_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_HDMI) - r600_hdmi_disable(encoder); - dig = radeon_encoder->enc_priv; - dig->dig_encoder = -1; - } - radeon_encoder->active_device = 0; + if (atombios_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_HDMI) { + if (rdev->asic->display.hdmi_enable) + radeon_hdmi_enable(rdev, encoder, false); + } + if (atombios_get_encoder_mode(encoder) != ATOM_ENCODER_MODE_DP_MST) { + dig = radeon_encoder->enc_priv; + radeon_atom_release_dig_encoder(rdev, dig->dig_encoder); + dig->dig_encoder = -1; + radeon_encoder->active_device = 0; + } + } else + radeon_encoder->active_device = 0; } /* these are handled by the primary encoders */ @@ -2640,6 +2817,7 @@ radeon_add_atom_encoder(struct drm_device *dev, case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { radeon_encoder->rmx_type = RMX_FULL; drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_LVDS); diff --git a/sys/dev/pci/drm/radeon/atombios_i2c.c b/sys/dev/pci/drm/radeon/atombios_i2c.c index 7440be07ad1..b817c037af6 100644 --- a/sys/dev/pci/drm/radeon/atombios_i2c.c +++ b/sys/dev/pci/drm/radeon/atombios_i2c.c @@ -1,4 +1,3 @@ -/* $OpenBSD: atombios_i2c.c,v 1.9 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2011 Advanced Micro Devices, Inc. * @@ -28,12 +27,10 @@ #include "radeon.h" #include "atom.h" -extern void radeon_atom_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le); - #define TARGET_HW_I2C_CLOCK 50 /* these are a limitation of ProcessI2cChannelTransaction not the hw */ -#define ATOM_MAX_HW_I2C_WRITE 2 +#define ATOM_MAX_HW_I2C_WRITE 3 #define ATOM_MAX_HW_I2C_READ 255 static int radeon_process_i2c_ch(struct radeon_i2c_chan *chan, @@ -46,15 +43,20 @@ static int radeon_process_i2c_ch(struct radeon_i2c_chan *chan, int index = GetIndexIntoMasterTable(COMMAND, ProcessI2cChannelTransaction); unsigned char *base; u16 out = cpu_to_le16(0); + int r = 0; memset(&args, 0, sizeof(args)); + mutex_lock(&chan->mutex); + mutex_lock(&rdev->mode_info.atom_context->scratch_mutex); + base = (unsigned char *)rdev->mode_info.atom_context->scratch; if (flags & HW_I2C_WRITE) { if (num > ATOM_MAX_HW_I2C_WRITE) { - DRM_ERROR("hw i2c: tried to write too many bytes (%d vs 2)\n", num); - return -EINVAL; + DRM_ERROR("hw i2c: tried to write too many bytes (%d vs 3)\n", num); + r = -EINVAL; + goto done; } if (buf == NULL) args.ucRegIndex = 0; @@ -65,33 +67,45 @@ static int radeon_process_i2c_ch(struct radeon_i2c_chan *chan, if (num) memcpy(&out, &buf[1], num); args.lpI2CDataOut = cpu_to_le16(out); -#if 0 } else { +#if 0 + /* + * gcc 4.2 gives 'warning: comparison is always false + * due to limited range of data type' + */ if (num > ATOM_MAX_HW_I2C_READ) { DRM_ERROR("hw i2c: tried to read too many bytes (%d vs 255)\n", num); - return -EINVAL; + r = -EINVAL; + goto done; } #endif + args.ucRegIndex = 0; + args.lpI2CDataOut = 0; } + args.ucFlag = flags; args.ucI2CSpeed = TARGET_HW_I2C_CLOCK; - args.ucRegIndex = 0; args.ucTransBytes = num; args.ucSlaveAddr = slave_addr << 1; args.ucLineNumber = chan->rec.i2c_id; - atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + atom_execute_table_scratch_unlocked(rdev->mode_info.atom_context, index, (uint32_t *)&args); /* error */ if (args.ucStatus != HW_ASSISTED_I2C_STATUS_SUCCESS) { DRM_DEBUG_KMS("hw_i2c error\n"); - return -EIO; + r = -EIO; + goto done; } if (!(flags & HW_I2C_WRITE)) radeon_atom_copy_swap(buf, base, num, false); - return 0; +done: + mutex_unlock(&rdev->mode_info.atom_context->scratch_mutex); + mutex_unlock(&chan->mutex); + + return r; } int radeon_atom_hw_i2c_xfer(struct i2c_adapter *i2c_adap, @@ -148,3 +162,4 @@ u32 radeon_atom_hw_i2c_func(struct i2c_adapter *adap) { return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; } + diff --git a/sys/dev/pci/drm/radeon/avivod.h b/sys/dev/pci/drm/radeon/avivod.h index 1a7526a1475..3c391e7e9fd 100644 --- a/sys/dev/pci/drm/radeon/avivod.h +++ b/sys/dev/pci/drm/radeon/avivod.h @@ -1,4 +1,3 @@ -/* $OpenBSD: avivod.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2009 Advanced Micro Devices, Inc. * Copyright 2009 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/btc_dpm.c b/sys/dev/pci/drm/radeon/btc_dpm.c new file mode 100644 index 00000000000..2ba7bce168e --- /dev/null +++ b/sys/dev/pci/drm/radeon/btc_dpm.c @@ -0,0 +1,2824 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "btcd.h" +#include "r600_dpm.h" +#include "cypress_dpm.h" +#include "btc_dpm.h" +#include "atom.h" + +#define MC_CG_ARB_FREQ_F0 0x0a +#define MC_CG_ARB_FREQ_F1 0x0b +#define MC_CG_ARB_FREQ_F2 0x0c +#define MC_CG_ARB_FREQ_F3 0x0d + +#define MC_CG_SEQ_DRAMCONF_S0 0x05 +#define MC_CG_SEQ_DRAMCONF_S1 0x06 +#define MC_CG_SEQ_YCLK_SUSPEND 0x04 +#define MC_CG_SEQ_YCLK_RESUME 0x0a + +#define SMC_RAM_END 0x8000 + +#ifndef BTC_MGCG_SEQUENCE +#define BTC_MGCG_SEQUENCE 300 + +struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps); +struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev); +struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev); + +extern int ni_mc_load_microcode(struct radeon_device *rdev); + +//********* BARTS **************// +static const u32 barts_cgcg_cgls_default[] = +{ + /* Register, Value, Mask bits */ + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff +}; +#define BARTS_CGCG_CGLS_DEFAULT_LENGTH sizeof(barts_cgcg_cgls_default) / (3 * sizeof(u32)) + +static const u32 barts_cgcg_cgls_disable[] = +{ + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x00000644, 0x000f7912, 0x001f4180, + 0x00000644, 0x000f3812, 0x001f4180 +}; +#define BARTS_CGCG_CGLS_DISABLE_LENGTH sizeof(barts_cgcg_cgls_disable) / (3 * sizeof(u32)) + +static const u32 barts_cgcg_cgls_enable[] = +{ + /* 0x0000c124, 0x84180000, 0x00180000, */ + 0x00000644, 0x000f7892, 0x001f4080, + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff +}; +#define BARTS_CGCG_CGLS_ENABLE_LENGTH sizeof(barts_cgcg_cgls_enable) / (3 * sizeof(u32)) + +static const u32 barts_mgcg_default[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x00005448, 0x00000100, 0xffffffff, + 0x000055e4, 0x00600100, 0xffffffff, + 0x0000160c, 0x00000100, 0xffffffff, + 0x0000c164, 0x00000100, 0xffffffff, + 0x00008a18, 0x00000100, 0xffffffff, + 0x0000897c, 0x06000100, 0xffffffff, + 0x00008b28, 0x00000100, 0xffffffff, + 0x00009144, 0x00000100, 0xffffffff, + 0x00009a60, 0x00000100, 0xffffffff, + 0x00009868, 0x00000100, 0xffffffff, + 0x00008d58, 0x00000100, 0xffffffff, + 0x00009510, 0x00000100, 0xffffffff, + 0x0000949c, 0x00000100, 0xffffffff, + 0x00009654, 0x00000100, 0xffffffff, + 0x00009030, 0x00000100, 0xffffffff, + 0x00009034, 0x00000100, 0xffffffff, + 0x00009038, 0x00000100, 0xffffffff, + 0x0000903c, 0x00000100, 0xffffffff, + 0x00009040, 0x00000100, 0xffffffff, + 0x0000a200, 0x00000100, 0xffffffff, + 0x0000a204, 0x00000100, 0xffffffff, + 0x0000a208, 0x00000100, 0xffffffff, + 0x0000a20c, 0x00000100, 0xffffffff, + 0x0000977c, 0x00000100, 0xffffffff, + 0x00003f80, 0x00000100, 0xffffffff, + 0x0000a210, 0x00000100, 0xffffffff, + 0x0000a214, 0x00000100, 0xffffffff, + 0x000004d8, 0x00000100, 0xffffffff, + 0x00009784, 0x00000100, 0xffffffff, + 0x00009698, 0x00000100, 0xffffffff, + 0x000004d4, 0x00000200, 0xffffffff, + 0x000004d0, 0x00000000, 0xffffffff, + 0x000030cc, 0x00000100, 0xffffffff, + 0x0000d0c0, 0xff000100, 0xffffffff, + 0x0000802c, 0x40000000, 0xffffffff, + 0x0000915c, 0x00010000, 0xffffffff, + 0x00009160, 0x00030002, 0xffffffff, + 0x00009164, 0x00050004, 0xffffffff, + 0x00009168, 0x00070006, 0xffffffff, + 0x00009178, 0x00070000, 0xffffffff, + 0x0000917c, 0x00030002, 0xffffffff, + 0x00009180, 0x00050004, 0xffffffff, + 0x0000918c, 0x00010006, 0xffffffff, + 0x00009190, 0x00090008, 0xffffffff, + 0x00009194, 0x00070000, 0xffffffff, + 0x00009198, 0x00030002, 0xffffffff, + 0x0000919c, 0x00050004, 0xffffffff, + 0x000091a8, 0x00010006, 0xffffffff, + 0x000091ac, 0x00090008, 0xffffffff, + 0x000091b0, 0x00070000, 0xffffffff, + 0x000091b4, 0x00030002, 0xffffffff, + 0x000091b8, 0x00050004, 0xffffffff, + 0x000091c4, 0x00010006, 0xffffffff, + 0x000091c8, 0x00090008, 0xffffffff, + 0x000091cc, 0x00070000, 0xffffffff, + 0x000091d0, 0x00030002, 0xffffffff, + 0x000091d4, 0x00050004, 0xffffffff, + 0x000091e0, 0x00010006, 0xffffffff, + 0x000091e4, 0x00090008, 0xffffffff, + 0x000091e8, 0x00000000, 0xffffffff, + 0x000091ec, 0x00070000, 0xffffffff, + 0x000091f0, 0x00030002, 0xffffffff, + 0x000091f4, 0x00050004, 0xffffffff, + 0x00009200, 0x00010006, 0xffffffff, + 0x00009204, 0x00090008, 0xffffffff, + 0x00009208, 0x00070000, 0xffffffff, + 0x0000920c, 0x00030002, 0xffffffff, + 0x00009210, 0x00050004, 0xffffffff, + 0x0000921c, 0x00010006, 0xffffffff, + 0x00009220, 0x00090008, 0xffffffff, + 0x00009224, 0x00070000, 0xffffffff, + 0x00009228, 0x00030002, 0xffffffff, + 0x0000922c, 0x00050004, 0xffffffff, + 0x00009238, 0x00010006, 0xffffffff, + 0x0000923c, 0x00090008, 0xffffffff, + 0x00009294, 0x00000000, 0xffffffff, + 0x0000802c, 0x40010000, 0xffffffff, + 0x0000915c, 0x00010000, 0xffffffff, + 0x00009160, 0x00030002, 0xffffffff, + 0x00009164, 0x00050004, 0xffffffff, + 0x00009168, 0x00070006, 0xffffffff, + 0x00009178, 0x00070000, 0xffffffff, + 0x0000917c, 0x00030002, 0xffffffff, + 0x00009180, 0x00050004, 0xffffffff, + 0x0000918c, 0x00010006, 0xffffffff, + 0x00009190, 0x00090008, 0xffffffff, + 0x00009194, 0x00070000, 0xffffffff, + 0x00009198, 0x00030002, 0xffffffff, + 0x0000919c, 0x00050004, 0xffffffff, + 0x000091a8, 0x00010006, 0xffffffff, + 0x000091ac, 0x00090008, 0xffffffff, + 0x000091b0, 0x00070000, 0xffffffff, + 0x000091b4, 0x00030002, 0xffffffff, + 0x000091b8, 0x00050004, 0xffffffff, + 0x000091c4, 0x00010006, 0xffffffff, + 0x000091c8, 0x00090008, 0xffffffff, + 0x000091cc, 0x00070000, 0xffffffff, + 0x000091d0, 0x00030002, 0xffffffff, + 0x000091d4, 0x00050004, 0xffffffff, + 0x000091e0, 0x00010006, 0xffffffff, + 0x000091e4, 0x00090008, 0xffffffff, + 0x000091e8, 0x00000000, 0xffffffff, + 0x000091ec, 0x00070000, 0xffffffff, + 0x000091f0, 0x00030002, 0xffffffff, + 0x000091f4, 0x00050004, 0xffffffff, + 0x00009200, 0x00010006, 0xffffffff, + 0x00009204, 0x00090008, 0xffffffff, + 0x00009208, 0x00070000, 0xffffffff, + 0x0000920c, 0x00030002, 0xffffffff, + 0x00009210, 0x00050004, 0xffffffff, + 0x0000921c, 0x00010006, 0xffffffff, + 0x00009220, 0x00090008, 0xffffffff, + 0x00009224, 0x00070000, 0xffffffff, + 0x00009228, 0x00030002, 0xffffffff, + 0x0000922c, 0x00050004, 0xffffffff, + 0x00009238, 0x00010006, 0xffffffff, + 0x0000923c, 0x00090008, 0xffffffff, + 0x00009294, 0x00000000, 0xffffffff, + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff +}; +#define BARTS_MGCG_DEFAULT_LENGTH sizeof(barts_mgcg_default) / (3 * sizeof(u32)) + +static const u32 barts_mgcg_disable[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000002, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000003, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x00009150, 0x00600000, 0xffffffff +}; +#define BARTS_MGCG_DISABLE_LENGTH sizeof(barts_mgcg_disable) / (3 * sizeof(u32)) + +static const u32 barts_mgcg_enable[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000002, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000003, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x00009150, 0x81944000, 0xffffffff +}; +#define BARTS_MGCG_ENABLE_LENGTH sizeof(barts_mgcg_enable) / (3 * sizeof(u32)) + +//********* CAICOS **************// +static const u32 caicos_cgcg_cgls_default[] = +{ + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff +}; +#define CAICOS_CGCG_CGLS_DEFAULT_LENGTH sizeof(caicos_cgcg_cgls_default) / (3 * sizeof(u32)) + +static const u32 caicos_cgcg_cgls_disable[] = +{ + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x00000644, 0x000f7912, 0x001f4180, + 0x00000644, 0x000f3812, 0x001f4180 +}; +#define CAICOS_CGCG_CGLS_DISABLE_LENGTH sizeof(caicos_cgcg_cgls_disable) / (3 * sizeof(u32)) + +static const u32 caicos_cgcg_cgls_enable[] = +{ + /* 0x0000c124, 0x84180000, 0x00180000, */ + 0x00000644, 0x000f7892, 0x001f4080, + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff +}; +#define CAICOS_CGCG_CGLS_ENABLE_LENGTH sizeof(caicos_cgcg_cgls_enable) / (3 * sizeof(u32)) + +static const u32 caicos_mgcg_default[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x00005448, 0x00000100, 0xffffffff, + 0x000055e4, 0x00600100, 0xffffffff, + 0x0000160c, 0x00000100, 0xffffffff, + 0x0000c164, 0x00000100, 0xffffffff, + 0x00008a18, 0x00000100, 0xffffffff, + 0x0000897c, 0x06000100, 0xffffffff, + 0x00008b28, 0x00000100, 0xffffffff, + 0x00009144, 0x00000100, 0xffffffff, + 0x00009a60, 0x00000100, 0xffffffff, + 0x00009868, 0x00000100, 0xffffffff, + 0x00008d58, 0x00000100, 0xffffffff, + 0x00009510, 0x00000100, 0xffffffff, + 0x0000949c, 0x00000100, 0xffffffff, + 0x00009654, 0x00000100, 0xffffffff, + 0x00009030, 0x00000100, 0xffffffff, + 0x00009034, 0x00000100, 0xffffffff, + 0x00009038, 0x00000100, 0xffffffff, + 0x0000903c, 0x00000100, 0xffffffff, + 0x00009040, 0x00000100, 0xffffffff, + 0x0000a200, 0x00000100, 0xffffffff, + 0x0000a204, 0x00000100, 0xffffffff, + 0x0000a208, 0x00000100, 0xffffffff, + 0x0000a20c, 0x00000100, 0xffffffff, + 0x0000977c, 0x00000100, 0xffffffff, + 0x00003f80, 0x00000100, 0xffffffff, + 0x0000a210, 0x00000100, 0xffffffff, + 0x0000a214, 0x00000100, 0xffffffff, + 0x000004d8, 0x00000100, 0xffffffff, + 0x00009784, 0x00000100, 0xffffffff, + 0x00009698, 0x00000100, 0xffffffff, + 0x000004d4, 0x00000200, 0xffffffff, + 0x000004d0, 0x00000000, 0xffffffff, + 0x000030cc, 0x00000100, 0xffffffff, + 0x0000d0c0, 0xff000100, 0xffffffff, + 0x0000915c, 0x00010000, 0xffffffff, + 0x00009160, 0x00030002, 0xffffffff, + 0x00009164, 0x00050004, 0xffffffff, + 0x00009168, 0x00070006, 0xffffffff, + 0x00009178, 0x00070000, 0xffffffff, + 0x0000917c, 0x00030002, 0xffffffff, + 0x00009180, 0x00050004, 0xffffffff, + 0x0000918c, 0x00010006, 0xffffffff, + 0x00009190, 0x00090008, 0xffffffff, + 0x00009194, 0x00070000, 0xffffffff, + 0x00009198, 0x00030002, 0xffffffff, + 0x0000919c, 0x00050004, 0xffffffff, + 0x000091a8, 0x00010006, 0xffffffff, + 0x000091ac, 0x00090008, 0xffffffff, + 0x000091e8, 0x00000000, 0xffffffff, + 0x00009294, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff +}; +#define CAICOS_MGCG_DEFAULT_LENGTH sizeof(caicos_mgcg_default) / (3 * sizeof(u32)) + +static const u32 caicos_mgcg_disable[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000002, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000003, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x00009150, 0x00600000, 0xffffffff +}; +#define CAICOS_MGCG_DISABLE_LENGTH sizeof(caicos_mgcg_disable) / (3 * sizeof(u32)) + +static const u32 caicos_mgcg_enable[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000002, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000003, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x00009150, 0x46944040, 0xffffffff +}; +#define CAICOS_MGCG_ENABLE_LENGTH sizeof(caicos_mgcg_enable) / (3 * sizeof(u32)) + +//********* TURKS **************// +static const u32 turks_cgcg_cgls_default[] = +{ + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff +}; +#define TURKS_CGCG_CGLS_DEFAULT_LENGTH sizeof(turks_cgcg_cgls_default) / (3 * sizeof(u32)) + +static const u32 turks_cgcg_cgls_disable[] = +{ + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x00000644, 0x000f7912, 0x001f4180, + 0x00000644, 0x000f3812, 0x001f4180 +}; +#define TURKS_CGCG_CGLS_DISABLE_LENGTH sizeof(turks_cgcg_cgls_disable) / (3 * sizeof(u32)) + +static const u32 turks_cgcg_cgls_enable[] = +{ + /* 0x0000c124, 0x84180000, 0x00180000, */ + 0x00000644, 0x000f7892, 0x001f4080, + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff +}; +#define TURKS_CGCG_CGLS_ENABLE_LENGTH sizeof(turks_cgcg_cgls_enable) / (3 * sizeof(u32)) + +// These are the sequences for turks_mgcg_shls +static const u32 turks_mgcg_default[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x00005448, 0x00000100, 0xffffffff, + 0x000055e4, 0x00600100, 0xffffffff, + 0x0000160c, 0x00000100, 0xffffffff, + 0x0000c164, 0x00000100, 0xffffffff, + 0x00008a18, 0x00000100, 0xffffffff, + 0x0000897c, 0x06000100, 0xffffffff, + 0x00008b28, 0x00000100, 0xffffffff, + 0x00009144, 0x00000100, 0xffffffff, + 0x00009a60, 0x00000100, 0xffffffff, + 0x00009868, 0x00000100, 0xffffffff, + 0x00008d58, 0x00000100, 0xffffffff, + 0x00009510, 0x00000100, 0xffffffff, + 0x0000949c, 0x00000100, 0xffffffff, + 0x00009654, 0x00000100, 0xffffffff, + 0x00009030, 0x00000100, 0xffffffff, + 0x00009034, 0x00000100, 0xffffffff, + 0x00009038, 0x00000100, 0xffffffff, + 0x0000903c, 0x00000100, 0xffffffff, + 0x00009040, 0x00000100, 0xffffffff, + 0x0000a200, 0x00000100, 0xffffffff, + 0x0000a204, 0x00000100, 0xffffffff, + 0x0000a208, 0x00000100, 0xffffffff, + 0x0000a20c, 0x00000100, 0xffffffff, + 0x0000977c, 0x00000100, 0xffffffff, + 0x00003f80, 0x00000100, 0xffffffff, + 0x0000a210, 0x00000100, 0xffffffff, + 0x0000a214, 0x00000100, 0xffffffff, + 0x000004d8, 0x00000100, 0xffffffff, + 0x00009784, 0x00000100, 0xffffffff, + 0x00009698, 0x00000100, 0xffffffff, + 0x000004d4, 0x00000200, 0xffffffff, + 0x000004d0, 0x00000000, 0xffffffff, + 0x000030cc, 0x00000100, 0xffffffff, + 0x0000d0c0, 0x00000100, 0xffffffff, + 0x0000915c, 0x00010000, 0xffffffff, + 0x00009160, 0x00030002, 0xffffffff, + 0x00009164, 0x00050004, 0xffffffff, + 0x00009168, 0x00070006, 0xffffffff, + 0x00009178, 0x00070000, 0xffffffff, + 0x0000917c, 0x00030002, 0xffffffff, + 0x00009180, 0x00050004, 0xffffffff, + 0x0000918c, 0x00010006, 0xffffffff, + 0x00009190, 0x00090008, 0xffffffff, + 0x00009194, 0x00070000, 0xffffffff, + 0x00009198, 0x00030002, 0xffffffff, + 0x0000919c, 0x00050004, 0xffffffff, + 0x000091a8, 0x00010006, 0xffffffff, + 0x000091ac, 0x00090008, 0xffffffff, + 0x000091b0, 0x00070000, 0xffffffff, + 0x000091b4, 0x00030002, 0xffffffff, + 0x000091b8, 0x00050004, 0xffffffff, + 0x000091c4, 0x00010006, 0xffffffff, + 0x000091c8, 0x00090008, 0xffffffff, + 0x000091cc, 0x00070000, 0xffffffff, + 0x000091d0, 0x00030002, 0xffffffff, + 0x000091d4, 0x00050004, 0xffffffff, + 0x000091e0, 0x00010006, 0xffffffff, + 0x000091e4, 0x00090008, 0xffffffff, + 0x000091e8, 0x00000000, 0xffffffff, + 0x000091ec, 0x00070000, 0xffffffff, + 0x000091f0, 0x00030002, 0xffffffff, + 0x000091f4, 0x00050004, 0xffffffff, + 0x00009200, 0x00010006, 0xffffffff, + 0x00009204, 0x00090008, 0xffffffff, + 0x00009208, 0x00070000, 0xffffffff, + 0x0000920c, 0x00030002, 0xffffffff, + 0x00009210, 0x00050004, 0xffffffff, + 0x0000921c, 0x00010006, 0xffffffff, + 0x00009220, 0x00090008, 0xffffffff, + 0x00009294, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff +}; +#define TURKS_MGCG_DEFAULT_LENGTH sizeof(turks_mgcg_default) / (3 * sizeof(u32)) + +static const u32 turks_mgcg_disable[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000002, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000003, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x00009150, 0x00600000, 0xffffffff +}; +#define TURKS_MGCG_DISABLE_LENGTH sizeof(turks_mgcg_disable) / (3 * sizeof(u32)) + +static const u32 turks_mgcg_enable[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000002, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000003, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x00009150, 0x6e944000, 0xffffffff +}; +#define TURKS_MGCG_ENABLE_LENGTH sizeof(turks_mgcg_enable) / (3 * sizeof(u32)) + +#endif + +#ifndef BTC_SYSLS_SEQUENCE +#define BTC_SYSLS_SEQUENCE 100 + + +//********* BARTS **************// +static const u32 barts_sysls_default[] = +{ + /* Register, Value, Mask bits */ + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x000020c0, 0x000c0c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x000064ec, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00006dfc, 0x00000000, 0xffffffff +}; +#define BARTS_SYSLS_DEFAULT_LENGTH sizeof(barts_sysls_default) / (3 * sizeof(u32)) + +static const u32 barts_sysls_disable[] = +{ + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x00041401, 0xffffffff, + 0x0000264c, 0x00040400, 0xffffffff, + 0x00002648, 0x00040400, 0xffffffff, + 0x00002650, 0x00040400, 0xffffffff, + 0x000020b8, 0x00040400, 0xffffffff, + 0x000020bc, 0x00040400, 0xffffffff, + 0x000020c0, 0x00040c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680000, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x000064ec, 0x00007ffd, 0xffffffff, + 0x00000c7c, 0x0000ff00, 0xffffffff, + 0x00006dfc, 0x0000007f, 0xffffffff +}; +#define BARTS_SYSLS_DISABLE_LENGTH sizeof(barts_sysls_disable) / (3 * sizeof(u32)) + +static const u32 barts_sysls_enable[] = +{ + 0x000055e8, 0x00000001, 0xffffffff, + 0x0000d0bc, 0x00000100, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x000020c0, 0x000c0c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x000004c8, 0x00000000, 0xffffffff, + 0x000064ec, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00006dfc, 0x00000000, 0xffffffff +}; +#define BARTS_SYSLS_ENABLE_LENGTH sizeof(barts_sysls_enable) / (3 * sizeof(u32)) + +//********* CAICOS **************// +static const u32 caicos_sysls_default[] = +{ + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x000064ec, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00006dfc, 0x00000000, 0xffffffff +}; +#define CAICOS_SYSLS_DEFAULT_LENGTH sizeof(caicos_sysls_default) / (3 * sizeof(u32)) + +static const u32 caicos_sysls_disable[] = +{ + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x00041401, 0xffffffff, + 0x0000264c, 0x00040400, 0xffffffff, + 0x00002648, 0x00040400, 0xffffffff, + 0x00002650, 0x00040400, 0xffffffff, + 0x000020b8, 0x00040400, 0xffffffff, + 0x000020bc, 0x00040400, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680000, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x000064ec, 0x00007ffd, 0xffffffff, + 0x00000c7c, 0x0000ff00, 0xffffffff, + 0x00006dfc, 0x0000007f, 0xffffffff +}; +#define CAICOS_SYSLS_DISABLE_LENGTH sizeof(caicos_sysls_disable) / (3 * sizeof(u32)) + +static const u32 caicos_sysls_enable[] = +{ + 0x000055e8, 0x00000001, 0xffffffff, + 0x0000d0bc, 0x00000100, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x000064ec, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00006dfc, 0x00000000, 0xffffffff, + 0x000004c8, 0x00000000, 0xffffffff +}; +#define CAICOS_SYSLS_ENABLE_LENGTH sizeof(caicos_sysls_enable) / (3 * sizeof(u32)) + +//********* TURKS **************// +static const u32 turks_sysls_default[] = +{ + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x000020c0, 0x000c0c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x000064ec, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00006dfc, 0x00000000, 0xffffffff +}; +#define TURKS_SYSLS_DEFAULT_LENGTH sizeof(turks_sysls_default) / (3 * sizeof(u32)) + +static const u32 turks_sysls_disable[] = +{ + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x00041401, 0xffffffff, + 0x0000264c, 0x00040400, 0xffffffff, + 0x00002648, 0x00040400, 0xffffffff, + 0x00002650, 0x00040400, 0xffffffff, + 0x000020b8, 0x00040400, 0xffffffff, + 0x000020bc, 0x00040400, 0xffffffff, + 0x000020c0, 0x00040c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680000, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x000064ec, 0x00007ffd, 0xffffffff, + 0x00000c7c, 0x0000ff00, 0xffffffff, + 0x00006dfc, 0x0000007f, 0xffffffff +}; +#define TURKS_SYSLS_DISABLE_LENGTH sizeof(turks_sysls_disable) / (3 * sizeof(u32)) + +static const u32 turks_sysls_enable[] = +{ + 0x000055e8, 0x00000001, 0xffffffff, + 0x0000d0bc, 0x00000100, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x000020c0, 0x000c0c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x000004c8, 0x00000000, 0xffffffff, + 0x000064ec, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00006dfc, 0x00000000, 0xffffffff +}; +#define TURKS_SYSLS_ENABLE_LENGTH sizeof(turks_sysls_enable) / (3 * sizeof(u32)) + +#endif + +u32 btc_valid_sclk[40] = +{ + 5000, 10000, 15000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, + 55000, 60000, 65000, 70000, 75000, 80000, 85000, 90000, 95000, 100000, + 105000, 110000, 11500, 120000, 125000, 130000, 135000, 140000, 145000, 150000, + 155000, 160000, 165000, 170000, 175000, 180000, 185000, 190000, 195000, 200000 +}; + +static const struct radeon_blacklist_clocks btc_blacklist_clocks[] = +{ + { 10000, 30000, RADEON_SCLK_UP }, + { 15000, 30000, RADEON_SCLK_UP }, + { 20000, 30000, RADEON_SCLK_UP }, + { 25000, 30000, RADEON_SCLK_UP } +}; + +void btc_get_max_clock_from_voltage_dependency_table(struct radeon_clock_voltage_dependency_table *table, + u32 *max_clock) +{ + u32 i, clock = 0; + + if ((table == NULL) || (table->count == 0)) { + *max_clock = clock; + return; + } + + for (i = 0; i < table->count; i++) { + if (clock < table->entries[i].clk) + clock = table->entries[i].clk; + } + *max_clock = clock; +} + +void btc_apply_voltage_dependency_rules(struct radeon_clock_voltage_dependency_table *table, + u32 clock, u16 max_voltage, u16 *voltage) +{ + u32 i; + + if ((table == NULL) || (table->count == 0)) + return; + + for (i= 0; i < table->count; i++) { + if (clock <= table->entries[i].clk) { + if (*voltage < table->entries[i].v) + *voltage = (u16)((table->entries[i].v < max_voltage) ? + table->entries[i].v : max_voltage); + return; + } + } + + *voltage = (*voltage > max_voltage) ? *voltage : max_voltage; +} + +static u32 btc_find_valid_clock(struct radeon_clock_array *clocks, + u32 max_clock, u32 requested_clock) +{ + unsigned int i; + + if ((clocks == NULL) || (clocks->count == 0)) + return (requested_clock < max_clock) ? requested_clock : max_clock; + + for (i = 0; i < clocks->count; i++) { + if (clocks->values[i] >= requested_clock) + return (clocks->values[i] < max_clock) ? clocks->values[i] : max_clock; + } + + return (clocks->values[clocks->count - 1] < max_clock) ? + clocks->values[clocks->count - 1] : max_clock; +} + +static u32 btc_get_valid_mclk(struct radeon_device *rdev, + u32 max_mclk, u32 requested_mclk) +{ + return btc_find_valid_clock(&rdev->pm.dpm.dyn_state.valid_mclk_values, + max_mclk, requested_mclk); +} + +static u32 btc_get_valid_sclk(struct radeon_device *rdev, + u32 max_sclk, u32 requested_sclk) +{ + return btc_find_valid_clock(&rdev->pm.dpm.dyn_state.valid_sclk_values, + max_sclk, requested_sclk); +} + +void btc_skip_blacklist_clocks(struct radeon_device *rdev, + const u32 max_sclk, const u32 max_mclk, + u32 *sclk, u32 *mclk) +{ + int i, num_blacklist_clocks; + + if ((sclk == NULL) || (mclk == NULL)) + return; + + num_blacklist_clocks = ARRAY_SIZE(btc_blacklist_clocks); + + for (i = 0; i < num_blacklist_clocks; i++) { + if ((btc_blacklist_clocks[i].sclk == *sclk) && + (btc_blacklist_clocks[i].mclk == *mclk)) + break; + } + + if (i < num_blacklist_clocks) { + if (btc_blacklist_clocks[i].action == RADEON_SCLK_UP) { + *sclk = btc_get_valid_sclk(rdev, max_sclk, *sclk + 1); + + if (*sclk < max_sclk) + btc_skip_blacklist_clocks(rdev, max_sclk, max_mclk, sclk, mclk); + } + } +} + +void btc_adjust_clock_combinations(struct radeon_device *rdev, + const struct radeon_clock_and_voltage_limits *max_limits, + struct rv7xx_pl *pl) +{ + + if ((pl->mclk == 0) || (pl->sclk == 0)) + return; + + if (pl->mclk == pl->sclk) + return; + + if (pl->mclk > pl->sclk) { + if (((pl->mclk + (pl->sclk - 1)) / pl->sclk) > rdev->pm.dpm.dyn_state.mclk_sclk_ratio) + pl->sclk = btc_get_valid_sclk(rdev, + max_limits->sclk, + (pl->mclk + + (rdev->pm.dpm.dyn_state.mclk_sclk_ratio - 1)) / + rdev->pm.dpm.dyn_state.mclk_sclk_ratio); + } else { + if ((pl->sclk - pl->mclk) > rdev->pm.dpm.dyn_state.sclk_mclk_delta) + pl->mclk = btc_get_valid_mclk(rdev, + max_limits->mclk, + pl->sclk - + rdev->pm.dpm.dyn_state.sclk_mclk_delta); + } +} + +static u16 btc_find_voltage(struct atom_voltage_table *table, u16 voltage) +{ + unsigned int i; + + for (i = 0; i < table->count; i++) { + if (voltage <= table->entries[i].value) + return table->entries[i].value; + } + + return table->entries[table->count - 1].value; +} + +void btc_apply_voltage_delta_rules(struct radeon_device *rdev, + u16 max_vddc, u16 max_vddci, + u16 *vddc, u16 *vddci) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u16 new_voltage; + + if ((0 == *vddc) || (0 == *vddci)) + return; + + if (*vddc > *vddci) { + if ((*vddc - *vddci) > rdev->pm.dpm.dyn_state.vddc_vddci_delta) { + new_voltage = btc_find_voltage(&eg_pi->vddci_voltage_table, + (*vddc - rdev->pm.dpm.dyn_state.vddc_vddci_delta)); + *vddci = (new_voltage < max_vddci) ? new_voltage : max_vddci; + } + } else { + if ((*vddci - *vddc) > rdev->pm.dpm.dyn_state.vddc_vddci_delta) { + new_voltage = btc_find_voltage(&eg_pi->vddc_voltage_table, + (*vddci - rdev->pm.dpm.dyn_state.vddc_vddci_delta)); + *vddc = (new_voltage < max_vddc) ? new_voltage : max_vddc; + } + } +} + +static void btc_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 tmp, bif; + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + if (enable) { + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) && + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) { + if (!pi->boot_in_gen2) { + bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK; + bif |= CG_CLIENT_REQ(0xd); + WREG32(CG_BIF_REQ_AND_RSP, bif); + + tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK; + tmp |= LC_HW_VOLTAGE_IF_CONTROL(1); + tmp |= LC_GEN2_EN_STRAP; + + tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + udelay(10); + tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + } + } + } else { + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) || + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) { + if (!pi->boot_in_gen2) { + bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK; + bif |= CG_CLIENT_REQ(0xd); + WREG32(CG_BIF_REQ_AND_RSP, bif); + + tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK; + tmp &= ~LC_GEN2_EN_STRAP; + } + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + } + } +} + +static void btc_enable_dynamic_pcie_gen2(struct radeon_device *rdev, + bool enable) +{ + btc_enable_bif_dynamic_pcie_gen2(rdev, enable); + + if (enable) + WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE); + else + WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE); +} + +static int btc_disable_ulv(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (eg_pi->ulv.supported) { + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableULV) != PPSMC_Result_OK) + return -EINVAL; + } + return 0; +} + +static int btc_populate_ulv_state(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table) +{ + int ret = -EINVAL; + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct rv7xx_pl *ulv_pl = eg_pi->ulv.pl; + + if (ulv_pl->vddc) { + ret = cypress_convert_power_level_to_smc(rdev, + ulv_pl, + &table->ULVState.levels[0], + PPSMC_DISPLAY_WATERMARK_LOW); + if (ret == 0) { + table->ULVState.levels[0].arbValue = MC_CG_ARB_FREQ_F0; + table->ULVState.levels[0].ACIndex = 1; + + table->ULVState.levels[1] = table->ULVState.levels[0]; + table->ULVState.levels[2] = table->ULVState.levels[0]; + + table->ULVState.flags |= PPSMC_SWSTATE_FLAG_DC; + + WREG32(CG_ULV_CONTROL, BTC_CGULVCONTROL_DFLT); + WREG32(CG_ULV_PARAMETER, BTC_CGULVPARAMETER_DFLT); + } + } + + return ret; +} + +static int btc_populate_smc_acpi_state(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table) +{ + int ret = cypress_populate_smc_acpi_state(rdev, table); + + if (ret == 0) { + table->ACPIState.levels[0].ACIndex = 0; + table->ACPIState.levels[1].ACIndex = 0; + table->ACPIState.levels[2].ACIndex = 0; + } + + return ret; +} + +void btc_program_mgcg_hw_sequence(struct radeon_device *rdev, + const u32 *sequence, u32 count) +{ + u32 i, length = count * 3; + u32 tmp; + + for (i = 0; i < length; i+=3) { + tmp = RREG32(sequence[i]); + tmp &= ~sequence[i+2]; + tmp |= sequence[i+1] & sequence[i+2]; + WREG32(sequence[i], tmp); + } +} + +static void btc_cg_clock_gating_default(struct radeon_device *rdev) +{ + u32 count; + const u32 *p = NULL; + + if (rdev->family == CHIP_BARTS) { + p = (const u32 *)&barts_cgcg_cgls_default; + count = BARTS_CGCG_CGLS_DEFAULT_LENGTH; + } else if (rdev->family == CHIP_TURKS) { + p = (const u32 *)&turks_cgcg_cgls_default; + count = TURKS_CGCG_CGLS_DEFAULT_LENGTH; + } else if (rdev->family == CHIP_CAICOS) { + p = (const u32 *)&caicos_cgcg_cgls_default; + count = CAICOS_CGCG_CGLS_DEFAULT_LENGTH; + } else + return; + + btc_program_mgcg_hw_sequence(rdev, p, count); +} + +static void btc_cg_clock_gating_enable(struct radeon_device *rdev, + bool enable) +{ + u32 count; + const u32 *p = NULL; + + if (enable) { + if (rdev->family == CHIP_BARTS) { + p = (const u32 *)&barts_cgcg_cgls_enable; + count = BARTS_CGCG_CGLS_ENABLE_LENGTH; + } else if (rdev->family == CHIP_TURKS) { + p = (const u32 *)&turks_cgcg_cgls_enable; + count = TURKS_CGCG_CGLS_ENABLE_LENGTH; + } else if (rdev->family == CHIP_CAICOS) { + p = (const u32 *)&caicos_cgcg_cgls_enable; + count = CAICOS_CGCG_CGLS_ENABLE_LENGTH; + } else + return; + } else { + if (rdev->family == CHIP_BARTS) { + p = (const u32 *)&barts_cgcg_cgls_disable; + count = BARTS_CGCG_CGLS_DISABLE_LENGTH; + } else if (rdev->family == CHIP_TURKS) { + p = (const u32 *)&turks_cgcg_cgls_disable; + count = TURKS_CGCG_CGLS_DISABLE_LENGTH; + } else if (rdev->family == CHIP_CAICOS) { + p = (const u32 *)&caicos_cgcg_cgls_disable; + count = CAICOS_CGCG_CGLS_DISABLE_LENGTH; + } else + return; + } + + btc_program_mgcg_hw_sequence(rdev, p, count); +} + +static void btc_mg_clock_gating_default(struct radeon_device *rdev) +{ + u32 count; + const u32 *p = NULL; + + if (rdev->family == CHIP_BARTS) { + p = (const u32 *)&barts_mgcg_default; + count = BARTS_MGCG_DEFAULT_LENGTH; + } else if (rdev->family == CHIP_TURKS) { + p = (const u32 *)&turks_mgcg_default; + count = TURKS_MGCG_DEFAULT_LENGTH; + } else if (rdev->family == CHIP_CAICOS) { + p = (const u32 *)&caicos_mgcg_default; + count = CAICOS_MGCG_DEFAULT_LENGTH; + } else + return; + + btc_program_mgcg_hw_sequence(rdev, p, count); +} + +static void btc_mg_clock_gating_enable(struct radeon_device *rdev, + bool enable) +{ + u32 count; + const u32 *p = NULL; + + if (enable) { + if (rdev->family == CHIP_BARTS) { + p = (const u32 *)&barts_mgcg_enable; + count = BARTS_MGCG_ENABLE_LENGTH; + } else if (rdev->family == CHIP_TURKS) { + p = (const u32 *)&turks_mgcg_enable; + count = TURKS_MGCG_ENABLE_LENGTH; + } else if (rdev->family == CHIP_CAICOS) { + p = (const u32 *)&caicos_mgcg_enable; + count = CAICOS_MGCG_ENABLE_LENGTH; + } else + return; + } else { + if (rdev->family == CHIP_BARTS) { + p = (const u32 *)&barts_mgcg_disable[0]; + count = BARTS_MGCG_DISABLE_LENGTH; + } else if (rdev->family == CHIP_TURKS) { + p = (const u32 *)&turks_mgcg_disable[0]; + count = TURKS_MGCG_DISABLE_LENGTH; + } else if (rdev->family == CHIP_CAICOS) { + p = (const u32 *)&caicos_mgcg_disable[0]; + count = CAICOS_MGCG_DISABLE_LENGTH; + } else + return; + } + + btc_program_mgcg_hw_sequence(rdev, p, count); +} + +static void btc_ls_clock_gating_default(struct radeon_device *rdev) +{ + u32 count; + const u32 *p = NULL; + + if (rdev->family == CHIP_BARTS) { + p = (const u32 *)&barts_sysls_default; + count = BARTS_SYSLS_DEFAULT_LENGTH; + } else if (rdev->family == CHIP_TURKS) { + p = (const u32 *)&turks_sysls_default; + count = TURKS_SYSLS_DEFAULT_LENGTH; + } else if (rdev->family == CHIP_CAICOS) { + p = (const u32 *)&caicos_sysls_default; + count = CAICOS_SYSLS_DEFAULT_LENGTH; + } else + return; + + btc_program_mgcg_hw_sequence(rdev, p, count); +} + +static void btc_ls_clock_gating_enable(struct radeon_device *rdev, + bool enable) +{ + u32 count; + const u32 *p = NULL; + + if (enable) { + if (rdev->family == CHIP_BARTS) { + p = (const u32 *)&barts_sysls_enable; + count = BARTS_SYSLS_ENABLE_LENGTH; + } else if (rdev->family == CHIP_TURKS) { + p = (const u32 *)&turks_sysls_enable; + count = TURKS_SYSLS_ENABLE_LENGTH; + } else if (rdev->family == CHIP_CAICOS) { + p = (const u32 *)&caicos_sysls_enable; + count = CAICOS_SYSLS_ENABLE_LENGTH; + } else + return; + } else { + if (rdev->family == CHIP_BARTS) { + p = (const u32 *)&barts_sysls_disable; + count = BARTS_SYSLS_DISABLE_LENGTH; + } else if (rdev->family == CHIP_TURKS) { + p = (const u32 *)&turks_sysls_disable; + count = TURKS_SYSLS_DISABLE_LENGTH; + } else if (rdev->family == CHIP_CAICOS) { + p = (const u32 *)&caicos_sysls_disable; + count = CAICOS_SYSLS_DISABLE_LENGTH; + } else + return; + } + + btc_program_mgcg_hw_sequence(rdev, p, count); +} + +bool btc_dpm_enabled(struct radeon_device *rdev) +{ + if (rv770_is_smc_running(rdev)) + return true; + else + return false; +} + +static int btc_init_smc_table(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + RV770_SMC_STATETABLE *table = &pi->smc_statetable; + int ret; + + memset(table, 0, sizeof(RV770_SMC_STATETABLE)); + + cypress_populate_smc_voltage_tables(rdev, table); + + switch (rdev->pm.int_thermal_type) { + case THERMAL_TYPE_EVERGREEN: + case THERMAL_TYPE_EMC2103_WITH_INTERNAL: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL; + break; + case THERMAL_TYPE_NONE: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE; + break; + default: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL; + break; + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) + table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT) + table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) + table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC; + + if (pi->mem_gddr5) + table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5; + + ret = cypress_populate_smc_initial_state(rdev, radeon_boot_state, table); + if (ret) + return ret; + + if (eg_pi->sclk_deep_sleep) + WREG32_P(SCLK_PSKIP_CNTL, PSKIP_ON_ALLOW_STOP_HI(32), + ~PSKIP_ON_ALLOW_STOP_HI_MASK); + + ret = btc_populate_smc_acpi_state(rdev, table); + if (ret) + return ret; + + if (eg_pi->ulv.supported) { + ret = btc_populate_ulv_state(rdev, table); + if (ret) + eg_pi->ulv.supported = false; + } + + table->driverState = table->initialState; + + return rv770_copy_bytes_to_smc(rdev, + pi->state_table_start, + (u8 *)table, + sizeof(RV770_SMC_STATETABLE), + pi->sram_end); +} + +static void btc_set_at_for_uvd(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + int idx = 0; + + if (r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) + idx = 1; + + if ((idx == 1) && !eg_pi->smu_uvd_hs) { + pi->rlp = 10; + pi->rmp = 100; + pi->lhp = 100; + pi->lmp = 10; + } else { + pi->rlp = eg_pi->ats[idx].rlp; + pi->rmp = eg_pi->ats[idx].rmp; + pi->lhp = eg_pi->ats[idx].lhp; + pi->lmp = eg_pi->ats[idx].lmp; + } + +} + +void btc_notify_uvd_to_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) { + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_uvd_enabled, 1); + eg_pi->uvd_enabled = true; + } else { + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_uvd_enabled, 0); + eg_pi->uvd_enabled = false; + } +} + +int btc_reset_to_default(struct radeon_device *rdev) +{ + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_ResetToDefaults) != PPSMC_Result_OK) + return -EINVAL; + + return 0; +} + +static void btc_stop_smc(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + if (((RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK) >> LB_SYNC_RESET_SEL_SHIFT) != 1) + break; + udelay(1); + } + udelay(100); + + r7xx_stop_smc(rdev); +} + +void btc_read_arb_registers(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct evergreen_arb_registers *arb_registers = + &eg_pi->bootup_arb_registers; + + arb_registers->mc_arb_dram_timing = RREG32(MC_ARB_DRAM_TIMING); + arb_registers->mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2); + arb_registers->mc_arb_rfsh_rate = RREG32(MC_ARB_RFSH_RATE); + arb_registers->mc_arb_burst_time = RREG32(MC_ARB_BURST_TIME); +} + + +static void btc_set_arb0_registers(struct radeon_device *rdev, + struct evergreen_arb_registers *arb_registers) +{ + u32 val; + + WREG32(MC_ARB_DRAM_TIMING, arb_registers->mc_arb_dram_timing); + WREG32(MC_ARB_DRAM_TIMING2, arb_registers->mc_arb_dram_timing2); + + val = (arb_registers->mc_arb_rfsh_rate & POWERMODE0_MASK) >> + POWERMODE0_SHIFT; + WREG32_P(MC_ARB_RFSH_RATE, POWERMODE0(val), ~POWERMODE0_MASK); + + val = (arb_registers->mc_arb_burst_time & STATE0_MASK) >> + STATE0_SHIFT; + WREG32_P(MC_ARB_BURST_TIME, STATE0(val), ~STATE0_MASK); +} + +static void btc_set_boot_state_timing(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (eg_pi->ulv.supported) + btc_set_arb0_registers(rdev, &eg_pi->bootup_arb_registers); +} + +static bool btc_is_state_ulv_compatible(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct rv7xx_ps *state = rv770_get_ps(radeon_state); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct rv7xx_pl *ulv_pl = eg_pi->ulv.pl; + + if (state->low.mclk != ulv_pl->mclk) + return false; + + if (state->low.vddci != ulv_pl->vddci) + return false; + + /* XXX check minclocks, etc. */ + + return true; +} + + +static int btc_set_ulv_dram_timing(struct radeon_device *rdev) +{ + u32 val; + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct rv7xx_pl *ulv_pl = eg_pi->ulv.pl; + + radeon_atom_set_engine_dram_timings(rdev, + ulv_pl->sclk, + ulv_pl->mclk); + + val = rv770_calculate_memory_refresh_rate(rdev, ulv_pl->sclk); + WREG32_P(MC_ARB_RFSH_RATE, POWERMODE0(val), ~POWERMODE0_MASK); + + val = cypress_calculate_burst_time(rdev, ulv_pl->sclk, ulv_pl->mclk); + WREG32_P(MC_ARB_BURST_TIME, STATE0(val), ~STATE0_MASK); + + return 0; +} + +static int btc_enable_ulv(struct radeon_device *rdev) +{ + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableULV) != PPSMC_Result_OK) + return -EINVAL; + + return 0; +} + +static int btc_set_power_state_conditionally_enable_ulv(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + int ret = 0; + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (eg_pi->ulv.supported) { + if (btc_is_state_ulv_compatible(rdev, radeon_new_state)) { + // Set ARB[0] to reflect the DRAM timing needed for ULV. + ret = btc_set_ulv_dram_timing(rdev); + if (ret == 0) + ret = btc_enable_ulv(rdev); + } + } + + return ret; +} + +static bool btc_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg) +{ + bool result = true; + + switch (in_reg) { + case MC_SEQ_RAS_TIMING >> 2: + *out_reg = MC_SEQ_RAS_TIMING_LP >> 2; + break; + case MC_SEQ_CAS_TIMING >> 2: + *out_reg = MC_SEQ_CAS_TIMING_LP >> 2; + break; + case MC_SEQ_MISC_TIMING >> 2: + *out_reg = MC_SEQ_MISC_TIMING_LP >> 2; + break; + case MC_SEQ_MISC_TIMING2 >> 2: + *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2; + break; + case MC_SEQ_RD_CTL_D0 >> 2: + *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2; + break; + case MC_SEQ_RD_CTL_D1 >> 2: + *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2; + break; + case MC_SEQ_WR_CTL_D0 >> 2: + *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2; + break; + case MC_SEQ_WR_CTL_D1 >> 2: + *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2; + break; + case MC_PMG_CMD_EMRS >> 2: + *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2; + break; + case MC_PMG_CMD_MRS >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2; + break; + case MC_PMG_CMD_MRS1 >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2; + break; + default: + result = false; + break; + } + + return result; +} + +static void btc_set_valid_flag(struct evergreen_mc_reg_table *table) +{ + u8 i, j; + + for (i = 0; i < table->last; i++) { + for (j = 1; j < table->num_entries; j++) { + if (table->mc_reg_table_entry[j-1].mc_data[i] != + table->mc_reg_table_entry[j].mc_data[i]) { + table->valid_flag |= (1 << i); + break; + } + } + } +} + +static int btc_set_mc_special_registers(struct radeon_device *rdev, + struct evergreen_mc_reg_table *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u8 i, j, k; + u32 tmp; + + for (i = 0, j = table->last; i < table->last; i++) { + switch (table->mc_reg_address[i].s1) { + case MC_SEQ_MISC1 >> 2: + tmp = RREG32(MC_PMG_CMD_EMRS); + table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2; + for (k = 0; k < table->num_entries; k++) { + table->mc_reg_table_entry[k].mc_data[j] = + ((tmp & 0xffff0000)) | + ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16); + } + j++; + + if (j >= SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + + tmp = RREG32(MC_PMG_CMD_MRS); + table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2; + for (k = 0; k < table->num_entries; k++) { + table->mc_reg_table_entry[k].mc_data[j] = + (tmp & 0xffff0000) | + (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); + if (!pi->mem_gddr5) + table->mc_reg_table_entry[k].mc_data[j] |= 0x100; + } + j++; + + if (j >= SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + break; + case MC_SEQ_RESERVE_M >> 2: + tmp = RREG32(MC_PMG_CMD_MRS1); + table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2; + for (k = 0; k < table->num_entries; k++) { + table->mc_reg_table_entry[k].mc_data[j] = + (tmp & 0xffff0000) | + (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); + } + j++; + + if (j >= SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + break; + default: + break; + } + } + + table->last = j; + + return 0; +} + +static void btc_set_s0_mc_reg_index(struct evergreen_mc_reg_table *table) +{ + u32 i; + u16 address; + + for (i = 0; i < table->last; i++) { + table->mc_reg_address[i].s0 = + btc_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ? + address : table->mc_reg_address[i].s1; + } +} + +static int btc_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table, + struct evergreen_mc_reg_table *eg_table) +{ + u8 i, j; + + if (table->last > SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + + if (table->num_entries > MAX_AC_TIMING_ENTRIES) + return -EINVAL; + + for (i = 0; i < table->last; i++) + eg_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1; + eg_table->last = table->last; + + for (i = 0; i < table->num_entries; i++) { + eg_table->mc_reg_table_entry[i].mclk_max = + table->mc_reg_table_entry[i].mclk_max; + for(j = 0; j < table->last; j++) + eg_table->mc_reg_table_entry[i].mc_data[j] = + table->mc_reg_table_entry[i].mc_data[j]; + } + eg_table->num_entries = table->num_entries; + + return 0; +} + +static int btc_initialize_mc_reg_table(struct radeon_device *rdev) +{ + int ret; + struct atom_mc_reg_table *table; + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct evergreen_mc_reg_table *eg_table = &eg_pi->mc_reg_table; + u8 module_index = rv770_get_memory_module_index(rdev); + + table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + /* Program additional LP registers that are no longer programmed by VBIOS */ + WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING)); + WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING)); + WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING)); + WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2)); + WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0)); + WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1)); + WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0)); + WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1)); + WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS)); + WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS)); + WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1)); + + ret = radeon_atom_init_mc_reg_table(rdev, module_index, table); + + if (ret) + goto init_mc_done; + + ret = btc_copy_vbios_mc_reg_table(table, eg_table); + + if (ret) + goto init_mc_done; + + btc_set_s0_mc_reg_index(eg_table); + ret = btc_set_mc_special_registers(rdev, eg_table); + + if (ret) + goto init_mc_done; + + btc_set_valid_flag(eg_table); + +init_mc_done: + kfree(table); + + return ret; +} + +static void btc_init_stutter_mode(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 tmp; + + if (pi->mclk_stutter_mode_threshold) { + if (pi->mem_gddr5) { + tmp = RREG32(MC_PMG_AUTO_CFG); + if ((0x200 & tmp) == 0) { + tmp = (tmp & 0xfffffc0b) | 0x204; + WREG32(MC_PMG_AUTO_CFG, tmp); + } + } + } +} + +bool btc_dpm_vblank_too_short(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 vblank_time = r600_dpm_get_vblank_time(rdev); + u32 switch_limit = pi->mem_gddr5 ? 450 : 100; + + if (vblank_time < switch_limit) + return true; + else + return false; + +} + +static void btc_apply_state_adjust_rules(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct rv7xx_ps *ps = rv770_get_ps(rps); + struct radeon_clock_and_voltage_limits *max_limits; + bool disable_mclk_switching; + u32 mclk, sclk; + u16 vddc, vddci; + + if ((rdev->pm.dpm.new_active_crtc_count > 1) || + btc_dpm_vblank_too_short(rdev)) + disable_mclk_switching = true; + else + disable_mclk_switching = false; + + if (rdev->pm.dpm.ac_power) + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + else + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc; + + if (rdev->pm.dpm.ac_power == false) { + if (ps->high.mclk > max_limits->mclk) + ps->high.mclk = max_limits->mclk; + if (ps->high.sclk > max_limits->sclk) + ps->high.sclk = max_limits->sclk; + if (ps->high.vddc > max_limits->vddc) + ps->high.vddc = max_limits->vddc; + if (ps->high.vddci > max_limits->vddci) + ps->high.vddci = max_limits->vddci; + + if (ps->medium.mclk > max_limits->mclk) + ps->medium.mclk = max_limits->mclk; + if (ps->medium.sclk > max_limits->sclk) + ps->medium.sclk = max_limits->sclk; + if (ps->medium.vddc > max_limits->vddc) + ps->medium.vddc = max_limits->vddc; + if (ps->medium.vddci > max_limits->vddci) + ps->medium.vddci = max_limits->vddci; + + if (ps->low.mclk > max_limits->mclk) + ps->low.mclk = max_limits->mclk; + if (ps->low.sclk > max_limits->sclk) + ps->low.sclk = max_limits->sclk; + if (ps->low.vddc > max_limits->vddc) + ps->low.vddc = max_limits->vddc; + if (ps->low.vddci > max_limits->vddci) + ps->low.vddci = max_limits->vddci; + } + + /* XXX validate the min clocks required for display */ + + if (disable_mclk_switching) { + sclk = ps->low.sclk; + mclk = ps->high.mclk; + vddc = ps->low.vddc; + vddci = ps->high.vddci; + } else { + sclk = ps->low.sclk; + mclk = ps->low.mclk; + vddc = ps->low.vddc; + vddci = ps->low.vddci; + } + + /* adjusted low state */ + ps->low.sclk = sclk; + ps->low.mclk = mclk; + ps->low.vddc = vddc; + ps->low.vddci = vddci; + + btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk, + &ps->low.sclk, &ps->low.mclk); + + /* adjusted medium, high states */ + if (ps->medium.sclk < ps->low.sclk) + ps->medium.sclk = ps->low.sclk; + if (ps->medium.vddc < ps->low.vddc) + ps->medium.vddc = ps->low.vddc; + if (ps->high.sclk < ps->medium.sclk) + ps->high.sclk = ps->medium.sclk; + if (ps->high.vddc < ps->medium.vddc) + ps->high.vddc = ps->medium.vddc; + + if (disable_mclk_switching) { + mclk = ps->low.mclk; + if (mclk < ps->medium.mclk) + mclk = ps->medium.mclk; + if (mclk < ps->high.mclk) + mclk = ps->high.mclk; + ps->low.mclk = mclk; + ps->low.vddci = vddci; + ps->medium.mclk = mclk; + ps->medium.vddci = vddci; + ps->high.mclk = mclk; + ps->high.vddci = vddci; + } else { + if (ps->medium.mclk < ps->low.mclk) + ps->medium.mclk = ps->low.mclk; + if (ps->medium.vddci < ps->low.vddci) + ps->medium.vddci = ps->low.vddci; + if (ps->high.mclk < ps->medium.mclk) + ps->high.mclk = ps->medium.mclk; + if (ps->high.vddci < ps->medium.vddci) + ps->high.vddci = ps->medium.vddci; + } + + btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk, + &ps->medium.sclk, &ps->medium.mclk); + btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk, + &ps->high.sclk, &ps->high.mclk); + + btc_adjust_clock_combinations(rdev, max_limits, &ps->low); + btc_adjust_clock_combinations(rdev, max_limits, &ps->medium); + btc_adjust_clock_combinations(rdev, max_limits, &ps->high); + + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk, + ps->low.sclk, max_limits->vddc, &ps->low.vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + ps->low.mclk, max_limits->vddci, &ps->low.vddci); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + ps->low.mclk, max_limits->vddc, &ps->low.vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk, + rdev->clock.current_dispclk, max_limits->vddc, &ps->low.vddc); + + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk, + ps->medium.sclk, max_limits->vddc, &ps->medium.vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + ps->medium.mclk, max_limits->vddci, &ps->medium.vddci); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + ps->medium.mclk, max_limits->vddc, &ps->medium.vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk, + rdev->clock.current_dispclk, max_limits->vddc, &ps->medium.vddc); + + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk, + ps->high.sclk, max_limits->vddc, &ps->high.vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + ps->high.mclk, max_limits->vddci, &ps->high.vddci); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + ps->high.mclk, max_limits->vddc, &ps->high.vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk, + rdev->clock.current_dispclk, max_limits->vddc, &ps->high.vddc); + + btc_apply_voltage_delta_rules(rdev, max_limits->vddc, max_limits->vddci, + &ps->low.vddc, &ps->low.vddci); + btc_apply_voltage_delta_rules(rdev, max_limits->vddc, max_limits->vddci, + &ps->medium.vddc, &ps->medium.vddci); + btc_apply_voltage_delta_rules(rdev, max_limits->vddc, max_limits->vddci, + &ps->high.vddc, &ps->high.vddci); + + if ((ps->high.vddc <= rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc) && + (ps->medium.vddc <= rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc) && + (ps->low.vddc <= rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)) + ps->dc_compatible = true; + else + ps->dc_compatible = false; + + if (ps->low.vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2) + ps->low.flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2; + if (ps->medium.vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2) + ps->medium.flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2; + if (ps->high.vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2) + ps->high.flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2; +} + +static void btc_update_current_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct rv7xx_ps *new_ps = rv770_get_ps(rps); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + eg_pi->current_rps = *rps; + eg_pi->current_ps = *new_ps; + eg_pi->current_rps.ps_priv = &eg_pi->current_ps; +} + +static void btc_update_requested_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct rv7xx_ps *new_ps = rv770_get_ps(rps); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + eg_pi->requested_rps = *rps; + eg_pi->requested_ps = *new_ps; + eg_pi->requested_rps.ps_priv = &eg_pi->requested_ps; +} + +#if 0 +void btc_dpm_reset_asic(struct radeon_device *rdev) +{ + rv770_restrict_performance_levels_before_switch(rdev); + btc_disable_ulv(rdev); + btc_set_boot_state_timing(rdev); + rv770_set_boot_state(rdev); +} +#endif + +int btc_dpm_pre_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps; + struct radeon_ps *new_ps = &requested_ps; + + btc_update_requested_ps(rdev, new_ps); + + btc_apply_state_adjust_rules(rdev, &eg_pi->requested_rps); + + return 0; +} + +int btc_dpm_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *new_ps = &eg_pi->requested_rps; + struct radeon_ps *old_ps = &eg_pi->current_rps; + int ret; + + ret = btc_disable_ulv(rdev); + btc_set_boot_state_timing(rdev); + ret = rv770_restrict_performance_levels_before_switch(rdev); + if (ret) { + DRM_ERROR("rv770_restrict_performance_levels_before_switch failed\n"); + return ret; + } + if (eg_pi->pcie_performance_request) + cypress_notify_link_speed_change_before_state_change(rdev, new_ps, old_ps); + + rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps); + ret = rv770_halt_smc(rdev); + if (ret) { + DRM_ERROR("rv770_halt_smc failed\n"); + return ret; + } + btc_set_at_for_uvd(rdev, new_ps); + if (eg_pi->smu_uvd_hs) + btc_notify_uvd_to_smc(rdev, new_ps); + ret = cypress_upload_sw_state(rdev, new_ps); + if (ret) { + DRM_ERROR("cypress_upload_sw_state failed\n"); + return ret; + } + if (eg_pi->dynamic_ac_timing) { + ret = cypress_upload_mc_reg_table(rdev, new_ps); + if (ret) { + DRM_ERROR("cypress_upload_mc_reg_table failed\n"); + return ret; + } + } + + cypress_program_memory_timing_parameters(rdev, new_ps); + + ret = rv770_resume_smc(rdev); + if (ret) { + DRM_ERROR("rv770_resume_smc failed\n"); + return ret; + } + ret = rv770_set_sw_state(rdev); + if (ret) { + DRM_ERROR("rv770_set_sw_state failed\n"); + return ret; + } + rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps); + + if (eg_pi->pcie_performance_request) + cypress_notify_link_speed_change_after_state_change(rdev, new_ps, old_ps); + + ret = btc_set_power_state_conditionally_enable_ulv(rdev, new_ps); + if (ret) { + DRM_ERROR("btc_set_power_state_conditionally_enable_ulv failed\n"); + return ret; + } + + return 0; +} + +void btc_dpm_post_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *new_ps = &eg_pi->requested_rps; + + btc_update_current_ps(rdev, new_ps); +} + +int btc_dpm_enable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + int ret; + + if (pi->gfx_clock_gating) + btc_cg_clock_gating_default(rdev); + + if (btc_dpm_enabled(rdev)) + return -EINVAL; + + if (pi->mg_clock_gating) + btc_mg_clock_gating_default(rdev); + + if (eg_pi->ls_clock_gating) + btc_ls_clock_gating_default(rdev); + + if (pi->voltage_control) { + rv770_enable_voltage_control(rdev, true); + ret = cypress_construct_voltage_tables(rdev); + if (ret) { + DRM_ERROR("cypress_construct_voltage_tables failed\n"); + return ret; + } + } + + if (pi->mvdd_control) { + ret = cypress_get_mvdd_configuration(rdev); + if (ret) { + DRM_ERROR("cypress_get_mvdd_configuration failed\n"); + return ret; + } + } + + if (eg_pi->dynamic_ac_timing) { + ret = btc_initialize_mc_reg_table(rdev); + if (ret) + eg_pi->dynamic_ac_timing = false; + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS) + rv770_enable_backbias(rdev, true); + + if (pi->dynamic_ss) + cypress_enable_spread_spectrum(rdev, true); + + if (pi->thermal_protection) + rv770_enable_thermal_protection(rdev, true); + + rv770_setup_bsp(rdev); + rv770_program_git(rdev); + rv770_program_tp(rdev); + rv770_program_tpp(rdev); + rv770_program_sstp(rdev); + rv770_program_engine_speed_parameters(rdev); + cypress_enable_display_gap(rdev); + rv770_program_vc(rdev); + + if (pi->dynamic_pcie_gen2) + btc_enable_dynamic_pcie_gen2(rdev, true); + + ret = rv770_upload_firmware(rdev); + if (ret) { + DRM_ERROR("rv770_upload_firmware failed\n"); + return ret; + } + ret = cypress_get_table_locations(rdev); + if (ret) { + DRM_ERROR("cypress_get_table_locations failed\n"); + return ret; + } + ret = btc_init_smc_table(rdev, boot_ps); + if (ret) + return ret; + + if (eg_pi->dynamic_ac_timing) { + ret = cypress_populate_mc_reg_table(rdev, boot_ps); + if (ret) { + DRM_ERROR("cypress_populate_mc_reg_table failed\n"); + return ret; + } + } + + cypress_program_response_times(rdev); + r7xx_start_smc(rdev); + ret = cypress_notify_smc_display_change(rdev, false); + if (ret) { + DRM_ERROR("cypress_notify_smc_display_change failed\n"); + return ret; + } + cypress_enable_sclk_control(rdev, true); + + if (eg_pi->memory_transition) + cypress_enable_mclk_control(rdev, true); + + cypress_start_dpm(rdev); + + if (pi->gfx_clock_gating) + btc_cg_clock_gating_enable(rdev, true); + + if (pi->mg_clock_gating) + btc_mg_clock_gating_enable(rdev, true); + + if (eg_pi->ls_clock_gating) + btc_ls_clock_gating_enable(rdev, true); + + rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + + btc_init_stutter_mode(rdev); + + btc_update_current_ps(rdev, rdev->pm.dpm.boot_ps); + + return 0; +}; + +void btc_dpm_disable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (!btc_dpm_enabled(rdev)) + return; + + rv770_clear_vc(rdev); + + if (pi->thermal_protection) + rv770_enable_thermal_protection(rdev, false); + + if (pi->dynamic_pcie_gen2) + btc_enable_dynamic_pcie_gen2(rdev, false); + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + rdev->irq.dpm_thermal = false; + radeon_irq_set(rdev); + } + + if (pi->gfx_clock_gating) + btc_cg_clock_gating_enable(rdev, false); + + if (pi->mg_clock_gating) + btc_mg_clock_gating_enable(rdev, false); + + if (eg_pi->ls_clock_gating) + btc_ls_clock_gating_enable(rdev, false); + + rv770_stop_dpm(rdev); + btc_reset_to_default(rdev); + btc_stop_smc(rdev); + cypress_enable_spread_spectrum(rdev, false); + + btc_update_current_ps(rdev, rdev->pm.dpm.boot_ps); +} + +void btc_dpm_setup_asic(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + int r; + + r = ni_mc_load_microcode(rdev); + if (r) + DRM_ERROR("Failed to load MC firmware!\n"); + rv770_get_memory_type(rdev); + rv740_read_clock_registers(rdev); + btc_read_arb_registers(rdev); + rv770_read_voltage_smio_registers(rdev); + + if (eg_pi->pcie_performance_request) + cypress_advertise_gen2_capability(rdev); + + rv770_get_pcie_gen2_status(rdev); + rv770_enable_acpi_pm(rdev); +} + +int btc_dpm_init(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi; + struct evergreen_power_info *eg_pi; + struct atom_clock_dividers dividers; + int ret; + + eg_pi = kzalloc(sizeof(struct evergreen_power_info), GFP_KERNEL); + if (eg_pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = eg_pi; + pi = &eg_pi->rv7xx; + + rv770_get_max_vddc(rdev); + + eg_pi->ulv.supported = false; + pi->acpi_vddc = 0; + eg_pi->acpi_vddci = 0; + pi->min_vddc_in_table = 0; + pi->max_vddc_in_table = 0; + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = rv7xx_parse_power_table(rdev); + if (ret) + return ret; + ret = r600_parse_extended_power_table(rdev); + if (ret) + return ret; + + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries = + kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 800; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 800; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 800; + + if (rdev->pm.dpm.voltage_response_time == 0) + rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT; + if (rdev->pm.dpm.backbias_response_time == 0) + rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + 0, false, ÷rs); + if (ret) + pi->ref_div = dividers.ref_div + 1; + else + pi->ref_div = R600_REFERENCEDIVIDER_DFLT; + + pi->mclk_strobe_mode_threshold = 40000; + pi->mclk_edc_enable_threshold = 40000; + eg_pi->mclk_edc_wr_enable_threshold = 40000; + + pi->rlp = RV770_RLP_DFLT; + pi->rmp = RV770_RMP_DFLT; + pi->lhp = RV770_LHP_DFLT; + pi->lmp = RV770_LMP_DFLT; + + eg_pi->ats[0].rlp = RV770_RLP_DFLT; + eg_pi->ats[0].rmp = RV770_RMP_DFLT; + eg_pi->ats[0].lhp = RV770_LHP_DFLT; + eg_pi->ats[0].lmp = RV770_LMP_DFLT; + + eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT; + eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT; + eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT; + eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT; + + eg_pi->smu_uvd_hs = true; + + pi->voltage_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0); + + pi->mvdd_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0); + + eg_pi->vddci_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0); + + rv770_get_engine_memory_ss(rdev); + + pi->asi = RV770_ASI_DFLT; + pi->pasi = CYPRESS_HASI_DFLT; + pi->vrc = CYPRESS_VRC_DFLT; + + pi->power_gating = false; + + pi->gfx_clock_gating = true; + + pi->mg_clock_gating = true; + pi->mgcgtssm = true; + eg_pi->ls_clock_gating = false; + eg_pi->sclk_deep_sleep = false; + + pi->dynamic_pcie_gen2 = true; + + if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE) + pi->thermal_protection = true; + else + pi->thermal_protection = false; + + pi->display_gap = true; + + if (rdev->flags & RADEON_IS_MOBILITY) + pi->dcodt = true; + else + pi->dcodt = false; + + pi->ulps = true; + + eg_pi->dynamic_ac_timing = true; + eg_pi->abm = true; + eg_pi->mcls = true; + eg_pi->light_sleep = true; + eg_pi->memory_transition = true; +#if defined(CONFIG_ACPI) + eg_pi->pcie_performance_request = + radeon_acpi_is_pcie_performance_request_supported(rdev); +#else + eg_pi->pcie_performance_request = false; +#endif + + if (rdev->family == CHIP_BARTS) + eg_pi->dll_default_on = true; + else + eg_pi->dll_default_on = false; + + eg_pi->sclk_deep_sleep = false; + if (ASIC_IS_LOMBOK(rdev)) + pi->mclk_stutter_mode_threshold = 30000; + else + pi->mclk_stutter_mode_threshold = 0; + + pi->sram_end = SMC_RAM_END; + + rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 4; + rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200; + rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900; + rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk); + rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk; + rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0; + rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL; + + if (rdev->family == CHIP_TURKS) + rdev->pm.dpm.dyn_state.sclk_mclk_delta = 15000; + else + rdev->pm.dpm.dyn_state.sclk_mclk_delta = 10000; + + /* make sure dc limits are valid */ + if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) || + (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0)) + rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc = + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + + return 0; +} + +void btc_dpm_fini(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); + kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries); + r600_free_extended_power_table(rdev); +} + +void btc_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *rps = &eg_pi->current_rps; + struct rv7xx_ps *ps = rv770_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >> + CURRENT_PROFILE_INDEX_SHIFT; + + if (current_index > 2) { + seq_printf(m, "invalid dpm profile %d\n", current_index); + } else { + if (current_index == 0) + pl = &ps->low; + else if (current_index == 1) + pl = &ps->medium; + else /* current_index == 2 */ + pl = &ps->high; + seq_printf(m, "uvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + seq_printf(m, "power level %d sclk: %u mclk: %u vddc: %u vddci: %u\n", + current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci); + } +} + +u32 btc_dpm_get_current_sclk(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *rps = &eg_pi->current_rps; + struct rv7xx_ps *ps = rv770_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >> + CURRENT_PROFILE_INDEX_SHIFT; + + if (current_index > 2) { + return 0; + } else { + if (current_index == 0) + pl = &ps->low; + else if (current_index == 1) + pl = &ps->medium; + else /* current_index == 2 */ + pl = &ps->high; + return pl->sclk; + } +} + +u32 btc_dpm_get_current_mclk(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *rps = &eg_pi->current_rps; + struct rv7xx_ps *ps = rv770_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >> + CURRENT_PROFILE_INDEX_SHIFT; + + if (current_index > 2) { + return 0; + } else { + if (current_index == 0) + pl = &ps->low; + else if (current_index == 1) + pl = &ps->medium; + else /* current_index == 2 */ + pl = &ps->high; + return pl->mclk; + } +} + +u32 btc_dpm_get_sclk(struct radeon_device *rdev, bool low) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct rv7xx_ps *requested_state = rv770_get_ps(&eg_pi->requested_rps); + + if (low) + return requested_state->low.sclk; + else + return requested_state->high.sclk; +} + +u32 btc_dpm_get_mclk(struct radeon_device *rdev, bool low) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct rv7xx_ps *requested_state = rv770_get_ps(&eg_pi->requested_rps); + + if (low) + return requested_state->low.mclk; + else + return requested_state->high.mclk; +} diff --git a/sys/dev/pci/drm/radeon/btc_dpm.h b/sys/dev/pci/drm/radeon/btc_dpm.h new file mode 100644 index 00000000000..3b6f12b7760 --- /dev/null +++ b/sys/dev/pci/drm/radeon/btc_dpm.h @@ -0,0 +1,59 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __BTC_DPM_H__ +#define __BTC_DPM_H__ + +#define BTC_RLP_UVD_DFLT 20 +#define BTC_RMP_UVD_DFLT 50 +#define BTC_LHP_UVD_DFLT 50 +#define BTC_LMP_UVD_DFLT 20 +#define BARTS_MGCGCGTSSMCTRL_DFLT 0x81944000 +#define TURKS_MGCGCGTSSMCTRL_DFLT 0x6e944000 +#define CAICOS_MGCGCGTSSMCTRL_DFLT 0x46944040 +#define BTC_CGULVPARAMETER_DFLT 0x00040035 +#define BTC_CGULVCONTROL_DFLT 0x00001450 + +extern u32 btc_valid_sclk[40]; + +void btc_read_arb_registers(struct radeon_device *rdev); +void btc_program_mgcg_hw_sequence(struct radeon_device *rdev, + const u32 *sequence, u32 count); +void btc_skip_blacklist_clocks(struct radeon_device *rdev, + const u32 max_sclk, const u32 max_mclk, + u32 *sclk, u32 *mclk); +void btc_adjust_clock_combinations(struct radeon_device *rdev, + const struct radeon_clock_and_voltage_limits *max_limits, + struct rv7xx_pl *pl); +void btc_apply_voltage_dependency_rules(struct radeon_clock_voltage_dependency_table *table, + u32 clock, u16 max_voltage, u16 *voltage); +void btc_get_max_clock_from_voltage_dependency_table(struct radeon_clock_voltage_dependency_table *table, + u32 *max_clock); +void btc_apply_voltage_delta_rules(struct radeon_device *rdev, + u16 max_vddc, u16 max_vddci, + u16 *vddc, u16 *vddci); +bool btc_dpm_enabled(struct radeon_device *rdev); +int btc_reset_to_default(struct radeon_device *rdev); +void btc_notify_uvd_to_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state); + +#endif diff --git a/sys/dev/pci/drm/radeon/btcd.h b/sys/dev/pci/drm/radeon/btcd.h new file mode 100644 index 00000000000..9c65be2d55a --- /dev/null +++ b/sys/dev/pci/drm/radeon/btcd.h @@ -0,0 +1,185 @@ +/* + * Copyright 2010 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#ifndef _BTCD_H_ +#define _BTCD_H_ + +/* pm registers */ + +#define GENERAL_PWRMGT 0x63c +# define GLOBAL_PWRMGT_EN (1 << 0) +# define STATIC_PM_EN (1 << 1) +# define THERMAL_PROTECTION_DIS (1 << 2) +# define THERMAL_PROTECTION_TYPE (1 << 3) +# define ENABLE_GEN2PCIE (1 << 4) +# define ENABLE_GEN2XSP (1 << 5) +# define SW_SMIO_INDEX(x) ((x) << 6) +# define SW_SMIO_INDEX_MASK (3 << 6) +# define SW_SMIO_INDEX_SHIFT 6 +# define LOW_VOLT_D2_ACPI (1 << 8) +# define LOW_VOLT_D3_ACPI (1 << 9) +# define VOLT_PWRMGT_EN (1 << 10) +# define BACKBIAS_PAD_EN (1 << 18) +# define BACKBIAS_VALUE (1 << 19) +# define DYN_SPREAD_SPECTRUM_EN (1 << 23) +# define AC_DC_SW (1 << 24) + +#define TARGET_AND_CURRENT_PROFILE_INDEX 0x66c +# define CURRENT_PROFILE_INDEX_MASK (0xf << 4) +# define CURRENT_PROFILE_INDEX_SHIFT 4 + +#define CG_BIF_REQ_AND_RSP 0x7f4 +#define CG_CLIENT_REQ(x) ((x) << 0) +#define CG_CLIENT_REQ_MASK (0xff << 0) +#define CG_CLIENT_REQ_SHIFT 0 +#define CG_CLIENT_RESP(x) ((x) << 8) +#define CG_CLIENT_RESP_MASK (0xff << 8) +#define CG_CLIENT_RESP_SHIFT 8 +#define CLIENT_CG_REQ(x) ((x) << 16) +#define CLIENT_CG_REQ_MASK (0xff << 16) +#define CLIENT_CG_REQ_SHIFT 16 +#define CLIENT_CG_RESP(x) ((x) << 24) +#define CLIENT_CG_RESP_MASK (0xff << 24) +#define CLIENT_CG_RESP_SHIFT 24 + +#define SCLK_PSKIP_CNTL 0x8c0 +#define PSKIP_ON_ALLOW_STOP_HI(x) ((x) << 16) +#define PSKIP_ON_ALLOW_STOP_HI_MASK (0xff << 16) +#define PSKIP_ON_ALLOW_STOP_HI_SHIFT 16 + +#define CG_ULV_CONTROL 0x8c8 +#define CG_ULV_PARAMETER 0x8cc + +#define MC_ARB_DRAM_TIMING 0x2774 +#define MC_ARB_DRAM_TIMING2 0x2778 + +#define MC_ARB_RFSH_RATE 0x27b0 +#define POWERMODE0(x) ((x) << 0) +#define POWERMODE0_MASK (0xff << 0) +#define POWERMODE0_SHIFT 0 +#define POWERMODE1(x) ((x) << 8) +#define POWERMODE1_MASK (0xff << 8) +#define POWERMODE1_SHIFT 8 +#define POWERMODE2(x) ((x) << 16) +#define POWERMODE2_MASK (0xff << 16) +#define POWERMODE2_SHIFT 16 +#define POWERMODE3(x) ((x) << 24) +#define POWERMODE3_MASK (0xff << 24) +#define POWERMODE3_SHIFT 24 + +#define MC_ARB_BURST_TIME 0x2808 +#define STATE0(x) ((x) << 0) +#define STATE0_MASK (0x1f << 0) +#define STATE0_SHIFT 0 +#define STATE1(x) ((x) << 5) +#define STATE1_MASK (0x1f << 5) +#define STATE1_SHIFT 5 +#define STATE2(x) ((x) << 10) +#define STATE2_MASK (0x1f << 10) +#define STATE2_SHIFT 10 +#define STATE3(x) ((x) << 15) +#define STATE3_MASK (0x1f << 15) +#define STATE3_SHIFT 15 + +#define MC_SEQ_RAS_TIMING 0x28a0 +#define MC_SEQ_CAS_TIMING 0x28a4 +#define MC_SEQ_MISC_TIMING 0x28a8 +#define MC_SEQ_MISC_TIMING2 0x28ac + +#define MC_SEQ_RD_CTL_D0 0x28b4 +#define MC_SEQ_RD_CTL_D1 0x28b8 +#define MC_SEQ_WR_CTL_D0 0x28bc +#define MC_SEQ_WR_CTL_D1 0x28c0 + +#define MC_PMG_AUTO_CFG 0x28d4 + +#define MC_SEQ_STATUS_M 0x29f4 +# define PMG_PWRSTATE (1 << 16) + +#define MC_SEQ_MISC0 0x2a00 +#define MC_SEQ_MISC0_GDDR5_SHIFT 28 +#define MC_SEQ_MISC0_GDDR5_MASK 0xf0000000 +#define MC_SEQ_MISC0_GDDR5_VALUE 5 +#define MC_SEQ_MISC1 0x2a04 +#define MC_SEQ_RESERVE_M 0x2a08 +#define MC_PMG_CMD_EMRS 0x2a0c + +#define MC_SEQ_MISC3 0x2a2c + +#define MC_SEQ_MISC5 0x2a54 +#define MC_SEQ_MISC6 0x2a58 + +#define MC_SEQ_MISC7 0x2a64 + +#define MC_SEQ_CG 0x2a68 +#define CG_SEQ_REQ(x) ((x) << 0) +#define CG_SEQ_REQ_MASK (0xff << 0) +#define CG_SEQ_REQ_SHIFT 0 +#define CG_SEQ_RESP(x) ((x) << 8) +#define CG_SEQ_RESP_MASK (0xff << 8) +#define CG_SEQ_RESP_SHIFT 8 +#define SEQ_CG_REQ(x) ((x) << 16) +#define SEQ_CG_REQ_MASK (0xff << 16) +#define SEQ_CG_REQ_SHIFT 16 +#define SEQ_CG_RESP(x) ((x) << 24) +#define SEQ_CG_RESP_MASK (0xff << 24) +#define SEQ_CG_RESP_SHIFT 24 +#define MC_SEQ_RAS_TIMING_LP 0x2a6c +#define MC_SEQ_CAS_TIMING_LP 0x2a70 +#define MC_SEQ_MISC_TIMING_LP 0x2a74 +#define MC_SEQ_MISC_TIMING2_LP 0x2a78 +#define MC_SEQ_WR_CTL_D0_LP 0x2a7c +#define MC_SEQ_WR_CTL_D1_LP 0x2a80 +#define MC_SEQ_PMG_CMD_EMRS_LP 0x2a84 +#define MC_SEQ_PMG_CMD_MRS_LP 0x2a88 + +#define MC_PMG_CMD_MRS 0x2aac + +#define MC_SEQ_RD_CTL_D0_LP 0x2b1c +#define MC_SEQ_RD_CTL_D1_LP 0x2b20 + +#define MC_PMG_CMD_MRS1 0x2b44 +#define MC_SEQ_PMG_CMD_MRS1_LP 0x2b48 + +#define LB_SYNC_RESET_SEL 0x6b28 +#define LB_SYNC_RESET_SEL_MASK (3 << 0) +#define LB_SYNC_RESET_SEL_SHIFT 0 + +/* PCIE link stuff */ +#define PCIE_LC_SPEED_CNTL 0xa4 /* PCIE_P */ +# define LC_GEN2_EN_STRAP (1 << 0) +# define LC_TARGET_LINK_SPEED_OVERRIDE_EN (1 << 1) +# define LC_FORCE_EN_HW_SPEED_CHANGE (1 << 5) +# define LC_FORCE_DIS_HW_SPEED_CHANGE (1 << 6) +# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK (0x3 << 8) +# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT 3 +# define LC_CURRENT_DATA_RATE (1 << 11) +# define LC_HW_VOLTAGE_IF_CONTROL(x) ((x) << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_MASK (3 << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_SHIFT 12 +# define LC_VOLTAGE_TIMER_SEL_MASK (0xf << 14) +# define LC_CLR_FAILED_SPD_CHANGE_CNT (1 << 21) +# define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 23) +# define LC_OTHER_SIDE_SUPPORTS_GEN2 (1 << 24) + +#endif diff --git a/sys/dev/pci/drm/radeon/cayman_blit_shaders.c b/sys/dev/pci/drm/radeon/cayman_blit_shaders.c index 052390c629d..b9ca2001bc5 100644 --- a/sys/dev/pci/drm/radeon/cayman_blit_shaders.c +++ b/sys/dev/pci/drm/radeon/cayman_blit_shaders.c @@ -1,4 +1,3 @@ -/* $OpenBSD: cayman_blit_shaders.c,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * @@ -25,15 +24,13 @@ * Alex Deucher */ -#include - -#include +#include /* * evergreen cards need to use the 3D engine to blit data which requires * quite a bit of hw state setup. Rather than pull the whole 3D driver * (which normally generates the 3D state) into the DRM, we opt to use - * statically generated state tables. The regsiter state and shaders + * statically generated state tables. The register state and shaders * were hand generated to support blitting functionality. See the 3D * driver or documentation for descriptions of the registers and * shader instructions. @@ -318,58 +315,4 @@ const u32 cayman_default_state[] = 0x00000010, /* */ }; -const u32 cayman_vs[] = -{ - 0x00000004, - 0x80400400, - 0x0000a03c, - 0x95000688, - 0x00004000, - 0x15000688, - 0x00000000, - 0x88000000, - 0x04000000, - 0x67961001, -#ifdef __BIG_ENDIAN - 0x00020000, -#else - 0x00000000, -#endif - 0x00000000, - 0x04000000, - 0x67961000, -#ifdef __BIG_ENDIAN - 0x00020008, -#else - 0x00000008, -#endif - 0x00000000, -}; - -const u32 cayman_ps[] = -{ - 0x00000004, - 0xa00c0000, - 0x00000008, - 0x80400000, - 0x00000000, - 0x95000688, - 0x00000000, - 0x88000000, - 0x00380400, - 0x00146b10, - 0x00380000, - 0x20146b10, - 0x00380400, - 0x40146b00, - 0x80380000, - 0x60146b00, - 0x00000010, - 0x000d1000, - 0xb0800000, - 0x00000000, -}; - -const u32 cayman_ps_size = ARRAY_SIZE(cayman_ps); -const u32 cayman_vs_size = ARRAY_SIZE(cayman_vs); const u32 cayman_default_size = ARRAY_SIZE(cayman_default_state); diff --git a/sys/dev/pci/drm/radeon/cayman_blit_shaders.h b/sys/dev/pci/drm/radeon/cayman_blit_shaders.h index da0b35aaac2..f5d0e9a6026 100644 --- a/sys/dev/pci/drm/radeon/cayman_blit_shaders.h +++ b/sys/dev/pci/drm/radeon/cayman_blit_shaders.h @@ -1,4 +1,3 @@ -/* $OpenBSD: cayman_blit_shaders.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * diff --git a/sys/dev/pci/drm/radeon/ci_dpm.c b/sys/dev/pci/drm/radeon/ci_dpm.c new file mode 100644 index 00000000000..6702407d7f9 --- /dev/null +++ b/sys/dev/pci/drm/radeon/ci_dpm.c @@ -0,0 +1,5969 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "radeon_ucode.h" +#include "cikd.h" +#include "r600_dpm.h" +#include "ci_dpm.h" +#include "atom.h" + +#define MC_CG_ARB_FREQ_F0 0x0a +#define MC_CG_ARB_FREQ_F1 0x0b +#define MC_CG_ARB_FREQ_F2 0x0c +#define MC_CG_ARB_FREQ_F3 0x0d + +#define SMC_RAM_END 0x40000 + +#define VOLTAGE_SCALE 4 +#define VOLTAGE_VID_OFFSET_SCALE1 625 +#define VOLTAGE_VID_OFFSET_SCALE2 100 + +static const struct ci_pt_defaults defaults_hawaii_xt = +{ + 1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0xB0000, + { 0x2E, 0x00, 0x00, 0x88, 0x00, 0x00, 0x72, 0x60, 0x51, 0xA7, 0x79, 0x6B, 0x90, 0xBD, 0x79 }, + { 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 } +}; + +static const struct ci_pt_defaults defaults_hawaii_pro = +{ + 1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0x65062, + { 0x2E, 0x00, 0x00, 0x88, 0x00, 0x00, 0x72, 0x60, 0x51, 0xA7, 0x79, 0x6B, 0x90, 0xBD, 0x79 }, + { 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 } +}; + +static const struct ci_pt_defaults defaults_bonaire_xt = +{ + 1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000, + { 0x79, 0x253, 0x25D, 0xAE, 0x72, 0x80, 0x83, 0x86, 0x6F, 0xC8, 0xC9, 0xC9, 0x2F, 0x4D, 0x61 }, + { 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 } +}; + +static const struct ci_pt_defaults defaults_bonaire_pro = +{ + 1, 0xF, 0xFD, 0x19, 5, 45, 0, 0x65062, + { 0x8C, 0x23F, 0x244, 0xA6, 0x83, 0x85, 0x86, 0x86, 0x83, 0xDB, 0xDB, 0xDA, 0x67, 0x60, 0x5F }, + { 0x187, 0x193, 0x193, 0x1C7, 0x1D1, 0x1D1, 0x210, 0x219, 0x219, 0x266, 0x26C, 0x26C, 0x2C9, 0x2CB, 0x2CB } +}; + +static const struct ci_pt_defaults defaults_saturn_xt = +{ + 1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x70000, + { 0x8C, 0x247, 0x249, 0xA6, 0x80, 0x81, 0x8B, 0x89, 0x86, 0xC9, 0xCA, 0xC9, 0x4D, 0x4D, 0x4D }, + { 0x187, 0x187, 0x187, 0x1C7, 0x1C7, 0x1C7, 0x210, 0x210, 0x210, 0x266, 0x266, 0x266, 0x2C9, 0x2C9, 0x2C9 } +}; + +static const struct ci_pt_defaults defaults_saturn_pro = +{ + 1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x30000, + { 0x96, 0x21D, 0x23B, 0xA1, 0x85, 0x87, 0x83, 0x84, 0x81, 0xE6, 0xE6, 0xE6, 0x71, 0x6A, 0x6A }, + { 0x193, 0x19E, 0x19E, 0x1D2, 0x1DC, 0x1DC, 0x21A, 0x223, 0x223, 0x26E, 0x27E, 0x274, 0x2CF, 0x2D2, 0x2D2 } +}; + +static const struct ci_pt_config_reg didt_config_ci[] = +{ + { 0x10, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x10, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x10, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x10, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x11, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x11, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x11, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x11, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x12, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x12, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x12, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x12, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x2, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x2, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x2, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x1, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x1, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x0, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x30, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x30, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x30, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x30, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x31, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x31, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x31, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x31, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x32, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x32, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x32, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x32, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x22, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x22, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x22, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x21, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x21, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x20, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x50, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x50, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x50, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x50, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x51, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x51, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x51, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x51, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x52, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x52, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x52, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x52, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x42, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x42, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x42, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x41, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x41, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x40, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x70, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x70, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x70, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x70, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x71, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x71, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x71, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x71, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x72, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x72, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x72, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x72, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x62, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x62, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x62, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x61, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x61, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND }, + { 0x60, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND }, + { 0xFFFFFFFF } +}; + +extern u8 rv770_get_memory_module_index(struct radeon_device *rdev); +extern int ni_copy_and_switch_arb_sets(struct radeon_device *rdev, + u32 arb_freq_src, u32 arb_freq_dest); +extern u8 si_get_ddr3_mclk_frequency_ratio(u32 memory_clock); +extern u8 si_get_mclk_frequency_ratio(u32 memory_clock, bool strobe_mode); +extern void si_trim_voltage_table_to_fit_state_table(struct radeon_device *rdev, + u32 max_voltage_steps, + struct atom_voltage_table *voltage_table); +extern void cik_enter_rlc_safe_mode(struct radeon_device *rdev); +extern void cik_exit_rlc_safe_mode(struct radeon_device *rdev); +extern int ci_mc_load_microcode(struct radeon_device *rdev); +extern void cik_update_cg(struct radeon_device *rdev, + u32 block, bool enable); + +static int ci_get_std_voltage_value_sidd(struct radeon_device *rdev, + struct atom_voltage_table_entry *voltage_table, + u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd); +static int ci_set_power_limit(struct radeon_device *rdev, u32 n); +static int ci_set_overdrive_target_tdp(struct radeon_device *rdev, + u32 target_tdp); +static int ci_update_uvd_dpm(struct radeon_device *rdev, bool gate); + +static PPSMC_Result ci_send_msg_to_smc_with_parameter(struct radeon_device *rdev, + PPSMC_Msg msg, u32 parameter); + +static void ci_thermal_start_smc_fan_control(struct radeon_device *rdev); +static void ci_fan_ctrl_set_default_mode(struct radeon_device *rdev); + +static struct ci_power_info *ci_get_pi(struct radeon_device *rdev) +{ + struct ci_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +static struct ci_ps *ci_get_ps(struct radeon_ps *rps) +{ + struct ci_ps *ps = rps->ps_priv; + + return ps; +} + +static void ci_initialize_powertune_defaults(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + switch (rdev->pdev->device) { + case 0x6649: + case 0x6650: + case 0x6651: + case 0x6658: + case 0x665C: + case 0x665D: + default: + pi->powertune_defaults = &defaults_bonaire_xt; + break; + case 0x6640: + case 0x6641: + case 0x6646: + case 0x6647: + pi->powertune_defaults = &defaults_saturn_xt; + break; + case 0x67B8: + case 0x67B0: + pi->powertune_defaults = &defaults_hawaii_xt; + break; + case 0x67BA: + case 0x67B1: + pi->powertune_defaults = &defaults_hawaii_pro; + break; + case 0x67A0: + case 0x67A1: + case 0x67A2: + case 0x67A8: + case 0x67A9: + case 0x67AA: + case 0x67B9: + case 0x67BE: + pi->powertune_defaults = &defaults_bonaire_xt; + break; + } + + pi->dte_tj_offset = 0; + + pi->caps_power_containment = true; + pi->caps_cac = false; + pi->caps_sq_ramping = false; + pi->caps_db_ramping = false; + pi->caps_td_ramping = false; + pi->caps_tcp_ramping = false; + + if (pi->caps_power_containment) { + pi->caps_cac = true; + if (rdev->family == CHIP_HAWAII) + pi->enable_bapm_feature = false; + else + pi->enable_bapm_feature = true; + pi->enable_tdc_limit_feature = true; + pi->enable_pkg_pwr_tracking_feature = true; + } +} + +static u8 ci_convert_to_vid(u16 vddc) +{ + return (6200 - (vddc * VOLTAGE_SCALE)) / 25; +} + +static int ci_populate_bapm_vddc_vid_sidd(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd; + u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd; + u8 *hi2_vid = pi->smc_powertune_table.BapmVddCVidHiSidd2; + u32 i; + + if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries == NULL) + return -EINVAL; + if (rdev->pm.dpm.dyn_state.cac_leakage_table.count > 8) + return -EINVAL; + if (rdev->pm.dpm.dyn_state.cac_leakage_table.count != + rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count) + return -EINVAL; + + for (i = 0; i < rdev->pm.dpm.dyn_state.cac_leakage_table.count; i++) { + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) { + lo_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc1); + hi_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc2); + hi2_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc3); + } else { + lo_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc); + hi_vid[i] = ci_convert_to_vid((u16)rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].leakage); + } + } + return 0; +} + +static int ci_populate_vddc_vid(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u8 *vid = pi->smc_powertune_table.VddCVid; + u32 i; + + if (pi->vddc_voltage_table.count > 8) + return -EINVAL; + + for (i = 0; i < pi->vddc_voltage_table.count; i++) + vid[i] = ci_convert_to_vid(pi->vddc_voltage_table.entries[i].value); + + return 0; +} + +static int ci_populate_svi_load_line(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults; + + pi->smc_powertune_table.SviLoadLineEn = pt_defaults->svi_load_line_en; + pi->smc_powertune_table.SviLoadLineVddC = pt_defaults->svi_load_line_vddc; + pi->smc_powertune_table.SviLoadLineTrimVddC = 3; + pi->smc_powertune_table.SviLoadLineOffsetVddC = 0; + + return 0; +} + +static int ci_populate_tdc_limit(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults; + u16 tdc_limit; + + tdc_limit = rdev->pm.dpm.dyn_state.cac_tdp_table->tdc * 256; + pi->smc_powertune_table.TDC_VDDC_PkgLimit = cpu_to_be16(tdc_limit); + pi->smc_powertune_table.TDC_VDDC_ThrottleReleaseLimitPerc = + pt_defaults->tdc_vddc_throttle_release_limit_perc; + pi->smc_powertune_table.TDC_MAWt = pt_defaults->tdc_mawt; + + return 0; +} + +static int ci_populate_dw8(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults; + int ret; + + ret = ci_read_smc_sram_dword(rdev, + SMU7_FIRMWARE_HEADER_LOCATION + + offsetof(SMU7_Firmware_Header, PmFuseTable) + + offsetof(SMU7_Discrete_PmFuses, TdcWaterfallCtl), + (u32 *)&pi->smc_powertune_table.TdcWaterfallCtl, + pi->sram_end); + if (ret) + return -EINVAL; + else + pi->smc_powertune_table.TdcWaterfallCtl = pt_defaults->tdc_waterfall_ctl; + + return 0; +} + +static int ci_populate_fuzzy_fan(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if ((rdev->pm.dpm.fan.fan_output_sensitivity & (1 << 15)) || + (rdev->pm.dpm.fan.fan_output_sensitivity == 0)) + rdev->pm.dpm.fan.fan_output_sensitivity = + rdev->pm.dpm.fan.default_fan_output_sensitivity; + + pi->smc_powertune_table.FuzzyFan_PwmSetDelta = + cpu_to_be16(rdev->pm.dpm.fan.fan_output_sensitivity); + + return 0; +} + +static int ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd; + u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd; + int i, min, max; + + min = max = hi_vid[0]; + for (i = 0; i < 8; i++) { + if (0 != hi_vid[i]) { + if (min > hi_vid[i]) + min = hi_vid[i]; + if (max < hi_vid[i]) + max = hi_vid[i]; + } + + if (0 != lo_vid[i]) { + if (min > lo_vid[i]) + min = lo_vid[i]; + if (max < lo_vid[i]) + max = lo_vid[i]; + } + } + + if ((min == 0) || (max == 0)) + return -EINVAL; + pi->smc_powertune_table.GnbLPMLMaxVid = (u8)max; + pi->smc_powertune_table.GnbLPMLMinVid = (u8)min; + + return 0; +} + +static int ci_populate_bapm_vddc_base_leakage_sidd(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u16 hi_sidd = pi->smc_powertune_table.BapmVddCBaseLeakageHiSidd; + u16 lo_sidd = pi->smc_powertune_table.BapmVddCBaseLeakageLoSidd; + struct radeon_cac_tdp_table *cac_tdp_table = + rdev->pm.dpm.dyn_state.cac_tdp_table; + + hi_sidd = cac_tdp_table->high_cac_leakage / 100 * 256; + lo_sidd = cac_tdp_table->low_cac_leakage / 100 * 256; + + pi->smc_powertune_table.BapmVddCBaseLeakageHiSidd = cpu_to_be16(hi_sidd); + pi->smc_powertune_table.BapmVddCBaseLeakageLoSidd = cpu_to_be16(lo_sidd); + + return 0; +} + +static int ci_populate_bapm_parameters_in_dpm_table(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults; + SMU7_Discrete_DpmTable *dpm_table = &pi->smc_state_table; + struct radeon_cac_tdp_table *cac_tdp_table = + rdev->pm.dpm.dyn_state.cac_tdp_table; + struct radeon_ppm_table *ppm = rdev->pm.dpm.dyn_state.ppm_table; + int i, j, k; + const u16 *def1; + const u16 *def2; + + dpm_table->DefaultTdp = cac_tdp_table->tdp * 256; + dpm_table->TargetTdp = cac_tdp_table->configurable_tdp * 256; + + dpm_table->DTETjOffset = (u8)pi->dte_tj_offset; + dpm_table->GpuTjMax = + (u8)(pi->thermal_temp_setting.temperature_high / 1000); + dpm_table->GpuTjHyst = 8; + + dpm_table->DTEAmbientTempBase = pt_defaults->dte_ambient_temp_base; + + if (ppm) { + dpm_table->PPM_PkgPwrLimit = cpu_to_be16((u16)ppm->dgpu_tdp * 256 / 1000); + dpm_table->PPM_TemperatureLimit = cpu_to_be16((u16)ppm->tj_max * 256); + } else { + dpm_table->PPM_PkgPwrLimit = cpu_to_be16(0); + dpm_table->PPM_TemperatureLimit = cpu_to_be16(0); + } + + dpm_table->BAPM_TEMP_GRADIENT = cpu_to_be32(pt_defaults->bapm_temp_gradient); + def1 = pt_defaults->bapmti_r; + def2 = pt_defaults->bapmti_rc; + + for (i = 0; i < SMU7_DTE_ITERATIONS; i++) { + for (j = 0; j < SMU7_DTE_SOURCES; j++) { + for (k = 0; k < SMU7_DTE_SINKS; k++) { + dpm_table->BAPMTI_R[i][j][k] = cpu_to_be16(*def1); + dpm_table->BAPMTI_RC[i][j][k] = cpu_to_be16(*def2); + def1++; + def2++; + } + } + } + + return 0; +} + +static int ci_populate_pm_base(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 pm_fuse_table_offset; + int ret; + + if (pi->caps_power_containment) { + ret = ci_read_smc_sram_dword(rdev, + SMU7_FIRMWARE_HEADER_LOCATION + + offsetof(SMU7_Firmware_Header, PmFuseTable), + &pm_fuse_table_offset, pi->sram_end); + if (ret) + return ret; + ret = ci_populate_bapm_vddc_vid_sidd(rdev); + if (ret) + return ret; + ret = ci_populate_vddc_vid(rdev); + if (ret) + return ret; + ret = ci_populate_svi_load_line(rdev); + if (ret) + return ret; + ret = ci_populate_tdc_limit(rdev); + if (ret) + return ret; + ret = ci_populate_dw8(rdev); + if (ret) + return ret; + ret = ci_populate_fuzzy_fan(rdev); + if (ret) + return ret; + ret = ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(rdev); + if (ret) + return ret; + ret = ci_populate_bapm_vddc_base_leakage_sidd(rdev); + if (ret) + return ret; + ret = ci_copy_bytes_to_smc(rdev, pm_fuse_table_offset, + (u8 *)&pi->smc_powertune_table, + sizeof(SMU7_Discrete_PmFuses), pi->sram_end); + if (ret) + return ret; + } + + return 0; +} + +static void ci_do_enable_didt(struct radeon_device *rdev, const bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 data; + + if (pi->caps_sq_ramping) { + data = RREG32_DIDT(DIDT_SQ_CTRL0); + if (enable) + data |= DIDT_CTRL_EN; + else + data &= ~DIDT_CTRL_EN; + WREG32_DIDT(DIDT_SQ_CTRL0, data); + } + + if (pi->caps_db_ramping) { + data = RREG32_DIDT(DIDT_DB_CTRL0); + if (enable) + data |= DIDT_CTRL_EN; + else + data &= ~DIDT_CTRL_EN; + WREG32_DIDT(DIDT_DB_CTRL0, data); + } + + if (pi->caps_td_ramping) { + data = RREG32_DIDT(DIDT_TD_CTRL0); + if (enable) + data |= DIDT_CTRL_EN; + else + data &= ~DIDT_CTRL_EN; + WREG32_DIDT(DIDT_TD_CTRL0, data); + } + + if (pi->caps_tcp_ramping) { + data = RREG32_DIDT(DIDT_TCP_CTRL0); + if (enable) + data |= DIDT_CTRL_EN; + else + data &= ~DIDT_CTRL_EN; + WREG32_DIDT(DIDT_TCP_CTRL0, data); + } +} + +static int ci_program_pt_config_registers(struct radeon_device *rdev, + const struct ci_pt_config_reg *cac_config_regs) +{ + const struct ci_pt_config_reg *config_regs = cac_config_regs; + u32 data; + u32 cache = 0; + + if (config_regs == NULL) + return -EINVAL; + + while (config_regs->offset != 0xFFFFFFFF) { + if (config_regs->type == CISLANDS_CONFIGREG_CACHE) { + cache |= ((config_regs->value << config_regs->shift) & config_regs->mask); + } else { + switch (config_regs->type) { + case CISLANDS_CONFIGREG_SMC_IND: + data = RREG32_SMC(config_regs->offset); + break; + case CISLANDS_CONFIGREG_DIDT_IND: + data = RREG32_DIDT(config_regs->offset); + break; + default: + data = RREG32(config_regs->offset << 2); + break; + } + + data &= ~config_regs->mask; + data |= ((config_regs->value << config_regs->shift) & config_regs->mask); + data |= cache; + + switch (config_regs->type) { + case CISLANDS_CONFIGREG_SMC_IND: + WREG32_SMC(config_regs->offset, data); + break; + case CISLANDS_CONFIGREG_DIDT_IND: + WREG32_DIDT(config_regs->offset, data); + break; + default: + WREG32(config_regs->offset << 2, data); + break; + } + cache = 0; + } + config_regs++; + } + return 0; +} + +static int ci_enable_didt(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + int ret; + + if (pi->caps_sq_ramping || pi->caps_db_ramping || + pi->caps_td_ramping || pi->caps_tcp_ramping) { + cik_enter_rlc_safe_mode(rdev); + + if (enable) { + ret = ci_program_pt_config_registers(rdev, didt_config_ci); + if (ret) { + cik_exit_rlc_safe_mode(rdev); + return ret; + } + } + + ci_do_enable_didt(rdev, enable); + + cik_exit_rlc_safe_mode(rdev); + } + + return 0; +} + +static int ci_enable_power_containment(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result smc_result; + int ret = 0; + + if (enable) { + pi->power_containment_features = 0; + if (pi->caps_power_containment) { + if (pi->enable_bapm_feature) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableDTE); + if (smc_result != PPSMC_Result_OK) + ret = -EINVAL; + else + pi->power_containment_features |= POWERCONTAINMENT_FEATURE_BAPM; + } + + if (pi->enable_tdc_limit_feature) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_TDCLimitEnable); + if (smc_result != PPSMC_Result_OK) + ret = -EINVAL; + else + pi->power_containment_features |= POWERCONTAINMENT_FEATURE_TDCLimit; + } + + if (pi->enable_pkg_pwr_tracking_feature) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_PkgPwrLimitEnable); + if (smc_result != PPSMC_Result_OK) { + ret = -EINVAL; + } else { + struct radeon_cac_tdp_table *cac_tdp_table = + rdev->pm.dpm.dyn_state.cac_tdp_table; + u32 default_pwr_limit = + (u32)(cac_tdp_table->maximum_power_delivery_limit * 256); + + pi->power_containment_features |= POWERCONTAINMENT_FEATURE_PkgPwrLimit; + + ci_set_power_limit(rdev, default_pwr_limit); + } + } + } + } else { + if (pi->caps_power_containment && pi->power_containment_features) { + if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_TDCLimit) + ci_send_msg_to_smc(rdev, PPSMC_MSG_TDCLimitDisable); + + if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_BAPM) + ci_send_msg_to_smc(rdev, PPSMC_MSG_DisableDTE); + + if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit) + ci_send_msg_to_smc(rdev, PPSMC_MSG_PkgPwrLimitDisable); + pi->power_containment_features = 0; + } + } + + return ret; +} + +static int ci_enable_smc_cac(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result smc_result; + int ret = 0; + + if (pi->caps_cac) { + if (enable) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac); + if (smc_result != PPSMC_Result_OK) { + ret = -EINVAL; + pi->cac_enabled = false; + } else { + pi->cac_enabled = true; + } + } else if (pi->cac_enabled) { + ci_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac); + pi->cac_enabled = false; + } + } + + return ret; +} + +static int ci_enable_thermal_based_sclk_dpm(struct radeon_device *rdev, + bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result smc_result = PPSMC_Result_OK; + + if (pi->thermal_sclk_dpm_enabled) { + if (enable) + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_ENABLE_THERMAL_DPM); + else + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DISABLE_THERMAL_DPM); + } + + if (smc_result == PPSMC_Result_OK) + return 0; + else + return -EINVAL; +} + +static int ci_power_control_set_level(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_cac_tdp_table *cac_tdp_table = + rdev->pm.dpm.dyn_state.cac_tdp_table; + s32 adjust_percent; + s32 target_tdp; + int ret = 0; + bool adjust_polarity = false; /* ??? */ + + if (pi->caps_power_containment) { + adjust_percent = adjust_polarity ? + rdev->pm.dpm.tdp_adjustment : (-1 * rdev->pm.dpm.tdp_adjustment); + target_tdp = ((100 + adjust_percent) * + (s32)cac_tdp_table->configurable_tdp) / 100; + + ret = ci_set_overdrive_target_tdp(rdev, (u32)target_tdp); + } + + return ret; +} + +void ci_dpm_powergate_uvd(struct radeon_device *rdev, bool gate) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (pi->uvd_power_gated == gate) + return; + + pi->uvd_power_gated = gate; + + ci_update_uvd_dpm(rdev, gate); +} + +bool ci_dpm_vblank_too_short(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 vblank_time = r600_dpm_get_vblank_time(rdev); + u32 switch_limit = pi->mem_gddr5 ? 450 : 300; + + /* disable mclk switching if the refresh is >120Hz, even if the + * blanking period would allow it + */ + if (r600_dpm_get_vrefresh(rdev) > 120) + return true; + + /* disable mclk switching if the refresh is >120Hz, even if the + * blanking period would allow it + */ + if (r600_dpm_get_vrefresh(rdev) > 120) + return true; + + if (vblank_time < switch_limit) + return true; + else + return false; + +} + +static void ci_apply_state_adjust_rules(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct ci_ps *ps = ci_get_ps(rps); + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_clock_and_voltage_limits *max_limits; + bool disable_mclk_switching; + u32 sclk, mclk; + int i; + + if (rps->vce_active) { + rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk; + rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk; + } else { + rps->evclk = 0; + rps->ecclk = 0; + } + + if ((rdev->pm.dpm.new_active_crtc_count > 1) || + ci_dpm_vblank_too_short(rdev)) + disable_mclk_switching = true; + else + disable_mclk_switching = false; + + if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) + pi->battery_state = true; + else + pi->battery_state = false; + + if (rdev->pm.dpm.ac_power) + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + else + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc; + + if (rdev->pm.dpm.ac_power == false) { + for (i = 0; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].mclk > max_limits->mclk) + ps->performance_levels[i].mclk = max_limits->mclk; + if (ps->performance_levels[i].sclk > max_limits->sclk) + ps->performance_levels[i].sclk = max_limits->sclk; + } + } + + /* XXX validate the min clocks required for display */ + + if (disable_mclk_switching) { + mclk = ps->performance_levels[ps->performance_level_count - 1].mclk; + sclk = ps->performance_levels[0].sclk; + } else { + mclk = ps->performance_levels[0].mclk; + sclk = ps->performance_levels[0].sclk; + } + + if (rps->vce_active) { + if (sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk) + sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk; + if (mclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk) + mclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk; + } + + ps->performance_levels[0].sclk = sclk; + ps->performance_levels[0].mclk = mclk; + + if (ps->performance_levels[1].sclk < ps->performance_levels[0].sclk) + ps->performance_levels[1].sclk = ps->performance_levels[0].sclk; + + if (disable_mclk_switching) { + if (ps->performance_levels[0].mclk < ps->performance_levels[1].mclk) + ps->performance_levels[0].mclk = ps->performance_levels[1].mclk; + } else { + if (ps->performance_levels[1].mclk < ps->performance_levels[0].mclk) + ps->performance_levels[1].mclk = ps->performance_levels[0].mclk; + } +} + +static int ci_thermal_set_temperature_range(struct radeon_device *rdev, + int min_temp, int max_temp) +{ + int low_temp = 0 * 1000; + int high_temp = 255 * 1000; + u32 tmp; + + if (low_temp < min_temp) + low_temp = min_temp; + if (high_temp > max_temp) + high_temp = max_temp; + if (high_temp < low_temp) { + DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp); + return -EINVAL; + } + + tmp = RREG32_SMC(CG_THERMAL_INT); + tmp &= ~(CI_DIG_THERM_INTH_MASK | CI_DIG_THERM_INTL_MASK); + tmp |= CI_DIG_THERM_INTH(high_temp / 1000) | + CI_DIG_THERM_INTL(low_temp / 1000); + WREG32_SMC(CG_THERMAL_INT, tmp); + +#if 0 + /* XXX: need to figure out how to handle this properly */ + tmp = RREG32_SMC(CG_THERMAL_CTRL); + tmp &= DIG_THERM_DPM_MASK; + tmp |= DIG_THERM_DPM(high_temp / 1000); + WREG32_SMC(CG_THERMAL_CTRL, tmp); +#endif + + rdev->pm.dpm.thermal.min_temp = low_temp; + rdev->pm.dpm.thermal.max_temp = high_temp; + + return 0; +} + +static int ci_thermal_enable_alert(struct radeon_device *rdev, + bool enable) +{ + u32 thermal_int = RREG32_SMC(CG_THERMAL_INT); + PPSMC_Result result; + + if (enable) { + thermal_int &= ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + WREG32_SMC(CG_THERMAL_INT, thermal_int); + rdev->irq.dpm_thermal = false; + result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Thermal_Cntl_Enable); + if (result != PPSMC_Result_OK) { + DRM_DEBUG_KMS("Could not enable thermal interrupts.\n"); + return -EINVAL; + } + } else { + thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; + WREG32_SMC(CG_THERMAL_INT, thermal_int); + rdev->irq.dpm_thermal = true; + result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Thermal_Cntl_Disable); + if (result != PPSMC_Result_OK) { + DRM_DEBUG_KMS("Could not disable thermal interrupts.\n"); + return -EINVAL; + } + } + + return 0; +} + +static void ci_fan_ctrl_set_static_mode(struct radeon_device *rdev, u32 mode) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + + if (pi->fan_ctrl_is_in_default_mode) { + tmp = (RREG32_SMC(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK) >> FDO_PWM_MODE_SHIFT; + pi->fan_ctrl_default_mode = tmp; + tmp = (RREG32_SMC(CG_FDO_CTRL2) & TMIN_MASK) >> TMIN_SHIFT; + pi->t_min = tmp; + pi->fan_ctrl_is_in_default_mode = false; + } + + tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TMIN_MASK; + tmp |= TMIN(0); + WREG32_SMC(CG_FDO_CTRL2, tmp); + + tmp = RREG32_SMC(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK; + tmp |= FDO_PWM_MODE(mode); + WREG32_SMC(CG_FDO_CTRL2, tmp); +} + +static int ci_thermal_setup_fan_table(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + SMU7_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE }; + u32 duty100; + u32 t_diff1, t_diff2, pwm_diff1, pwm_diff2; + u16 fdo_min, slope1, slope2; + u32 reference_clock, tmp; + int ret; + u64 tmp64; + + if (!pi->fan_table_start) { + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; + } + + duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + + if (duty100 == 0) { + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; + } + + tmp64 = (u64)rdev->pm.dpm.fan.pwm_min * duty100; + do_div(tmp64, 10000); + fdo_min = (u16)tmp64; + + t_diff1 = rdev->pm.dpm.fan.t_med - rdev->pm.dpm.fan.t_min; + t_diff2 = rdev->pm.dpm.fan.t_high - rdev->pm.dpm.fan.t_med; + + pwm_diff1 = rdev->pm.dpm.fan.pwm_med - rdev->pm.dpm.fan.pwm_min; + pwm_diff2 = rdev->pm.dpm.fan.pwm_high - rdev->pm.dpm.fan.pwm_med; + + slope1 = (u16)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100); + slope2 = (u16)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100); + + fan_table.TempMin = cpu_to_be16((50 + rdev->pm.dpm.fan.t_min) / 100); + fan_table.TempMed = cpu_to_be16((50 + rdev->pm.dpm.fan.t_med) / 100); + fan_table.TempMax = cpu_to_be16((50 + rdev->pm.dpm.fan.t_max) / 100); + + fan_table.Slope1 = cpu_to_be16(slope1); + fan_table.Slope2 = cpu_to_be16(slope2); + + fan_table.FdoMin = cpu_to_be16(fdo_min); + + fan_table.HystDown = cpu_to_be16(rdev->pm.dpm.fan.t_hyst); + + fan_table.HystUp = cpu_to_be16(1); + + fan_table.HystSlope = cpu_to_be16(1); + + fan_table.TempRespLim = cpu_to_be16(5); + + reference_clock = radeon_get_xclk(rdev); + + fan_table.RefreshPeriod = cpu_to_be32((rdev->pm.dpm.fan.cycle_delay * + reference_clock) / 1600); + + fan_table.FdoMax = cpu_to_be16((u16)duty100); + + tmp = (RREG32_SMC(CG_MULT_THERMAL_CTRL) & TEMP_SEL_MASK) >> TEMP_SEL_SHIFT; + fan_table.TempSrc = (uint8_t)tmp; + + ret = ci_copy_bytes_to_smc(rdev, + pi->fan_table_start, + (u8 *)(&fan_table), + sizeof(fan_table), + pi->sram_end); + + if (ret) { + DRM_ERROR("Failed to load fan table to the SMC."); + rdev->pm.dpm.fan.ucode_fan_control = false; + } + + return 0; +} + +static int ci_fan_ctrl_start_smc_fan_control(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result ret; + + if (pi->caps_od_fuzzy_fan_control_support) { + ret = ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_StartFanControl, + FAN_CONTROL_FUZZY); + if (ret != PPSMC_Result_OK) + return -EINVAL; + ret = ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_SetFanPwmMax, + rdev->pm.dpm.fan.default_max_fan_pwm); + if (ret != PPSMC_Result_OK) + return -EINVAL; + } else { + ret = ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_StartFanControl, + FAN_CONTROL_TABLE); + if (ret != PPSMC_Result_OK) + return -EINVAL; + } + + pi->fan_is_controlled_by_smc = true; + return 0; +} + +static int ci_fan_ctrl_stop_smc_fan_control(struct radeon_device *rdev) +{ + PPSMC_Result ret; + struct ci_power_info *pi = ci_get_pi(rdev); + + ret = ci_send_msg_to_smc(rdev, PPSMC_StopFanControl); + if (ret == PPSMC_Result_OK) { + pi->fan_is_controlled_by_smc = false; + return 0; + } else + return -EINVAL; +} + +int ci_fan_ctrl_get_fan_speed_percent(struct radeon_device *rdev, + u32 *speed) +{ + u32 duty, duty100; + u64 tmp64; + + if (rdev->pm.no_fan) + return -ENOENT; + + duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + duty = (RREG32_SMC(CG_THERMAL_STATUS) & FDO_PWM_DUTY_MASK) >> FDO_PWM_DUTY_SHIFT; + + if (duty100 == 0) + return -EINVAL; + + tmp64 = (u64)duty * 100; + do_div(tmp64, duty100); + *speed = (u32)tmp64; + + if (*speed > 100) + *speed = 100; + + return 0; +} + +int ci_fan_ctrl_set_fan_speed_percent(struct radeon_device *rdev, + u32 speed) +{ + u32 tmp; + u32 duty, duty100; + u64 tmp64; + struct ci_power_info *pi = ci_get_pi(rdev); + + if (rdev->pm.no_fan) + return -ENOENT; + + if (pi->fan_is_controlled_by_smc) + return -EINVAL; + + if (speed > 100) + return -EINVAL; + + duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + + if (duty100 == 0) + return -EINVAL; + + tmp64 = (u64)speed * duty100; + do_div(tmp64, 100); + duty = (u32)tmp64; + + tmp = RREG32_SMC(CG_FDO_CTRL0) & ~FDO_STATIC_DUTY_MASK; + tmp |= FDO_STATIC_DUTY(duty); + WREG32_SMC(CG_FDO_CTRL0, tmp); + + return 0; +} + +void ci_fan_ctrl_set_mode(struct radeon_device *rdev, u32 mode) +{ + if (mode) { + /* stop auto-manage */ + if (rdev->pm.dpm.fan.ucode_fan_control) + ci_fan_ctrl_stop_smc_fan_control(rdev); + ci_fan_ctrl_set_static_mode(rdev, mode); + } else { + /* restart auto-manage */ + if (rdev->pm.dpm.fan.ucode_fan_control) + ci_thermal_start_smc_fan_control(rdev); + else + ci_fan_ctrl_set_default_mode(rdev); + } +} + +u32 ci_fan_ctrl_get_mode(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + + if (pi->fan_is_controlled_by_smc) + return 0; + + tmp = RREG32_SMC(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK; + return (tmp >> FDO_PWM_MODE_SHIFT); +} + +#if 0 +static int ci_fan_ctrl_get_fan_speed_rpm(struct radeon_device *rdev, + u32 *speed) +{ + u32 tach_period; + u32 xclk = radeon_get_xclk(rdev); + + if (rdev->pm.no_fan) + return -ENOENT; + + if (rdev->pm.fan_pulses_per_revolution == 0) + return -ENOENT; + + tach_period = (RREG32_SMC(CG_TACH_STATUS) & TACH_PERIOD_MASK) >> TACH_PERIOD_SHIFT; + if (tach_period == 0) + return -ENOENT; + + *speed = 60 * xclk * 10000 / tach_period; + + return 0; +} + +static int ci_fan_ctrl_set_fan_speed_rpm(struct radeon_device *rdev, + u32 speed) +{ + u32 tach_period, tmp; + u32 xclk = radeon_get_xclk(rdev); + + if (rdev->pm.no_fan) + return -ENOENT; + + if (rdev->pm.fan_pulses_per_revolution == 0) + return -ENOENT; + + if ((speed < rdev->pm.fan_min_rpm) || + (speed > rdev->pm.fan_max_rpm)) + return -EINVAL; + + if (rdev->pm.dpm.fan.ucode_fan_control) + ci_fan_ctrl_stop_smc_fan_control(rdev); + + tach_period = 60 * xclk * 10000 / (8 * speed); + tmp = RREG32_SMC(CG_TACH_CTRL) & ~TARGET_PERIOD_MASK; + tmp |= TARGET_PERIOD(tach_period); + WREG32_SMC(CG_TACH_CTRL, tmp); + + ci_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC_RPM); + + return 0; +} +#endif + +static void ci_fan_ctrl_set_default_mode(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + + if (!pi->fan_ctrl_is_in_default_mode) { + tmp = RREG32_SMC(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK; + tmp |= FDO_PWM_MODE(pi->fan_ctrl_default_mode); + WREG32_SMC(CG_FDO_CTRL2, tmp); + + tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TMIN_MASK; + tmp |= TMIN(pi->t_min); + WREG32_SMC(CG_FDO_CTRL2, tmp); + pi->fan_ctrl_is_in_default_mode = true; + } +} + +static void ci_thermal_start_smc_fan_control(struct radeon_device *rdev) +{ + if (rdev->pm.dpm.fan.ucode_fan_control) { + ci_fan_ctrl_start_smc_fan_control(rdev); + ci_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC); + } +} + +static void ci_thermal_initialize(struct radeon_device *rdev) +{ + u32 tmp; + + if (rdev->pm.fan_pulses_per_revolution) { + tmp = RREG32_SMC(CG_TACH_CTRL) & ~EDGE_PER_REV_MASK; + tmp |= EDGE_PER_REV(rdev->pm.fan_pulses_per_revolution -1); + WREG32_SMC(CG_TACH_CTRL, tmp); + } + + tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TACH_PWM_RESP_RATE_MASK; + tmp |= TACH_PWM_RESP_RATE(0x28); + WREG32_SMC(CG_FDO_CTRL2, tmp); +} + +static int ci_thermal_start_thermal_controller(struct radeon_device *rdev) +{ + int ret; + + ci_thermal_initialize(rdev); + ret = ci_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) + return ret; + ret = ci_thermal_enable_alert(rdev, true); + if (ret) + return ret; + if (rdev->pm.dpm.fan.ucode_fan_control) { + ret = ci_thermal_setup_fan_table(rdev); + if (ret) + return ret; + ci_thermal_start_smc_fan_control(rdev); + } + + return 0; +} + +static void ci_thermal_stop_thermal_controller(struct radeon_device *rdev) +{ + if (!rdev->pm.no_fan) + ci_fan_ctrl_set_default_mode(rdev); +} + +#if 0 +static int ci_read_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 *value) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + return ci_read_smc_sram_dword(rdev, + pi->soft_regs_start + reg_offset, + value, pi->sram_end); +} +#endif + +static int ci_write_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 value) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + return ci_write_smc_sram_dword(rdev, + pi->soft_regs_start + reg_offset, + value, pi->sram_end); +} + +static void ci_init_fps_limits(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + SMU7_Discrete_DpmTable *table = &pi->smc_state_table; + + if (pi->caps_fps) { + u16 tmp; + + tmp = 45; + table->FpsHighT = cpu_to_be16(tmp); + + tmp = 30; + table->FpsLowT = cpu_to_be16(tmp); + } +} + +static int ci_update_sclk_t(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + int ret = 0; + u32 low_sclk_interrupt_t = 0; + + if (pi->caps_sclk_throttle_low_notification) { + low_sclk_interrupt_t = cpu_to_be32(pi->low_sclk_interrupt_t); + + ret = ci_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Discrete_DpmTable, LowSclkInterruptT), + (u8 *)&low_sclk_interrupt_t, + sizeof(u32), pi->sram_end); + + } + + return ret; +} + +static void ci_get_leakage_voltages(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u16 leakage_id, virtual_voltage_id; + u16 vddc, vddci; + int i; + + pi->vddc_leakage.count = 0; + pi->vddci_leakage.count = 0; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) { + for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) { + virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i; + if (radeon_atom_get_voltage_evv(rdev, virtual_voltage_id, &vddc) != 0) + continue; + if (vddc != 0 && vddc != virtual_voltage_id) { + pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc; + pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id; + pi->vddc_leakage.count++; + } + } + } else if (radeon_atom_get_leakage_id_from_vbios(rdev, &leakage_id) == 0) { + for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) { + virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i; + if (radeon_atom_get_leakage_vddc_based_on_leakage_params(rdev, &vddc, &vddci, + virtual_voltage_id, + leakage_id) == 0) { + if (vddc != 0 && vddc != virtual_voltage_id) { + pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc; + pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id; + pi->vddc_leakage.count++; + } + if (vddci != 0 && vddci != virtual_voltage_id) { + pi->vddci_leakage.actual_voltage[pi->vddci_leakage.count] = vddci; + pi->vddci_leakage.leakage_id[pi->vddci_leakage.count] = virtual_voltage_id; + pi->vddci_leakage.count++; + } + } + } + } +} + +static void ci_set_dpm_event_sources(struct radeon_device *rdev, u32 sources) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + bool want_thermal_protection; + enum radeon_dpm_event_src dpm_event_src; + u32 tmp; + + switch (sources) { + case 0: + default: + want_thermal_protection = false; + break; + case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL; + break; + case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL; + break; + case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) | + (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL; + break; + } + + if (want_thermal_protection) { +#if 0 + /* XXX: need to figure out how to handle this properly */ + tmp = RREG32_SMC(CG_THERMAL_CTRL); + tmp &= DPM_EVENT_SRC_MASK; + tmp |= DPM_EVENT_SRC(dpm_event_src); + WREG32_SMC(CG_THERMAL_CTRL, tmp); +#endif + + tmp = RREG32_SMC(GENERAL_PWRMGT); + if (pi->thermal_protection) + tmp &= ~THERMAL_PROTECTION_DIS; + else + tmp |= THERMAL_PROTECTION_DIS; + WREG32_SMC(GENERAL_PWRMGT, tmp); + } else { + tmp = RREG32_SMC(GENERAL_PWRMGT); + tmp |= THERMAL_PROTECTION_DIS; + WREG32_SMC(GENERAL_PWRMGT, tmp); + } +} + +static void ci_enable_auto_throttle_source(struct radeon_device *rdev, + enum radeon_dpm_auto_throttle_src source, + bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (enable) { + if (!(pi->active_auto_throttle_sources & (1 << source))) { + pi->active_auto_throttle_sources |= 1 << source; + ci_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources); + } + } else { + if (pi->active_auto_throttle_sources & (1 << source)) { + pi->active_auto_throttle_sources &= ~(1 << source); + ci_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources); + } + } +} + +static void ci_enable_vr_hot_gpio_interrupt(struct radeon_device *rdev) +{ + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT) + ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableVRHotGPIOInterrupt); +} + +static int ci_unfreeze_sclk_mclk_dpm(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result smc_result; + + if (!pi->need_update_smu7_dpm_table) + return 0; + + if ((!pi->sclk_dpm_key_disabled) && + (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_SCLKDPM_UnfreezeLevel); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + if ((!pi->mclk_dpm_key_disabled) && + (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_UnfreezeLevel); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + pi->need_update_smu7_dpm_table = 0; + return 0; +} + +static int ci_enable_sclk_mclk_dpm(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result smc_result; + + if (enable) { + if (!pi->sclk_dpm_key_disabled) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DPM_Enable); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + if (!pi->mclk_dpm_key_disabled) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_Enable); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + + WREG32_P(MC_SEQ_CNTL_3, CAC_EN, ~CAC_EN); + + WREG32_SMC(LCAC_MC0_CNTL, 0x05); + WREG32_SMC(LCAC_MC1_CNTL, 0x05); + WREG32_SMC(LCAC_CPL_CNTL, 0x100005); + + udelay(10); + + WREG32_SMC(LCAC_MC0_CNTL, 0x400005); + WREG32_SMC(LCAC_MC1_CNTL, 0x400005); + WREG32_SMC(LCAC_CPL_CNTL, 0x500005); + } + } else { + if (!pi->sclk_dpm_key_disabled) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DPM_Disable); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + if (!pi->mclk_dpm_key_disabled) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_Disable); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + } + + return 0; +} + +static int ci_start_dpm(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result smc_result; + int ret; + u32 tmp; + + tmp = RREG32_SMC(GENERAL_PWRMGT); + tmp |= GLOBAL_PWRMGT_EN; + WREG32_SMC(GENERAL_PWRMGT, tmp); + + tmp = RREG32_SMC(SCLK_PWRMGT_CNTL); + tmp |= DYNAMIC_PM_EN; + WREG32_SMC(SCLK_PWRMGT_CNTL, tmp); + + ci_write_smc_soft_register(rdev, offsetof(SMU7_SoftRegisters, VoltageChangeTimeout), 0x1000); + + WREG32_P(BIF_LNCNT_RESET, 0, ~RESET_LNCNT_EN); + + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Voltage_Cntl_Enable); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + + ret = ci_enable_sclk_mclk_dpm(rdev, true); + if (ret) + return ret; + + if (!pi->pcie_dpm_key_disabled) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_PCIeDPM_Enable); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + return 0; +} + +static int ci_freeze_sclk_mclk_dpm(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result smc_result; + + if (!pi->need_update_smu7_dpm_table) + return 0; + + if ((!pi->sclk_dpm_key_disabled) && + (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_SCLKDPM_FreezeLevel); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + if ((!pi->mclk_dpm_key_disabled) && + (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_FreezeLevel); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + return 0; +} + +static int ci_stop_dpm(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result smc_result; + int ret; + u32 tmp; + + tmp = RREG32_SMC(GENERAL_PWRMGT); + tmp &= ~GLOBAL_PWRMGT_EN; + WREG32_SMC(GENERAL_PWRMGT, tmp); + + tmp = RREG32_SMC(SCLK_PWRMGT_CNTL); + tmp &= ~DYNAMIC_PM_EN; + WREG32_SMC(SCLK_PWRMGT_CNTL, tmp); + + if (!pi->pcie_dpm_key_disabled) { + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_PCIeDPM_Disable); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + ret = ci_enable_sclk_mclk_dpm(rdev, false); + if (ret) + return ret; + + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Voltage_Cntl_Disable); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + + return 0; +} + +static void ci_enable_sclk_control(struct radeon_device *rdev, bool enable) +{ + u32 tmp = RREG32_SMC(SCLK_PWRMGT_CNTL); + + if (enable) + tmp &= ~SCLK_PWRMGT_OFF; + else + tmp |= SCLK_PWRMGT_OFF; + WREG32_SMC(SCLK_PWRMGT_CNTL, tmp); +} + +#if 0 +static int ci_notify_hw_of_power_source(struct radeon_device *rdev, + bool ac_power) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_cac_tdp_table *cac_tdp_table = + rdev->pm.dpm.dyn_state.cac_tdp_table; + u32 power_limit; + + if (ac_power) + power_limit = (u32)(cac_tdp_table->maximum_power_delivery_limit * 256); + else + power_limit = (u32)(cac_tdp_table->battery_power_limit * 256); + + ci_set_power_limit(rdev, power_limit); + + if (pi->caps_automatic_dc_transition) { + if (ac_power) + ci_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC); + else + ci_send_msg_to_smc(rdev, PPSMC_MSG_Remove_DC_Clamp); + } + + return 0; +} +#endif + +static PPSMC_Result ci_send_msg_to_smc_with_parameter(struct radeon_device *rdev, + PPSMC_Msg msg, u32 parameter) +{ + WREG32(SMC_MSG_ARG_0, parameter); + return ci_send_msg_to_smc(rdev, msg); +} + +static PPSMC_Result ci_send_msg_to_smc_return_parameter(struct radeon_device *rdev, + PPSMC_Msg msg, u32 *parameter) +{ + PPSMC_Result smc_result; + + smc_result = ci_send_msg_to_smc(rdev, msg); + + if ((smc_result == PPSMC_Result_OK) && parameter) + *parameter = RREG32(SMC_MSG_ARG_0); + + return smc_result; +} + +static int ci_dpm_force_state_sclk(struct radeon_device *rdev, u32 n) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (!pi->sclk_dpm_key_disabled) { + PPSMC_Result smc_result = + ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SCLKDPM_SetEnabledMask, 1 << n); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + return 0; +} + +static int ci_dpm_force_state_mclk(struct radeon_device *rdev, u32 n) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (!pi->mclk_dpm_key_disabled) { + PPSMC_Result smc_result = + ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_MCLKDPM_SetEnabledMask, 1 << n); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + return 0; +} + +static int ci_dpm_force_state_pcie(struct radeon_device *rdev, u32 n) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (!pi->pcie_dpm_key_disabled) { + PPSMC_Result smc_result = + ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_PCIeDPM_ForceLevel, n); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + return 0; +} + +static int ci_set_power_limit(struct radeon_device *rdev, u32 n) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit) { + PPSMC_Result smc_result = + ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_PkgPwrSetLimit, n); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + + return 0; +} + +static int ci_set_overdrive_target_tdp(struct radeon_device *rdev, + u32 target_tdp) +{ + PPSMC_Result smc_result = + ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_OverDriveSetTargetTdp, target_tdp); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + return 0; +} + +#if 0 +static int ci_set_boot_state(struct radeon_device *rdev) +{ + return ci_enable_sclk_mclk_dpm(rdev, false); +} +#endif + +static u32 ci_get_average_sclk_freq(struct radeon_device *rdev) +{ + u32 sclk_freq; + PPSMC_Result smc_result = + ci_send_msg_to_smc_return_parameter(rdev, + PPSMC_MSG_API_GetSclkFrequency, + &sclk_freq); + if (smc_result != PPSMC_Result_OK) + sclk_freq = 0; + + return sclk_freq; +} + +static u32 ci_get_average_mclk_freq(struct radeon_device *rdev) +{ + u32 mclk_freq; + PPSMC_Result smc_result = + ci_send_msg_to_smc_return_parameter(rdev, + PPSMC_MSG_API_GetMclkFrequency, + &mclk_freq); + if (smc_result != PPSMC_Result_OK) + mclk_freq = 0; + + return mclk_freq; +} + +static void ci_dpm_start_smc(struct radeon_device *rdev) +{ + int i; + + ci_program_jump_on_start(rdev); + ci_start_smc_clock(rdev); + ci_start_smc(rdev); + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32_SMC(FIRMWARE_FLAGS) & INTERRUPTS_ENABLED) + break; + } +} + +static void ci_dpm_stop_smc(struct radeon_device *rdev) +{ + ci_reset_smc(rdev); + ci_stop_smc_clock(rdev); +} + +static int ci_process_firmware_header(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + int ret; + + ret = ci_read_smc_sram_dword(rdev, + SMU7_FIRMWARE_HEADER_LOCATION + + offsetof(SMU7_Firmware_Header, DpmTable), + &tmp, pi->sram_end); + if (ret) + return ret; + + pi->dpm_table_start = tmp; + + ret = ci_read_smc_sram_dword(rdev, + SMU7_FIRMWARE_HEADER_LOCATION + + offsetof(SMU7_Firmware_Header, SoftRegisters), + &tmp, pi->sram_end); + if (ret) + return ret; + + pi->soft_regs_start = tmp; + + ret = ci_read_smc_sram_dword(rdev, + SMU7_FIRMWARE_HEADER_LOCATION + + offsetof(SMU7_Firmware_Header, mcRegisterTable), + &tmp, pi->sram_end); + if (ret) + return ret; + + pi->mc_reg_table_start = tmp; + + ret = ci_read_smc_sram_dword(rdev, + SMU7_FIRMWARE_HEADER_LOCATION + + offsetof(SMU7_Firmware_Header, FanTable), + &tmp, pi->sram_end); + if (ret) + return ret; + + pi->fan_table_start = tmp; + + ret = ci_read_smc_sram_dword(rdev, + SMU7_FIRMWARE_HEADER_LOCATION + + offsetof(SMU7_Firmware_Header, mcArbDramTimingTable), + &tmp, pi->sram_end); + if (ret) + return ret; + + pi->arb_table_start = tmp; + + return 0; +} + +static void ci_read_clock_registers(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + pi->clock_registers.cg_spll_func_cntl = + RREG32_SMC(CG_SPLL_FUNC_CNTL); + pi->clock_registers.cg_spll_func_cntl_2 = + RREG32_SMC(CG_SPLL_FUNC_CNTL_2); + pi->clock_registers.cg_spll_func_cntl_3 = + RREG32_SMC(CG_SPLL_FUNC_CNTL_3); + pi->clock_registers.cg_spll_func_cntl_4 = + RREG32_SMC(CG_SPLL_FUNC_CNTL_4); + pi->clock_registers.cg_spll_spread_spectrum = + RREG32_SMC(CG_SPLL_SPREAD_SPECTRUM); + pi->clock_registers.cg_spll_spread_spectrum_2 = + RREG32_SMC(CG_SPLL_SPREAD_SPECTRUM_2); + pi->clock_registers.dll_cntl = RREG32(DLL_CNTL); + pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL); + pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL); + pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL); + pi->clock_registers.mpll_func_cntl = RREG32(MPLL_FUNC_CNTL); + pi->clock_registers.mpll_func_cntl_1 = RREG32(MPLL_FUNC_CNTL_1); + pi->clock_registers.mpll_func_cntl_2 = RREG32(MPLL_FUNC_CNTL_2); + pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1); + pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2); +} + +static void ci_init_sclk_t(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + pi->low_sclk_interrupt_t = 0; +} + +static void ci_enable_thermal_protection(struct radeon_device *rdev, + bool enable) +{ + u32 tmp = RREG32_SMC(GENERAL_PWRMGT); + + if (enable) + tmp &= ~THERMAL_PROTECTION_DIS; + else + tmp |= THERMAL_PROTECTION_DIS; + WREG32_SMC(GENERAL_PWRMGT, tmp); +} + +static void ci_enable_acpi_power_management(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(GENERAL_PWRMGT); + + tmp |= STATIC_PM_EN; + + WREG32_SMC(GENERAL_PWRMGT, tmp); +} + +#if 0 +static int ci_enter_ulp_state(struct radeon_device *rdev) +{ + + WREG32(SMC_MESSAGE_0, PPSMC_MSG_SwitchToMinimumPower); + + udelay(25000); + + return 0; +} + +static int ci_exit_ulp_state(struct radeon_device *rdev) +{ + int i; + + WREG32(SMC_MESSAGE_0, PPSMC_MSG_ResumeFromMinimumPower); + + udelay(7000); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(SMC_RESP_0) == 1) + break; + udelay(1000); + } + + return 0; +} +#endif + +static int ci_notify_smc_display_change(struct radeon_device *rdev, + bool has_display) +{ + PPSMC_Msg msg = has_display ? PPSMC_MSG_HasDisplay : PPSMC_MSG_NoDisplay; + + return (ci_send_msg_to_smc(rdev, msg) == PPSMC_Result_OK) ? 0 : -EINVAL; +} + +static int ci_enable_ds_master_switch(struct radeon_device *rdev, + bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (enable) { + if (pi->caps_sclk_ds) { + if (ci_send_msg_to_smc(rdev, PPSMC_MSG_MASTER_DeepSleep_ON) != PPSMC_Result_OK) + return -EINVAL; + } else { + if (ci_send_msg_to_smc(rdev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK) + return -EINVAL; + } + } else { + if (pi->caps_sclk_ds) { + if (ci_send_msg_to_smc(rdev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK) + return -EINVAL; + } + } + + return 0; +} + +static void ci_program_display_gap(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(CG_DISPLAY_GAP_CNTL); + u32 pre_vbi_time_in_us; + u32 frame_time_in_us; + u32 ref_clock = rdev->clock.spll.reference_freq; + u32 refresh_rate = r600_dpm_get_vrefresh(rdev); + u32 vblank_time = r600_dpm_get_vblank_time(rdev); + + tmp &= ~DISP_GAP_MASK; + if (rdev->pm.dpm.new_active_crtc_count > 0) + tmp |= DISP_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM); + else + tmp |= DISP_GAP(R600_PM_DISPLAY_GAP_IGNORE); + WREG32_SMC(CG_DISPLAY_GAP_CNTL, tmp); + + if (refresh_rate == 0) + refresh_rate = 60; + if (vblank_time == 0xffffffff) + vblank_time = 500; + frame_time_in_us = 1000000 / refresh_rate; + pre_vbi_time_in_us = + frame_time_in_us - 200 - vblank_time; + tmp = pre_vbi_time_in_us * (ref_clock / 100); + + WREG32_SMC(CG_DISPLAY_GAP_CNTL2, tmp); + ci_write_smc_soft_register(rdev, offsetof(SMU7_SoftRegisters, PreVBlankGap), 0x64); + ci_write_smc_soft_register(rdev, offsetof(SMU7_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us)); + + + ci_notify_smc_display_change(rdev, (rdev->pm.dpm.new_active_crtc_count == 1)); + +} + +static void ci_enable_spread_spectrum(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + + if (enable) { + if (pi->caps_sclk_ss_support) { + tmp = RREG32_SMC(GENERAL_PWRMGT); + tmp |= DYN_SPREAD_SPECTRUM_EN; + WREG32_SMC(GENERAL_PWRMGT, tmp); + } + } else { + tmp = RREG32_SMC(CG_SPLL_SPREAD_SPECTRUM); + tmp &= ~SSEN; + WREG32_SMC(CG_SPLL_SPREAD_SPECTRUM, tmp); + + tmp = RREG32_SMC(GENERAL_PWRMGT); + tmp &= ~DYN_SPREAD_SPECTRUM_EN; + WREG32_SMC(GENERAL_PWRMGT, tmp); + } +} + +static void ci_program_sstp(struct radeon_device *rdev) +{ + WREG32_SMC(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT))); +} + +static void ci_enable_display_gap(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(CG_DISPLAY_GAP_CNTL); + + tmp &= ~(DISP_GAP_MASK | DISP_GAP_MCHG_MASK); + tmp |= (DISP_GAP(R600_PM_DISPLAY_GAP_IGNORE) | + DISP_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK)); + + WREG32_SMC(CG_DISPLAY_GAP_CNTL, tmp); +} + +static void ci_program_vc(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32_SMC(SCLK_PWRMGT_CNTL); + tmp &= ~(RESET_SCLK_CNT | RESET_BUSY_CNT); + WREG32_SMC(SCLK_PWRMGT_CNTL, tmp); + + WREG32_SMC(CG_FTV_0, CISLANDS_VRC_DFLT0); + WREG32_SMC(CG_FTV_1, CISLANDS_VRC_DFLT1); + WREG32_SMC(CG_FTV_2, CISLANDS_VRC_DFLT2); + WREG32_SMC(CG_FTV_3, CISLANDS_VRC_DFLT3); + WREG32_SMC(CG_FTV_4, CISLANDS_VRC_DFLT4); + WREG32_SMC(CG_FTV_5, CISLANDS_VRC_DFLT5); + WREG32_SMC(CG_FTV_6, CISLANDS_VRC_DFLT6); + WREG32_SMC(CG_FTV_7, CISLANDS_VRC_DFLT7); +} + +static void ci_clear_vc(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32_SMC(SCLK_PWRMGT_CNTL); + tmp |= (RESET_SCLK_CNT | RESET_BUSY_CNT); + WREG32_SMC(SCLK_PWRMGT_CNTL, tmp); + + WREG32_SMC(CG_FTV_0, 0); + WREG32_SMC(CG_FTV_1, 0); + WREG32_SMC(CG_FTV_2, 0); + WREG32_SMC(CG_FTV_3, 0); + WREG32_SMC(CG_FTV_4, 0); + WREG32_SMC(CG_FTV_5, 0); + WREG32_SMC(CG_FTV_6, 0); + WREG32_SMC(CG_FTV_7, 0); +} + +static int ci_upload_firmware(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + int i, ret; + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32_SMC(RCU_UC_EVENTS) & BOOT_SEQ_DONE) + break; + } + WREG32_SMC(SMC_SYSCON_MISC_CNTL, 1); + + ci_stop_smc_clock(rdev); + ci_reset_smc(rdev); + + ret = ci_load_smc_ucode(rdev, pi->sram_end); + + return ret; + +} + +static int ci_get_svi2_voltage_table(struct radeon_device *rdev, + struct radeon_clock_voltage_dependency_table *voltage_dependency_table, + struct atom_voltage_table *voltage_table) +{ + u32 i; + + if (voltage_dependency_table == NULL) + return -EINVAL; + + voltage_table->mask_low = 0; + voltage_table->phase_delay = 0; + + voltage_table->count = voltage_dependency_table->count; + for (i = 0; i < voltage_table->count; i++) { + voltage_table->entries[i].value = voltage_dependency_table->entries[i].v; + voltage_table->entries[i].smio_low = 0; + } + + return 0; +} + +static int ci_construct_voltage_tables(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + int ret; + + if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) { + ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDC, + VOLTAGE_OBJ_GPIO_LUT, + &pi->vddc_voltage_table); + if (ret) + return ret; + } else if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) { + ret = ci_get_svi2_voltage_table(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + &pi->vddc_voltage_table); + if (ret) + return ret; + } + + if (pi->vddc_voltage_table.count > SMU7_MAX_LEVELS_VDDC) + si_trim_voltage_table_to_fit_state_table(rdev, SMU7_MAX_LEVELS_VDDC, + &pi->vddc_voltage_table); + + if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) { + ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDCI, + VOLTAGE_OBJ_GPIO_LUT, + &pi->vddci_voltage_table); + if (ret) + return ret; + } else if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) { + ret = ci_get_svi2_voltage_table(rdev, + &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + &pi->vddci_voltage_table); + if (ret) + return ret; + } + + if (pi->vddci_voltage_table.count > SMU7_MAX_LEVELS_VDDCI) + si_trim_voltage_table_to_fit_state_table(rdev, SMU7_MAX_LEVELS_VDDCI, + &pi->vddci_voltage_table); + + if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) { + ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_MVDDC, + VOLTAGE_OBJ_GPIO_LUT, + &pi->mvdd_voltage_table); + if (ret) + return ret; + } else if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) { + ret = ci_get_svi2_voltage_table(rdev, + &rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk, + &pi->mvdd_voltage_table); + if (ret) + return ret; + } + + if (pi->mvdd_voltage_table.count > SMU7_MAX_LEVELS_MVDD) + si_trim_voltage_table_to_fit_state_table(rdev, SMU7_MAX_LEVELS_MVDD, + &pi->mvdd_voltage_table); + + return 0; +} + +static void ci_populate_smc_voltage_table(struct radeon_device *rdev, + struct atom_voltage_table_entry *voltage_table, + SMU7_Discrete_VoltageLevel *smc_voltage_table) +{ + int ret; + + ret = ci_get_std_voltage_value_sidd(rdev, voltage_table, + &smc_voltage_table->StdVoltageHiSidd, + &smc_voltage_table->StdVoltageLoSidd); + + if (ret) { + smc_voltage_table->StdVoltageHiSidd = voltage_table->value * VOLTAGE_SCALE; + smc_voltage_table->StdVoltageLoSidd = voltage_table->value * VOLTAGE_SCALE; + } + + smc_voltage_table->Voltage = cpu_to_be16(voltage_table->value * VOLTAGE_SCALE); + smc_voltage_table->StdVoltageHiSidd = + cpu_to_be16(smc_voltage_table->StdVoltageHiSidd); + smc_voltage_table->StdVoltageLoSidd = + cpu_to_be16(smc_voltage_table->StdVoltageLoSidd); +} + +static int ci_populate_smc_vddc_table(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + unsigned int count; + + table->VddcLevelCount = pi->vddc_voltage_table.count; + for (count = 0; count < table->VddcLevelCount; count++) { + ci_populate_smc_voltage_table(rdev, + &pi->vddc_voltage_table.entries[count], + &table->VddcLevel[count]); + + if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) + table->VddcLevel[count].Smio |= + pi->vddc_voltage_table.entries[count].smio_low; + else + table->VddcLevel[count].Smio = 0; + } + table->VddcLevelCount = cpu_to_be32(table->VddcLevelCount); + + return 0; +} + +static int ci_populate_smc_vddci_table(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + unsigned int count; + struct ci_power_info *pi = ci_get_pi(rdev); + + table->VddciLevelCount = pi->vddci_voltage_table.count; + for (count = 0; count < table->VddciLevelCount; count++) { + ci_populate_smc_voltage_table(rdev, + &pi->vddci_voltage_table.entries[count], + &table->VddciLevel[count]); + + if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) + table->VddciLevel[count].Smio |= + pi->vddci_voltage_table.entries[count].smio_low; + else + table->VddciLevel[count].Smio = 0; + } + table->VddciLevelCount = cpu_to_be32(table->VddciLevelCount); + + return 0; +} + +static int ci_populate_smc_mvdd_table(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + unsigned int count; + + table->MvddLevelCount = pi->mvdd_voltage_table.count; + for (count = 0; count < table->MvddLevelCount; count++) { + ci_populate_smc_voltage_table(rdev, + &pi->mvdd_voltage_table.entries[count], + &table->MvddLevel[count]); + + if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) + table->MvddLevel[count].Smio |= + pi->mvdd_voltage_table.entries[count].smio_low; + else + table->MvddLevel[count].Smio = 0; + } + table->MvddLevelCount = cpu_to_be32(table->MvddLevelCount); + + return 0; +} + +static int ci_populate_smc_voltage_tables(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + int ret; + + ret = ci_populate_smc_vddc_table(rdev, table); + if (ret) + return ret; + + ret = ci_populate_smc_vddci_table(rdev, table); + if (ret) + return ret; + + ret = ci_populate_smc_mvdd_table(rdev, table); + if (ret) + return ret; + + return 0; +} + +static int ci_populate_mvdd_value(struct radeon_device *rdev, u32 mclk, + SMU7_Discrete_VoltageLevel *voltage) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 i = 0; + + if (pi->mvdd_control != CISLANDS_VOLTAGE_CONTROL_NONE) { + for (i = 0; i < rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count; i++) { + if (mclk <= rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries[i].clk) { + voltage->Voltage = pi->mvdd_voltage_table.entries[i].value; + break; + } + } + + if (i >= rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count) + return -EINVAL; + } + + return -EINVAL; +} + +static int ci_get_std_voltage_value_sidd(struct radeon_device *rdev, + struct atom_voltage_table_entry *voltage_table, + u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd) +{ + u16 v_index, idx; + bool voltage_found = false; + *std_voltage_hi_sidd = voltage_table->value * VOLTAGE_SCALE; + *std_voltage_lo_sidd = voltage_table->value * VOLTAGE_SCALE; + + if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries == NULL) + return -EINVAL; + + if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries) { + for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) { + if (voltage_table->value == + rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) { + voltage_found = true; + if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count) + idx = v_index; + else + idx = rdev->pm.dpm.dyn_state.cac_leakage_table.count - 1; + *std_voltage_lo_sidd = + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE; + *std_voltage_hi_sidd = + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE; + break; + } + } + + if (!voltage_found) { + for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) { + if (voltage_table->value <= + rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) { + voltage_found = true; + if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count) + idx = v_index; + else + idx = rdev->pm.dpm.dyn_state.cac_leakage_table.count - 1; + *std_voltage_lo_sidd = + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE; + *std_voltage_hi_sidd = + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE; + break; + } + } + } + } + + return 0; +} + +static void ci_populate_phase_value_based_on_sclk(struct radeon_device *rdev, + const struct radeon_phase_shedding_limits_table *limits, + u32 sclk, + u32 *phase_shedding) +{ + unsigned int i; + + *phase_shedding = 1; + + for (i = 0; i < limits->count; i++) { + if (sclk < limits->entries[i].sclk) { + *phase_shedding = i; + break; + } + } +} + +static void ci_populate_phase_value_based_on_mclk(struct radeon_device *rdev, + const struct radeon_phase_shedding_limits_table *limits, + u32 mclk, + u32 *phase_shedding) +{ + unsigned int i; + + *phase_shedding = 1; + + for (i = 0; i < limits->count; i++) { + if (mclk < limits->entries[i].mclk) { + *phase_shedding = i; + break; + } + } +} + +static int ci_init_arb_table_index(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + int ret; + + ret = ci_read_smc_sram_dword(rdev, pi->arb_table_start, + &tmp, pi->sram_end); + if (ret) + return ret; + + tmp &= 0x00FFFFFF; + tmp |= MC_CG_ARB_FREQ_F1 << 24; + + return ci_write_smc_sram_dword(rdev, pi->arb_table_start, + tmp, pi->sram_end); +} + +static int ci_get_dependency_volt_by_clk(struct radeon_device *rdev, + struct radeon_clock_voltage_dependency_table *allowed_clock_voltage_table, + u32 clock, u32 *voltage) +{ + u32 i = 0; + + if (allowed_clock_voltage_table->count == 0) + return -EINVAL; + + for (i = 0; i < allowed_clock_voltage_table->count; i++) { + if (allowed_clock_voltage_table->entries[i].clk >= clock) { + *voltage = allowed_clock_voltage_table->entries[i].v; + return 0; + } + } + + *voltage = allowed_clock_voltage_table->entries[i-1].v; + + return 0; +} + +static u8 ci_get_sleep_divider_id_from_clock(struct radeon_device *rdev, + u32 sclk, u32 min_sclk_in_sr) +{ + u32 i; + u32 tmp; + u32 min = (min_sclk_in_sr > CISLAND_MINIMUM_ENGINE_CLOCK) ? + min_sclk_in_sr : CISLAND_MINIMUM_ENGINE_CLOCK; + + if (sclk < min) + return 0; + + for (i = CISLAND_MAX_DEEPSLEEP_DIVIDER_ID; ; i--) { + tmp = sclk / (1 << i); + if (tmp >= min || i == 0) + break; + } + + return (u8)i; +} + +static int ci_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev) +{ + return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1); +} + +static int ci_reset_to_default(struct radeon_device *rdev) +{ + return (ci_send_msg_to_smc(rdev, PPSMC_MSG_ResetToDefaults) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +static int ci_force_switch_to_arb_f0(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = (RREG32_SMC(SMC_SCRATCH9) & 0x0000ff00) >> 8; + + if (tmp == MC_CG_ARB_FREQ_F0) + return 0; + + return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0); +} + +static void ci_register_patching_mc_arb(struct radeon_device *rdev, + const u32 engine_clock, + const u32 memory_clock, + u32 *dram_timimg2) +{ + bool patch; + u32 tmp, tmp2; + + tmp = RREG32(MC_SEQ_MISC0); + patch = ((tmp & 0x0000f00) == 0x300) ? true : false; + + if (patch && + ((rdev->pdev->device == 0x67B0) || + (rdev->pdev->device == 0x67B1))) { + if ((memory_clock > 100000) && (memory_clock <= 125000)) { + tmp2 = (((0x31 * engine_clock) / 125000) - 1) & 0xff; + *dram_timimg2 &= ~0x00ff0000; + *dram_timimg2 |= tmp2 << 16; + } else if ((memory_clock > 125000) && (memory_clock <= 137500)) { + tmp2 = (((0x36 * engine_clock) / 137500) - 1) & 0xff; + *dram_timimg2 &= ~0x00ff0000; + *dram_timimg2 |= tmp2 << 16; + } + } +} + + +static int ci_populate_memory_timing_parameters(struct radeon_device *rdev, + u32 sclk, + u32 mclk, + SMU7_Discrete_MCArbDramTimingTableEntry *arb_regs) +{ + u32 dram_timing; + u32 dram_timing2; + u32 burst_time; + + radeon_atom_set_engine_dram_timings(rdev, sclk, mclk); + + dram_timing = RREG32(MC_ARB_DRAM_TIMING); + dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2); + burst_time = RREG32(MC_ARB_BURST_TIME) & STATE0_MASK; + + ci_register_patching_mc_arb(rdev, sclk, mclk, &dram_timing2); + + arb_regs->McArbDramTiming = cpu_to_be32(dram_timing); + arb_regs->McArbDramTiming2 = cpu_to_be32(dram_timing2); + arb_regs->McArbBurstTime = (u8)burst_time; + + return 0; +} + +static int ci_do_program_memory_timing_parameters(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + SMU7_Discrete_MCArbDramTimingTable arb_regs; + u32 i, j; + int ret = 0; + + memset(&arb_regs, 0, sizeof(SMU7_Discrete_MCArbDramTimingTable)); + + for (i = 0; i < pi->dpm_table.sclk_table.count; i++) { + for (j = 0; j < pi->dpm_table.mclk_table.count; j++) { + ret = ci_populate_memory_timing_parameters(rdev, + pi->dpm_table.sclk_table.dpm_levels[i].value, + pi->dpm_table.mclk_table.dpm_levels[j].value, + &arb_regs.entries[i][j]); + if (ret) + break; + } + } + + if (ret == 0) + ret = ci_copy_bytes_to_smc(rdev, + pi->arb_table_start, + (u8 *)&arb_regs, + sizeof(SMU7_Discrete_MCArbDramTimingTable), + pi->sram_end); + + return ret; +} + +static int ci_program_memory_timing_parameters(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (pi->need_update_smu7_dpm_table == 0) + return 0; + + return ci_do_program_memory_timing_parameters(rdev); +} + +static void ci_populate_smc_initial_state(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state) +{ + struct ci_ps *boot_state = ci_get_ps(radeon_boot_state); + struct ci_power_info *pi = ci_get_pi(rdev); + u32 level = 0; + + for (level = 0; level < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; level++) { + if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[level].clk >= + boot_state->performance_levels[0].sclk) { + pi->smc_state_table.GraphicsBootLevel = level; + break; + } + } + + for (level = 0; level < rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.count; level++) { + if (rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries[level].clk >= + boot_state->performance_levels[0].mclk) { + pi->smc_state_table.MemoryBootLevel = level; + break; + } + } +} + +static u32 ci_get_dpm_level_enable_mask_value(struct ci_single_dpm_table *dpm_table) +{ + u32 i; + u32 mask_value = 0; + + for (i = dpm_table->count; i > 0; i--) { + mask_value = mask_value << 1; + if (dpm_table->dpm_levels[i-1].enabled) + mask_value |= 0x1; + else + mask_value &= 0xFFFFFFFE; + } + + return mask_value; +} + +static void ci_populate_smc_link_level(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_dpm_table *dpm_table = &pi->dpm_table; + u32 i; + + for (i = 0; i < dpm_table->pcie_speed_table.count; i++) { + table->LinkLevel[i].PcieGenSpeed = + (u8)dpm_table->pcie_speed_table.dpm_levels[i].value; + table->LinkLevel[i].PcieLaneCount = + r600_encode_pci_lane_width(dpm_table->pcie_speed_table.dpm_levels[i].param1); + table->LinkLevel[i].EnabledForActivity = 1; + table->LinkLevel[i].DownT = cpu_to_be32(5); + table->LinkLevel[i].UpT = cpu_to_be32(30); + } + + pi->smc_state_table.LinkLevelCount = (u8)dpm_table->pcie_speed_table.count; + pi->dpm_level_enable_mask.pcie_dpm_enable_mask = + ci_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table); +} + +static int ci_populate_smc_uvd_level(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + u32 count; + struct atom_clock_dividers dividers; + int ret = -EINVAL; + + table->UvdLevelCount = + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count; + + for (count = 0; count < table->UvdLevelCount; count++) { + table->UvdLevel[count].VclkFrequency = + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].vclk; + table->UvdLevel[count].DclkFrequency = + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].dclk; + table->UvdLevel[count].MinVddc = + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE; + table->UvdLevel[count].MinVddcPhases = 1; + + ret = radeon_atom_get_clock_dividers(rdev, + COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, + table->UvdLevel[count].VclkFrequency, false, ÷rs); + if (ret) + return ret; + + table->UvdLevel[count].VclkDivider = (u8)dividers.post_divider; + + ret = radeon_atom_get_clock_dividers(rdev, + COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, + table->UvdLevel[count].DclkFrequency, false, ÷rs); + if (ret) + return ret; + + table->UvdLevel[count].DclkDivider = (u8)dividers.post_divider; + + table->UvdLevel[count].VclkFrequency = cpu_to_be32(table->UvdLevel[count].VclkFrequency); + table->UvdLevel[count].DclkFrequency = cpu_to_be32(table->UvdLevel[count].DclkFrequency); + table->UvdLevel[count].MinVddc = cpu_to_be16(table->UvdLevel[count].MinVddc); + } + + return ret; +} + +static int ci_populate_smc_vce_level(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + u32 count; + struct atom_clock_dividers dividers; + int ret = -EINVAL; + + table->VceLevelCount = + rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count; + + for (count = 0; count < table->VceLevelCount; count++) { + table->VceLevel[count].Frequency = + rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].evclk; + table->VceLevel[count].MinVoltage = + (u16)rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE; + table->VceLevel[count].MinPhases = 1; + + ret = radeon_atom_get_clock_dividers(rdev, + COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, + table->VceLevel[count].Frequency, false, ÷rs); + if (ret) + return ret; + + table->VceLevel[count].Divider = (u8)dividers.post_divider; + + table->VceLevel[count].Frequency = cpu_to_be32(table->VceLevel[count].Frequency); + table->VceLevel[count].MinVoltage = cpu_to_be16(table->VceLevel[count].MinVoltage); + } + + return ret; + +} + +static int ci_populate_smc_acp_level(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + u32 count; + struct atom_clock_dividers dividers; + int ret = -EINVAL; + + table->AcpLevelCount = (u8) + (rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count); + + for (count = 0; count < table->AcpLevelCount; count++) { + table->AcpLevel[count].Frequency = + rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].clk; + table->AcpLevel[count].MinVoltage = + rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].v; + table->AcpLevel[count].MinPhases = 1; + + ret = radeon_atom_get_clock_dividers(rdev, + COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, + table->AcpLevel[count].Frequency, false, ÷rs); + if (ret) + return ret; + + table->AcpLevel[count].Divider = (u8)dividers.post_divider; + + table->AcpLevel[count].Frequency = cpu_to_be32(table->AcpLevel[count].Frequency); + table->AcpLevel[count].MinVoltage = cpu_to_be16(table->AcpLevel[count].MinVoltage); + } + + return ret; +} + +static int ci_populate_smc_samu_level(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + u32 count; + struct atom_clock_dividers dividers; + int ret = -EINVAL; + + table->SamuLevelCount = + rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count; + + for (count = 0; count < table->SamuLevelCount; count++) { + table->SamuLevel[count].Frequency = + rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].clk; + table->SamuLevel[count].MinVoltage = + rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE; + table->SamuLevel[count].MinPhases = 1; + + ret = radeon_atom_get_clock_dividers(rdev, + COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, + table->SamuLevel[count].Frequency, false, ÷rs); + if (ret) + return ret; + + table->SamuLevel[count].Divider = (u8)dividers.post_divider; + + table->SamuLevel[count].Frequency = cpu_to_be32(table->SamuLevel[count].Frequency); + table->SamuLevel[count].MinVoltage = cpu_to_be16(table->SamuLevel[count].MinVoltage); + } + + return ret; +} + +static int ci_calculate_mclk_params(struct radeon_device *rdev, + u32 memory_clock, + SMU7_Discrete_MemoryLevel *mclk, + bool strobe_mode, + bool dll_state_on) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 dll_cntl = pi->clock_registers.dll_cntl; + u32 mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl; + u32 mpll_ad_func_cntl = pi->clock_registers.mpll_ad_func_cntl; + u32 mpll_dq_func_cntl = pi->clock_registers.mpll_dq_func_cntl; + u32 mpll_func_cntl = pi->clock_registers.mpll_func_cntl; + u32 mpll_func_cntl_1 = pi->clock_registers.mpll_func_cntl_1; + u32 mpll_func_cntl_2 = pi->clock_registers.mpll_func_cntl_2; + u32 mpll_ss1 = pi->clock_registers.mpll_ss1; + u32 mpll_ss2 = pi->clock_registers.mpll_ss2; + struct atom_mpll_param mpll_param; + int ret; + + ret = radeon_atom_get_memory_pll_dividers(rdev, memory_clock, strobe_mode, &mpll_param); + if (ret) + return ret; + + mpll_func_cntl &= ~BWCTRL_MASK; + mpll_func_cntl |= BWCTRL(mpll_param.bwcntl); + + mpll_func_cntl_1 &= ~(CLKF_MASK | CLKFRAC_MASK | VCO_MODE_MASK); + mpll_func_cntl_1 |= CLKF(mpll_param.clkf) | + CLKFRAC(mpll_param.clkfrac) | VCO_MODE(mpll_param.vco_mode); + + mpll_ad_func_cntl &= ~YCLK_POST_DIV_MASK; + mpll_ad_func_cntl |= YCLK_POST_DIV(mpll_param.post_div); + + if (pi->mem_gddr5) { + mpll_dq_func_cntl &= ~(YCLK_SEL_MASK | YCLK_POST_DIV_MASK); + mpll_dq_func_cntl |= YCLK_SEL(mpll_param.yclk_sel) | + YCLK_POST_DIV(mpll_param.post_div); + } + + if (pi->caps_mclk_ss_support) { + struct radeon_atom_ss ss; + u32 freq_nom; + u32 tmp; + u32 reference_clock = rdev->clock.mpll.reference_freq; + + if (mpll_param.qdr == 1) + freq_nom = memory_clock * 4 * (1 << mpll_param.post_div); + else + freq_nom = memory_clock * 2 * (1 << mpll_param.post_div); + + tmp = (freq_nom / reference_clock); + tmp = tmp * tmp; + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_MEMORY_SS, freq_nom)) { + u32 clks = reference_clock * 5 / ss.rate; + u32 clkv = (u32)((((131 * ss.percentage * ss.rate) / 100) * tmp) / freq_nom); + + mpll_ss1 &= ~CLKV_MASK; + mpll_ss1 |= CLKV(clkv); + + mpll_ss2 &= ~CLKS_MASK; + mpll_ss2 |= CLKS(clks); + } + } + + mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK; + mclk_pwrmgt_cntl |= DLL_SPEED(mpll_param.dll_speed); + + if (dll_state_on) + mclk_pwrmgt_cntl |= MRDCK0_PDNB | MRDCK1_PDNB; + else + mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB); + + mclk->MclkFrequency = memory_clock; + mclk->MpllFuncCntl = mpll_func_cntl; + mclk->MpllFuncCntl_1 = mpll_func_cntl_1; + mclk->MpllFuncCntl_2 = mpll_func_cntl_2; + mclk->MpllAdFuncCntl = mpll_ad_func_cntl; + mclk->MpllDqFuncCntl = mpll_dq_func_cntl; + mclk->MclkPwrmgtCntl = mclk_pwrmgt_cntl; + mclk->DllCntl = dll_cntl; + mclk->MpllSs1 = mpll_ss1; + mclk->MpllSs2 = mpll_ss2; + + return 0; +} + +static int ci_populate_single_memory_level(struct radeon_device *rdev, + u32 memory_clock, + SMU7_Discrete_MemoryLevel *memory_level) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + int ret; + bool dll_state_on; + + if (rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries) { + ret = ci_get_dependency_volt_by_clk(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + memory_clock, &memory_level->MinVddc); + if (ret) + return ret; + } + + if (rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries) { + ret = ci_get_dependency_volt_by_clk(rdev, + &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + memory_clock, &memory_level->MinVddci); + if (ret) + return ret; + } + + if (rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries) { + ret = ci_get_dependency_volt_by_clk(rdev, + &rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk, + memory_clock, &memory_level->MinMvdd); + if (ret) + return ret; + } + + memory_level->MinVddcPhases = 1; + + if (pi->vddc_phase_shed_control) + ci_populate_phase_value_based_on_mclk(rdev, + &rdev->pm.dpm.dyn_state.phase_shedding_limits_table, + memory_clock, + &memory_level->MinVddcPhases); + + memory_level->EnabledForThrottle = 1; + memory_level->UpH = 0; + memory_level->DownH = 100; + memory_level->VoltageDownH = 0; + memory_level->ActivityLevel = (u16)pi->mclk_activity_target; + + memory_level->StutterEnable = false; + memory_level->StrobeEnable = false; + memory_level->EdcReadEnable = false; + memory_level->EdcWriteEnable = false; + memory_level->RttEnable = false; + + memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW; + + if (pi->mclk_stutter_mode_threshold && + (memory_clock <= pi->mclk_stutter_mode_threshold) && + (pi->uvd_enabled == false) && + (RREG32(DPG_PIPE_STUTTER_CONTROL) & STUTTER_ENABLE) && + (rdev->pm.dpm.new_active_crtc_count <= 2)) + memory_level->StutterEnable = true; + + if (pi->mclk_strobe_mode_threshold && + (memory_clock <= pi->mclk_strobe_mode_threshold)) + memory_level->StrobeEnable = 1; + + if (pi->mem_gddr5) { + memory_level->StrobeRatio = + si_get_mclk_frequency_ratio(memory_clock, memory_level->StrobeEnable); + if (pi->mclk_edc_enable_threshold && + (memory_clock > pi->mclk_edc_enable_threshold)) + memory_level->EdcReadEnable = true; + + if (pi->mclk_edc_wr_enable_threshold && + (memory_clock > pi->mclk_edc_wr_enable_threshold)) + memory_level->EdcWriteEnable = true; + + if (memory_level->StrobeEnable) { + if (si_get_mclk_frequency_ratio(memory_clock, true) >= + ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf)) + dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false; + else + dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false; + } else { + dll_state_on = pi->dll_default_on; + } + } else { + memory_level->StrobeRatio = si_get_ddr3_mclk_frequency_ratio(memory_clock); + dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false; + } + + ret = ci_calculate_mclk_params(rdev, memory_clock, memory_level, memory_level->StrobeEnable, dll_state_on); + if (ret) + return ret; + + memory_level->MinVddc = cpu_to_be32(memory_level->MinVddc * VOLTAGE_SCALE); + memory_level->MinVddcPhases = cpu_to_be32(memory_level->MinVddcPhases); + memory_level->MinVddci = cpu_to_be32(memory_level->MinVddci * VOLTAGE_SCALE); + memory_level->MinMvdd = cpu_to_be32(memory_level->MinMvdd * VOLTAGE_SCALE); + + memory_level->MclkFrequency = cpu_to_be32(memory_level->MclkFrequency); + memory_level->ActivityLevel = cpu_to_be16(memory_level->ActivityLevel); + memory_level->MpllFuncCntl = cpu_to_be32(memory_level->MpllFuncCntl); + memory_level->MpllFuncCntl_1 = cpu_to_be32(memory_level->MpllFuncCntl_1); + memory_level->MpllFuncCntl_2 = cpu_to_be32(memory_level->MpllFuncCntl_2); + memory_level->MpllAdFuncCntl = cpu_to_be32(memory_level->MpllAdFuncCntl); + memory_level->MpllDqFuncCntl = cpu_to_be32(memory_level->MpllDqFuncCntl); + memory_level->MclkPwrmgtCntl = cpu_to_be32(memory_level->MclkPwrmgtCntl); + memory_level->DllCntl = cpu_to_be32(memory_level->DllCntl); + memory_level->MpllSs1 = cpu_to_be32(memory_level->MpllSs1); + memory_level->MpllSs2 = cpu_to_be32(memory_level->MpllSs2); + + return 0; +} + +static int ci_populate_smc_acpi_level(struct radeon_device *rdev, + SMU7_Discrete_DpmTable *table) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct atom_clock_dividers dividers; + SMU7_Discrete_VoltageLevel voltage_level; + u32 spll_func_cntl = pi->clock_registers.cg_spll_func_cntl; + u32 spll_func_cntl_2 = pi->clock_registers.cg_spll_func_cntl_2; + u32 dll_cntl = pi->clock_registers.dll_cntl; + u32 mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl; + int ret; + + table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC; + + if (pi->acpi_vddc) + table->ACPILevel.MinVddc = cpu_to_be32(pi->acpi_vddc * VOLTAGE_SCALE); + else + table->ACPILevel.MinVddc = cpu_to_be32(pi->min_vddc_in_pp_table * VOLTAGE_SCALE); + + table->ACPILevel.MinVddcPhases = pi->vddc_phase_shed_control ? 0 : 1; + + table->ACPILevel.SclkFrequency = rdev->clock.spll.reference_freq; + + ret = radeon_atom_get_clock_dividers(rdev, + COMPUTE_GPUCLK_INPUT_FLAG_SCLK, + table->ACPILevel.SclkFrequency, false, ÷rs); + if (ret) + return ret; + + table->ACPILevel.SclkDid = (u8)dividers.post_divider; + table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW; + table->ACPILevel.DeepSleepDivId = 0; + + spll_func_cntl &= ~SPLL_PWRON; + spll_func_cntl |= SPLL_RESET; + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(4); + + table->ACPILevel.CgSpllFuncCntl = spll_func_cntl; + table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2; + table->ACPILevel.CgSpllFuncCntl3 = pi->clock_registers.cg_spll_func_cntl_3; + table->ACPILevel.CgSpllFuncCntl4 = pi->clock_registers.cg_spll_func_cntl_4; + table->ACPILevel.SpllSpreadSpectrum = pi->clock_registers.cg_spll_spread_spectrum; + table->ACPILevel.SpllSpreadSpectrum2 = pi->clock_registers.cg_spll_spread_spectrum_2; + table->ACPILevel.CcPwrDynRm = 0; + table->ACPILevel.CcPwrDynRm1 = 0; + + table->ACPILevel.Flags = cpu_to_be32(table->ACPILevel.Flags); + table->ACPILevel.MinVddcPhases = cpu_to_be32(table->ACPILevel.MinVddcPhases); + table->ACPILevel.SclkFrequency = cpu_to_be32(table->ACPILevel.SclkFrequency); + table->ACPILevel.CgSpllFuncCntl = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl); + table->ACPILevel.CgSpllFuncCntl2 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl2); + table->ACPILevel.CgSpllFuncCntl3 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl3); + table->ACPILevel.CgSpllFuncCntl4 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl4); + table->ACPILevel.SpllSpreadSpectrum = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum); + table->ACPILevel.SpllSpreadSpectrum2 = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum2); + table->ACPILevel.CcPwrDynRm = cpu_to_be32(table->ACPILevel.CcPwrDynRm); + table->ACPILevel.CcPwrDynRm1 = cpu_to_be32(table->ACPILevel.CcPwrDynRm1); + + table->MemoryACPILevel.MinVddc = table->ACPILevel.MinVddc; + table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases; + + if (pi->vddci_control != CISLANDS_VOLTAGE_CONTROL_NONE) { + if (pi->acpi_vddci) + table->MemoryACPILevel.MinVddci = + cpu_to_be32(pi->acpi_vddci * VOLTAGE_SCALE); + else + table->MemoryACPILevel.MinVddci = + cpu_to_be32(pi->min_vddci_in_pp_table * VOLTAGE_SCALE); + } + + if (ci_populate_mvdd_value(rdev, 0, &voltage_level)) + table->MemoryACPILevel.MinMvdd = 0; + else + table->MemoryACPILevel.MinMvdd = + cpu_to_be32(voltage_level.Voltage * VOLTAGE_SCALE); + + mclk_pwrmgt_cntl |= MRDCK0_RESET | MRDCK1_RESET; + mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB); + + dll_cntl &= ~(MRDCK0_BYPASS | MRDCK1_BYPASS); + + table->MemoryACPILevel.DllCntl = cpu_to_be32(dll_cntl); + table->MemoryACPILevel.MclkPwrmgtCntl = cpu_to_be32(mclk_pwrmgt_cntl); + table->MemoryACPILevel.MpllAdFuncCntl = + cpu_to_be32(pi->clock_registers.mpll_ad_func_cntl); + table->MemoryACPILevel.MpllDqFuncCntl = + cpu_to_be32(pi->clock_registers.mpll_dq_func_cntl); + table->MemoryACPILevel.MpllFuncCntl = + cpu_to_be32(pi->clock_registers.mpll_func_cntl); + table->MemoryACPILevel.MpllFuncCntl_1 = + cpu_to_be32(pi->clock_registers.mpll_func_cntl_1); + table->MemoryACPILevel.MpllFuncCntl_2 = + cpu_to_be32(pi->clock_registers.mpll_func_cntl_2); + table->MemoryACPILevel.MpllSs1 = cpu_to_be32(pi->clock_registers.mpll_ss1); + table->MemoryACPILevel.MpllSs2 = cpu_to_be32(pi->clock_registers.mpll_ss2); + + table->MemoryACPILevel.EnabledForThrottle = 0; + table->MemoryACPILevel.EnabledForActivity = 0; + table->MemoryACPILevel.UpH = 0; + table->MemoryACPILevel.DownH = 100; + table->MemoryACPILevel.VoltageDownH = 0; + table->MemoryACPILevel.ActivityLevel = + cpu_to_be16((u16)pi->mclk_activity_target); + + table->MemoryACPILevel.StutterEnable = false; + table->MemoryACPILevel.StrobeEnable = false; + table->MemoryACPILevel.EdcReadEnable = false; + table->MemoryACPILevel.EdcWriteEnable = false; + table->MemoryACPILevel.RttEnable = false; + + return 0; +} + + +static int ci_enable_ulv(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_ulv_parm *ulv = &pi->ulv; + + if (ulv->supported) { + if (enable) + return (ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableULV) == PPSMC_Result_OK) ? + 0 : -EINVAL; + else + return (ci_send_msg_to_smc(rdev, PPSMC_MSG_DisableULV) == PPSMC_Result_OK) ? + 0 : -EINVAL; + } + + return 0; +} + +static int ci_populate_ulv_level(struct radeon_device *rdev, + SMU7_Discrete_Ulv *state) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u16 ulv_voltage = rdev->pm.dpm.backbias_response_time; + + state->CcPwrDynRm = 0; + state->CcPwrDynRm1 = 0; + + if (ulv_voltage == 0) { + pi->ulv.supported = false; + return 0; + } + + if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_BY_SVID2) { + if (ulv_voltage > rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v) + state->VddcOffset = 0; + else + state->VddcOffset = + rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage; + } else { + if (ulv_voltage > rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v) + state->VddcOffsetVid = 0; + else + state->VddcOffsetVid = (u8) + ((rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage) * + VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1); + } + state->VddcPhase = pi->vddc_phase_shed_control ? 0 : 1; + + state->CcPwrDynRm = cpu_to_be32(state->CcPwrDynRm); + state->CcPwrDynRm1 = cpu_to_be32(state->CcPwrDynRm1); + state->VddcOffset = cpu_to_be16(state->VddcOffset); + + return 0; +} + +static int ci_calculate_sclk_params(struct radeon_device *rdev, + u32 engine_clock, + SMU7_Discrete_GraphicsLevel *sclk) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct atom_clock_dividers dividers; + u32 spll_func_cntl_3 = pi->clock_registers.cg_spll_func_cntl_3; + u32 spll_func_cntl_4 = pi->clock_registers.cg_spll_func_cntl_4; + u32 cg_spll_spread_spectrum = pi->clock_registers.cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2 = pi->clock_registers.cg_spll_spread_spectrum_2; + u32 reference_clock = rdev->clock.spll.reference_freq; + u32 reference_divider; + u32 fbdiv; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, + COMPUTE_GPUCLK_INPUT_FLAG_SCLK, + engine_clock, false, ÷rs); + if (ret) + return ret; + + reference_divider = 1 + dividers.ref_div; + fbdiv = dividers.fb_div & 0x3FFFFFF; + + spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK; + spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv); + spll_func_cntl_3 |= SPLL_DITHEN; + + if (pi->caps_sclk_ss_support) { + struct radeon_atom_ss ss; + u32 vco_freq = engine_clock * dividers.post_div; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_ENGINE_SS, vco_freq)) { + u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate); + u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000); + + cg_spll_spread_spectrum &= ~CLK_S_MASK; + cg_spll_spread_spectrum |= CLK_S(clk_s); + cg_spll_spread_spectrum |= SSEN; + + cg_spll_spread_spectrum_2 &= ~CLK_V_MASK; + cg_spll_spread_spectrum_2 |= CLK_V(clk_v); + } + } + + sclk->SclkFrequency = engine_clock; + sclk->CgSpllFuncCntl3 = spll_func_cntl_3; + sclk->CgSpllFuncCntl4 = spll_func_cntl_4; + sclk->SpllSpreadSpectrum = cg_spll_spread_spectrum; + sclk->SpllSpreadSpectrum2 = cg_spll_spread_spectrum_2; + sclk->SclkDid = (u8)dividers.post_divider; + + return 0; +} + +static int ci_populate_single_graphic_level(struct radeon_device *rdev, + u32 engine_clock, + u16 sclk_activity_level_t, + SMU7_Discrete_GraphicsLevel *graphic_level) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + int ret; + + ret = ci_calculate_sclk_params(rdev, engine_clock, graphic_level); + if (ret) + return ret; + + ret = ci_get_dependency_volt_by_clk(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk, + engine_clock, &graphic_level->MinVddc); + if (ret) + return ret; + + graphic_level->SclkFrequency = engine_clock; + + graphic_level->Flags = 0; + graphic_level->MinVddcPhases = 1; + + if (pi->vddc_phase_shed_control) + ci_populate_phase_value_based_on_sclk(rdev, + &rdev->pm.dpm.dyn_state.phase_shedding_limits_table, + engine_clock, + &graphic_level->MinVddcPhases); + + graphic_level->ActivityLevel = sclk_activity_level_t; + + graphic_level->CcPwrDynRm = 0; + graphic_level->CcPwrDynRm1 = 0; + graphic_level->EnabledForThrottle = 1; + graphic_level->UpH = 0; + graphic_level->DownH = 0; + graphic_level->VoltageDownH = 0; + graphic_level->PowerThrottle = 0; + + if (pi->caps_sclk_ds) + graphic_level->DeepSleepDivId = ci_get_sleep_divider_id_from_clock(rdev, + engine_clock, + CISLAND_MINIMUM_ENGINE_CLOCK); + + graphic_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW; + + graphic_level->Flags = cpu_to_be32(graphic_level->Flags); + graphic_level->MinVddc = cpu_to_be32(graphic_level->MinVddc * VOLTAGE_SCALE); + graphic_level->MinVddcPhases = cpu_to_be32(graphic_level->MinVddcPhases); + graphic_level->SclkFrequency = cpu_to_be32(graphic_level->SclkFrequency); + graphic_level->ActivityLevel = cpu_to_be16(graphic_level->ActivityLevel); + graphic_level->CgSpllFuncCntl3 = cpu_to_be32(graphic_level->CgSpllFuncCntl3); + graphic_level->CgSpllFuncCntl4 = cpu_to_be32(graphic_level->CgSpllFuncCntl4); + graphic_level->SpllSpreadSpectrum = cpu_to_be32(graphic_level->SpllSpreadSpectrum); + graphic_level->SpllSpreadSpectrum2 = cpu_to_be32(graphic_level->SpllSpreadSpectrum2); + graphic_level->CcPwrDynRm = cpu_to_be32(graphic_level->CcPwrDynRm); + graphic_level->CcPwrDynRm1 = cpu_to_be32(graphic_level->CcPwrDynRm1); + + return 0; +} + +static int ci_populate_all_graphic_levels(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_dpm_table *dpm_table = &pi->dpm_table; + u32 level_array_address = pi->dpm_table_start + + offsetof(SMU7_Discrete_DpmTable, GraphicsLevel); + u32 level_array_size = sizeof(SMU7_Discrete_GraphicsLevel) * + SMU7_MAX_LEVELS_GRAPHICS; + SMU7_Discrete_GraphicsLevel *levels = pi->smc_state_table.GraphicsLevel; + u32 i, ret; + + memset(levels, 0, level_array_size); + + for (i = 0; i < dpm_table->sclk_table.count; i++) { + ret = ci_populate_single_graphic_level(rdev, + dpm_table->sclk_table.dpm_levels[i].value, + (u16)pi->activity_target[i], + &pi->smc_state_table.GraphicsLevel[i]); + if (ret) + return ret; + if (i > 1) + pi->smc_state_table.GraphicsLevel[i].DeepSleepDivId = 0; + if (i == (dpm_table->sclk_table.count - 1)) + pi->smc_state_table.GraphicsLevel[i].DisplayWatermark = + PPSMC_DISPLAY_WATERMARK_HIGH; + } + pi->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1; + + pi->smc_state_table.GraphicsDpmLevelCount = (u8)dpm_table->sclk_table.count; + pi->dpm_level_enable_mask.sclk_dpm_enable_mask = + ci_get_dpm_level_enable_mask_value(&dpm_table->sclk_table); + + ret = ci_copy_bytes_to_smc(rdev, level_array_address, + (u8 *)levels, level_array_size, + pi->sram_end); + if (ret) + return ret; + + return 0; +} + +static int ci_populate_ulv_state(struct radeon_device *rdev, + SMU7_Discrete_Ulv *ulv_level) +{ + return ci_populate_ulv_level(rdev, ulv_level); +} + +static int ci_populate_all_memory_levels(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_dpm_table *dpm_table = &pi->dpm_table; + u32 level_array_address = pi->dpm_table_start + + offsetof(SMU7_Discrete_DpmTable, MemoryLevel); + u32 level_array_size = sizeof(SMU7_Discrete_MemoryLevel) * + SMU7_MAX_LEVELS_MEMORY; + SMU7_Discrete_MemoryLevel *levels = pi->smc_state_table.MemoryLevel; + u32 i, ret; + + memset(levels, 0, level_array_size); + + for (i = 0; i < dpm_table->mclk_table.count; i++) { + if (dpm_table->mclk_table.dpm_levels[i].value == 0) + return -EINVAL; + ret = ci_populate_single_memory_level(rdev, + dpm_table->mclk_table.dpm_levels[i].value, + &pi->smc_state_table.MemoryLevel[i]); + if (ret) + return ret; + } + + pi->smc_state_table.MemoryLevel[0].EnabledForActivity = 1; + + if ((dpm_table->mclk_table.count >= 2) && + ((rdev->pdev->device == 0x67B0) || (rdev->pdev->device == 0x67B1))) { + pi->smc_state_table.MemoryLevel[1].MinVddc = + pi->smc_state_table.MemoryLevel[0].MinVddc; + pi->smc_state_table.MemoryLevel[1].MinVddcPhases = + pi->smc_state_table.MemoryLevel[0].MinVddcPhases; + } + + pi->smc_state_table.MemoryLevel[0].ActivityLevel = cpu_to_be16(0x1F); + + pi->smc_state_table.MemoryDpmLevelCount = (u8)dpm_table->mclk_table.count; + pi->dpm_level_enable_mask.mclk_dpm_enable_mask = + ci_get_dpm_level_enable_mask_value(&dpm_table->mclk_table); + + pi->smc_state_table.MemoryLevel[dpm_table->mclk_table.count - 1].DisplayWatermark = + PPSMC_DISPLAY_WATERMARK_HIGH; + + ret = ci_copy_bytes_to_smc(rdev, level_array_address, + (u8 *)levels, level_array_size, + pi->sram_end); + if (ret) + return ret; + + return 0; +} + +static void ci_reset_single_dpm_table(struct radeon_device *rdev, + struct ci_single_dpm_table* dpm_table, + u32 count) +{ + u32 i; + + dpm_table->count = count; + for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++) + dpm_table->dpm_levels[i].enabled = false; +} + +static void ci_setup_pcie_table_entry(struct ci_single_dpm_table* dpm_table, + u32 index, u32 pcie_gen, u32 pcie_lanes) +{ + dpm_table->dpm_levels[index].value = pcie_gen; + dpm_table->dpm_levels[index].param1 = pcie_lanes; + dpm_table->dpm_levels[index].enabled = true; +} + +static int ci_setup_default_pcie_tables(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (!pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels) + return -EINVAL; + + if (pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels) { + pi->pcie_gen_powersaving = pi->pcie_gen_performance; + pi->pcie_lane_powersaving = pi->pcie_lane_performance; + } else if (!pi->use_pcie_performance_levels && pi->use_pcie_powersaving_levels) { + pi->pcie_gen_performance = pi->pcie_gen_powersaving; + pi->pcie_lane_performance = pi->pcie_lane_powersaving; + } + + ci_reset_single_dpm_table(rdev, + &pi->dpm_table.pcie_speed_table, + SMU7_MAX_LEVELS_LINK); + + if (rdev->family == CHIP_BONAIRE) + ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0, + pi->pcie_gen_powersaving.min, + pi->pcie_lane_powersaving.max); + else + ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0, + pi->pcie_gen_powersaving.min, + pi->pcie_lane_powersaving.min); + ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 1, + pi->pcie_gen_performance.min, + pi->pcie_lane_performance.min); + ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 2, + pi->pcie_gen_powersaving.min, + pi->pcie_lane_powersaving.max); + ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 3, + pi->pcie_gen_performance.min, + pi->pcie_lane_performance.max); + ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 4, + pi->pcie_gen_powersaving.max, + pi->pcie_lane_powersaving.max); + ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 5, + pi->pcie_gen_performance.max, + pi->pcie_lane_performance.max); + + pi->dpm_table.pcie_speed_table.count = 6; + + return 0; +} + +static int ci_setup_default_dpm_tables(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_clock_voltage_dependency_table *allowed_sclk_vddc_table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + struct radeon_clock_voltage_dependency_table *allowed_mclk_table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk; + struct radeon_cac_leakage_table *std_voltage_table = + &rdev->pm.dpm.dyn_state.cac_leakage_table; + u32 i; + + if (allowed_sclk_vddc_table == NULL) + return -EINVAL; + if (allowed_sclk_vddc_table->count < 1) + return -EINVAL; + if (allowed_mclk_table == NULL) + return -EINVAL; + if (allowed_mclk_table->count < 1) + return -EINVAL; + + memset(&pi->dpm_table, 0, sizeof(struct ci_dpm_table)); + + ci_reset_single_dpm_table(rdev, + &pi->dpm_table.sclk_table, + SMU7_MAX_LEVELS_GRAPHICS); + ci_reset_single_dpm_table(rdev, + &pi->dpm_table.mclk_table, + SMU7_MAX_LEVELS_MEMORY); + ci_reset_single_dpm_table(rdev, + &pi->dpm_table.vddc_table, + SMU7_MAX_LEVELS_VDDC); + ci_reset_single_dpm_table(rdev, + &pi->dpm_table.vddci_table, + SMU7_MAX_LEVELS_VDDCI); + ci_reset_single_dpm_table(rdev, + &pi->dpm_table.mvdd_table, + SMU7_MAX_LEVELS_MVDD); + + pi->dpm_table.sclk_table.count = 0; + for (i = 0; i < allowed_sclk_vddc_table->count; i++) { + if ((i == 0) || + (pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count-1].value != + allowed_sclk_vddc_table->entries[i].clk)) { + pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].value = + allowed_sclk_vddc_table->entries[i].clk; + pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].enabled = + (i == 0) ? true : false; + pi->dpm_table.sclk_table.count++; + } + } + + pi->dpm_table.mclk_table.count = 0; + for (i = 0; i < allowed_mclk_table->count; i++) { + if ((i == 0) || + (pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count-1].value != + allowed_mclk_table->entries[i].clk)) { + pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].value = + allowed_mclk_table->entries[i].clk; + pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].enabled = + (i == 0) ? true : false; + pi->dpm_table.mclk_table.count++; + } + } + + for (i = 0; i < allowed_sclk_vddc_table->count; i++) { + pi->dpm_table.vddc_table.dpm_levels[i].value = + allowed_sclk_vddc_table->entries[i].v; + pi->dpm_table.vddc_table.dpm_levels[i].param1 = + std_voltage_table->entries[i].leakage; + pi->dpm_table.vddc_table.dpm_levels[i].enabled = true; + } + pi->dpm_table.vddc_table.count = allowed_sclk_vddc_table->count; + + allowed_mclk_table = &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk; + if (allowed_mclk_table) { + for (i = 0; i < allowed_mclk_table->count; i++) { + pi->dpm_table.vddci_table.dpm_levels[i].value = + allowed_mclk_table->entries[i].v; + pi->dpm_table.vddci_table.dpm_levels[i].enabled = true; + } + pi->dpm_table.vddci_table.count = allowed_mclk_table->count; + } + + allowed_mclk_table = &rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk; + if (allowed_mclk_table) { + for (i = 0; i < allowed_mclk_table->count; i++) { + pi->dpm_table.mvdd_table.dpm_levels[i].value = + allowed_mclk_table->entries[i].v; + pi->dpm_table.mvdd_table.dpm_levels[i].enabled = true; + } + pi->dpm_table.mvdd_table.count = allowed_mclk_table->count; + } + + ci_setup_default_pcie_tables(rdev); + + return 0; +} + +static int ci_find_boot_level(struct ci_single_dpm_table *table, + u32 value, u32 *boot_level) +{ + u32 i; + int ret = -EINVAL; + + for(i = 0; i < table->count; i++) { + if (value == table->dpm_levels[i].value) { + *boot_level = i; + ret = 0; + } + } + + return ret; +} + +static int ci_init_smc_table(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_ulv_parm *ulv = &pi->ulv; + struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps; + SMU7_Discrete_DpmTable *table = &pi->smc_state_table; + int ret; + + ret = ci_setup_default_dpm_tables(rdev); + if (ret) + return ret; + + if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE) + ci_populate_smc_voltage_tables(rdev, table); + + ci_init_fps_limits(rdev); + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) + table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) + table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC; + + if (pi->mem_gddr5) + table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5; + + if (ulv->supported) { + ret = ci_populate_ulv_state(rdev, &pi->smc_state_table.Ulv); + if (ret) + return ret; + WREG32_SMC(CG_ULV_PARAMETER, ulv->cg_ulv_parameter); + } + + ret = ci_populate_all_graphic_levels(rdev); + if (ret) + return ret; + + ret = ci_populate_all_memory_levels(rdev); + if (ret) + return ret; + + ci_populate_smc_link_level(rdev, table); + + ret = ci_populate_smc_acpi_level(rdev, table); + if (ret) + return ret; + + ret = ci_populate_smc_vce_level(rdev, table); + if (ret) + return ret; + + ret = ci_populate_smc_acp_level(rdev, table); + if (ret) + return ret; + + ret = ci_populate_smc_samu_level(rdev, table); + if (ret) + return ret; + + ret = ci_do_program_memory_timing_parameters(rdev); + if (ret) + return ret; + + ret = ci_populate_smc_uvd_level(rdev, table); + if (ret) + return ret; + + table->UvdBootLevel = 0; + table->VceBootLevel = 0; + table->AcpBootLevel = 0; + table->SamuBootLevel = 0; + table->GraphicsBootLevel = 0; + table->MemoryBootLevel = 0; + + ret = ci_find_boot_level(&pi->dpm_table.sclk_table, + pi->vbios_boot_state.sclk_bootup_value, + (u32 *)&pi->smc_state_table.GraphicsBootLevel); + + ret = ci_find_boot_level(&pi->dpm_table.mclk_table, + pi->vbios_boot_state.mclk_bootup_value, + (u32 *)&pi->smc_state_table.MemoryBootLevel); + + table->BootVddc = pi->vbios_boot_state.vddc_bootup_value; + table->BootVddci = pi->vbios_boot_state.vddci_bootup_value; + table->BootMVdd = pi->vbios_boot_state.mvdd_bootup_value; + + ci_populate_smc_initial_state(rdev, radeon_boot_state); + + ret = ci_populate_bapm_parameters_in_dpm_table(rdev); + if (ret) + return ret; + + table->UVDInterval = 1; + table->VCEInterval = 1; + table->ACPInterval = 1; + table->SAMUInterval = 1; + table->GraphicsVoltageChangeEnable = 1; + table->GraphicsThermThrottleEnable = 1; + table->GraphicsInterval = 1; + table->VoltageInterval = 1; + table->ThermalInterval = 1; + table->TemperatureLimitHigh = (u16)((pi->thermal_temp_setting.temperature_high * + CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000); + table->TemperatureLimitLow = (u16)((pi->thermal_temp_setting.temperature_low * + CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000); + table->MemoryVoltageChangeEnable = 1; + table->MemoryInterval = 1; + table->VoltageResponseTime = 0; + table->VddcVddciDelta = 4000; + table->PhaseResponseTime = 0; + table->MemoryThermThrottleEnable = 1; + table->PCIeBootLinkLevel = pi->dpm_table.pcie_speed_table.count - 1; + table->PCIeGenInterval = 1; + if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) + table->SVI2Enable = 1; + else + table->SVI2Enable = 0; + + table->ThermGpio = 17; + table->SclkStepSize = 0x4000; + + table->SystemFlags = cpu_to_be32(table->SystemFlags); + table->SmioMaskVddcVid = cpu_to_be32(table->SmioMaskVddcVid); + table->SmioMaskVddcPhase = cpu_to_be32(table->SmioMaskVddcPhase); + table->SmioMaskVddciVid = cpu_to_be32(table->SmioMaskVddciVid); + table->SmioMaskMvddVid = cpu_to_be32(table->SmioMaskMvddVid); + table->SclkStepSize = cpu_to_be32(table->SclkStepSize); + table->TemperatureLimitHigh = cpu_to_be16(table->TemperatureLimitHigh); + table->TemperatureLimitLow = cpu_to_be16(table->TemperatureLimitLow); + table->VddcVddciDelta = cpu_to_be16(table->VddcVddciDelta); + table->VoltageResponseTime = cpu_to_be16(table->VoltageResponseTime); + table->PhaseResponseTime = cpu_to_be16(table->PhaseResponseTime); + table->BootVddc = cpu_to_be16(table->BootVddc * VOLTAGE_SCALE); + table->BootVddci = cpu_to_be16(table->BootVddci * VOLTAGE_SCALE); + table->BootMVdd = cpu_to_be16(table->BootMVdd * VOLTAGE_SCALE); + + ret = ci_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Discrete_DpmTable, SystemFlags), + (u8 *)&table->SystemFlags, + sizeof(SMU7_Discrete_DpmTable) - 3 * sizeof(SMU7_PIDController), + pi->sram_end); + if (ret) + return ret; + + return 0; +} + +static void ci_trim_single_dpm_states(struct radeon_device *rdev, + struct ci_single_dpm_table *dpm_table, + u32 low_limit, u32 high_limit) +{ + u32 i; + + for (i = 0; i < dpm_table->count; i++) { + if ((dpm_table->dpm_levels[i].value < low_limit) || + (dpm_table->dpm_levels[i].value > high_limit)) + dpm_table->dpm_levels[i].enabled = false; + else + dpm_table->dpm_levels[i].enabled = true; + } +} + +static void ci_trim_pcie_dpm_states(struct radeon_device *rdev, + u32 speed_low, u32 lanes_low, + u32 speed_high, u32 lanes_high) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_single_dpm_table *pcie_table = &pi->dpm_table.pcie_speed_table; + u32 i, j; + + for (i = 0; i < pcie_table->count; i++) { + if ((pcie_table->dpm_levels[i].value < speed_low) || + (pcie_table->dpm_levels[i].param1 < lanes_low) || + (pcie_table->dpm_levels[i].value > speed_high) || + (pcie_table->dpm_levels[i].param1 > lanes_high)) + pcie_table->dpm_levels[i].enabled = false; + else + pcie_table->dpm_levels[i].enabled = true; + } + + for (i = 0; i < pcie_table->count; i++) { + if (pcie_table->dpm_levels[i].enabled) { + for (j = i + 1; j < pcie_table->count; j++) { + if (pcie_table->dpm_levels[j].enabled) { + if ((pcie_table->dpm_levels[i].value == pcie_table->dpm_levels[j].value) && + (pcie_table->dpm_levels[i].param1 == pcie_table->dpm_levels[j].param1)) + pcie_table->dpm_levels[j].enabled = false; + } + } + } + } +} + +static int ci_trim_dpm_states(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct ci_ps *state = ci_get_ps(radeon_state); + struct ci_power_info *pi = ci_get_pi(rdev); + u32 high_limit_count; + + if (state->performance_level_count < 1) + return -EINVAL; + + if (state->performance_level_count == 1) + high_limit_count = 0; + else + high_limit_count = 1; + + ci_trim_single_dpm_states(rdev, + &pi->dpm_table.sclk_table, + state->performance_levels[0].sclk, + state->performance_levels[high_limit_count].sclk); + + ci_trim_single_dpm_states(rdev, + &pi->dpm_table.mclk_table, + state->performance_levels[0].mclk, + state->performance_levels[high_limit_count].mclk); + + ci_trim_pcie_dpm_states(rdev, + state->performance_levels[0].pcie_gen, + state->performance_levels[0].pcie_lane, + state->performance_levels[high_limit_count].pcie_gen, + state->performance_levels[high_limit_count].pcie_lane); + + return 0; +} + +static int ci_apply_disp_minimum_voltage_request(struct radeon_device *rdev) +{ + struct radeon_clock_voltage_dependency_table *disp_voltage_table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk; + struct radeon_clock_voltage_dependency_table *vddc_table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + u32 requested_voltage = 0; + u32 i; + + if (disp_voltage_table == NULL) + return -EINVAL; + if (!disp_voltage_table->count) + return -EINVAL; + + for (i = 0; i < disp_voltage_table->count; i++) { + if (rdev->clock.current_dispclk == disp_voltage_table->entries[i].clk) + requested_voltage = disp_voltage_table->entries[i].v; + } + + for (i = 0; i < vddc_table->count; i++) { + if (requested_voltage <= vddc_table->entries[i].v) { + requested_voltage = vddc_table->entries[i].v; + return (ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_VddC_Request, + requested_voltage * VOLTAGE_SCALE) == PPSMC_Result_OK) ? + 0 : -EINVAL; + } + } + + return -EINVAL; +} + +static int ci_upload_dpm_level_enable_mask(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result result; + + ci_apply_disp_minimum_voltage_request(rdev); + + if (!pi->sclk_dpm_key_disabled) { + if (pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { + result = ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_SCLKDPM_SetEnabledMask, + pi->dpm_level_enable_mask.sclk_dpm_enable_mask); + if (result != PPSMC_Result_OK) + return -EINVAL; + } + } + + if (!pi->mclk_dpm_key_disabled) { + if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask) { + result = ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_MCLKDPM_SetEnabledMask, + pi->dpm_level_enable_mask.mclk_dpm_enable_mask); + if (result != PPSMC_Result_OK) + return -EINVAL; + } + } +#if 0 + if (!pi->pcie_dpm_key_disabled) { + if (pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { + result = ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_PCIeDPM_SetEnabledMask, + pi->dpm_level_enable_mask.pcie_dpm_enable_mask); + if (result != PPSMC_Result_OK) + return -EINVAL; + } + } +#endif + return 0; +} + +static void ci_find_dpm_states_clocks_in_dpm_table(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_ps *state = ci_get_ps(radeon_state); + struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table; + u32 sclk = state->performance_levels[state->performance_level_count-1].sclk; + struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table; + u32 mclk = state->performance_levels[state->performance_level_count-1].mclk; + u32 i; + + pi->need_update_smu7_dpm_table = 0; + + for (i = 0; i < sclk_table->count; i++) { + if (sclk == sclk_table->dpm_levels[i].value) + break; + } + + if (i >= sclk_table->count) { + pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK; + } else { + /* XXX check display min clock requirements */ + if (CISLAND_MINIMUM_ENGINE_CLOCK != CISLAND_MINIMUM_ENGINE_CLOCK) + pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK; + } + + for (i = 0; i < mclk_table->count; i++) { + if (mclk == mclk_table->dpm_levels[i].value) + break; + } + + if (i >= mclk_table->count) + pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK; + + if (rdev->pm.dpm.current_active_crtc_count != + rdev->pm.dpm.new_active_crtc_count) + pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK; +} + +static int ci_populate_and_upload_sclk_mclk_dpm_levels(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_ps *state = ci_get_ps(radeon_state); + u32 sclk = state->performance_levels[state->performance_level_count-1].sclk; + u32 mclk = state->performance_levels[state->performance_level_count-1].mclk; + struct ci_dpm_table *dpm_table = &pi->dpm_table; + int ret; + + if (!pi->need_update_smu7_dpm_table) + return 0; + + if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK) + dpm_table->sclk_table.dpm_levels[dpm_table->sclk_table.count-1].value = sclk; + + if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK) + dpm_table->mclk_table.dpm_levels[dpm_table->mclk_table.count-1].value = mclk; + + if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK)) { + ret = ci_populate_all_graphic_levels(rdev); + if (ret) + return ret; + } + + if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_MCLK | DPMTABLE_UPDATE_MCLK)) { + ret = ci_populate_all_memory_levels(rdev); + if (ret) + return ret; + } + + return 0; +} + +static int ci_enable_uvd_dpm(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + const struct radeon_clock_and_voltage_limits *max_limits; + int i; + + if (rdev->pm.dpm.ac_power) + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + else + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc; + + if (enable) { + pi->dpm_level_enable_mask.uvd_dpm_enable_mask = 0; + + for (i = rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1; i >= 0; i--) { + if (rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) { + pi->dpm_level_enable_mask.uvd_dpm_enable_mask |= 1 << i; + + if (!pi->caps_uvd_dpm) + break; + } + } + + ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_UVDDPM_SetEnabledMask, + pi->dpm_level_enable_mask.uvd_dpm_enable_mask); + + if (pi->last_mclk_dpm_enable_mask & 0x1) { + pi->uvd_enabled = true; + pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE; + ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_MCLKDPM_SetEnabledMask, + pi->dpm_level_enable_mask.mclk_dpm_enable_mask); + } + } else { + if (pi->last_mclk_dpm_enable_mask & 0x1) { + pi->uvd_enabled = false; + pi->dpm_level_enable_mask.mclk_dpm_enable_mask |= 1; + ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_MCLKDPM_SetEnabledMask, + pi->dpm_level_enable_mask.mclk_dpm_enable_mask); + } + } + + return (ci_send_msg_to_smc(rdev, enable ? + PPSMC_MSG_UVDDPM_Enable : PPSMC_MSG_UVDDPM_Disable) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +static int ci_enable_vce_dpm(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + const struct radeon_clock_and_voltage_limits *max_limits; + int i; + + if (rdev->pm.dpm.ac_power) + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + else + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc; + + if (enable) { + pi->dpm_level_enable_mask.vce_dpm_enable_mask = 0; + for (i = rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count - 1; i >= 0; i--) { + if (rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) { + pi->dpm_level_enable_mask.vce_dpm_enable_mask |= 1 << i; + + if (!pi->caps_vce_dpm) + break; + } + } + + ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_VCEDPM_SetEnabledMask, + pi->dpm_level_enable_mask.vce_dpm_enable_mask); + } + + return (ci_send_msg_to_smc(rdev, enable ? + PPSMC_MSG_VCEDPM_Enable : PPSMC_MSG_VCEDPM_Disable) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +#if 0 +static int ci_enable_samu_dpm(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + const struct radeon_clock_and_voltage_limits *max_limits; + int i; + + if (rdev->pm.dpm.ac_power) + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + else + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc; + + if (enable) { + pi->dpm_level_enable_mask.samu_dpm_enable_mask = 0; + for (i = rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count - 1; i >= 0; i--) { + if (rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) { + pi->dpm_level_enable_mask.samu_dpm_enable_mask |= 1 << i; + + if (!pi->caps_samu_dpm) + break; + } + } + + ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_SAMUDPM_SetEnabledMask, + pi->dpm_level_enable_mask.samu_dpm_enable_mask); + } + return (ci_send_msg_to_smc(rdev, enable ? + PPSMC_MSG_SAMUDPM_Enable : PPSMC_MSG_SAMUDPM_Disable) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +static int ci_enable_acp_dpm(struct radeon_device *rdev, bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + const struct radeon_clock_and_voltage_limits *max_limits; + int i; + + if (rdev->pm.dpm.ac_power) + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + else + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc; + + if (enable) { + pi->dpm_level_enable_mask.acp_dpm_enable_mask = 0; + for (i = rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count - 1; i >= 0; i--) { + if (rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) { + pi->dpm_level_enable_mask.acp_dpm_enable_mask |= 1 << i; + + if (!pi->caps_acp_dpm) + break; + } + } + + ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_ACPDPM_SetEnabledMask, + pi->dpm_level_enable_mask.acp_dpm_enable_mask); + } + + return (ci_send_msg_to_smc(rdev, enable ? + PPSMC_MSG_ACPDPM_Enable : PPSMC_MSG_ACPDPM_Disable) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} +#endif + +static int ci_update_uvd_dpm(struct radeon_device *rdev, bool gate) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + + if (!gate) { + if (pi->caps_uvd_dpm || + (rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count <= 0)) + pi->smc_state_table.UvdBootLevel = 0; + else + pi->smc_state_table.UvdBootLevel = + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1; + + tmp = RREG32_SMC(DPM_TABLE_475); + tmp &= ~UvdBootLevel_MASK; + tmp |= UvdBootLevel(pi->smc_state_table.UvdBootLevel); + WREG32_SMC(DPM_TABLE_475, tmp); + } + + return ci_enable_uvd_dpm(rdev, !gate); +} + +static u8 ci_get_vce_boot_level(struct radeon_device *rdev) +{ + u8 i; + u32 min_evclk = 30000; /* ??? */ + struct radeon_vce_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table; + + for (i = 0; i < table->count; i++) { + if (table->entries[i].evclk >= min_evclk) + return i; + } + + return table->count - 1; +} + +static int ci_update_vce_dpm(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + int ret = 0; + u32 tmp; + + if (radeon_current_state->evclk != radeon_new_state->evclk) { + if (radeon_new_state->evclk) { + /* turn the clocks on when encoding */ + cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, false); + + pi->smc_state_table.VceBootLevel = ci_get_vce_boot_level(rdev); + tmp = RREG32_SMC(DPM_TABLE_475); + tmp &= ~VceBootLevel_MASK; + tmp |= VceBootLevel(pi->smc_state_table.VceBootLevel); + WREG32_SMC(DPM_TABLE_475, tmp); + + ret = ci_enable_vce_dpm(rdev, true); + } else { + /* turn the clocks off when not encoding */ + cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, true); + + ret = ci_enable_vce_dpm(rdev, false); + } + } + return ret; +} + +#if 0 +static int ci_update_samu_dpm(struct radeon_device *rdev, bool gate) +{ + return ci_enable_samu_dpm(rdev, gate); +} + +static int ci_update_acp_dpm(struct radeon_device *rdev, bool gate) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + + if (!gate) { + pi->smc_state_table.AcpBootLevel = 0; + + tmp = RREG32_SMC(DPM_TABLE_475); + tmp &= ~AcpBootLevel_MASK; + tmp |= AcpBootLevel(pi->smc_state_table.AcpBootLevel); + WREG32_SMC(DPM_TABLE_475, tmp); + } + + return ci_enable_acp_dpm(rdev, !gate); +} +#endif + +static int ci_generate_dpm_level_enable_mask(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + int ret; + + ret = ci_trim_dpm_states(rdev, radeon_state); + if (ret) + return ret; + + pi->dpm_level_enable_mask.sclk_dpm_enable_mask = + ci_get_dpm_level_enable_mask_value(&pi->dpm_table.sclk_table); + pi->dpm_level_enable_mask.mclk_dpm_enable_mask = + ci_get_dpm_level_enable_mask_value(&pi->dpm_table.mclk_table); + pi->last_mclk_dpm_enable_mask = + pi->dpm_level_enable_mask.mclk_dpm_enable_mask; + if (pi->uvd_enabled) { + if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask & 1) + pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE; + } + pi->dpm_level_enable_mask.pcie_dpm_enable_mask = + ci_get_dpm_level_enable_mask_value(&pi->dpm_table.pcie_speed_table); + + return 0; +} + +static u32 ci_get_lowest_enabled_level(struct radeon_device *rdev, + u32 level_mask) +{ + u32 level = 0; + + while ((level_mask & (1 << level)) == 0) + level++; + + return level; +} + + +int ci_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp, levels, i; + int ret; + + if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + if ((!pi->pcie_dpm_key_disabled) && + pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { + levels = 0; + tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask; + while (tmp >>= 1) + levels++; + if (levels) { + ret = ci_dpm_force_state_pcie(rdev, level); + if (ret) + return ret; + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX_1) & + CURR_PCIE_INDEX_MASK) >> CURR_PCIE_INDEX_SHIFT; + if (tmp == levels) + break; + udelay(1); + } + } + } + if ((!pi->sclk_dpm_key_disabled) && + pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { + levels = 0; + tmp = pi->dpm_level_enable_mask.sclk_dpm_enable_mask; + while (tmp >>= 1) + levels++; + if (levels) { + ret = ci_dpm_force_state_sclk(rdev, levels); + if (ret) + return ret; + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) & + CURR_SCLK_INDEX_MASK) >> CURR_SCLK_INDEX_SHIFT; + if (tmp == levels) + break; + udelay(1); + } + } + } + if ((!pi->mclk_dpm_key_disabled) && + pi->dpm_level_enable_mask.mclk_dpm_enable_mask) { + levels = 0; + tmp = pi->dpm_level_enable_mask.mclk_dpm_enable_mask; + while (tmp >>= 1) + levels++; + if (levels) { + ret = ci_dpm_force_state_mclk(rdev, levels); + if (ret) + return ret; + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) & + CURR_MCLK_INDEX_MASK) >> CURR_MCLK_INDEX_SHIFT; + if (tmp == levels) + break; + udelay(1); + } + } + } + } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { + if ((!pi->sclk_dpm_key_disabled) && + pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { + levels = ci_get_lowest_enabled_level(rdev, + pi->dpm_level_enable_mask.sclk_dpm_enable_mask); + ret = ci_dpm_force_state_sclk(rdev, levels); + if (ret) + return ret; + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) & + CURR_SCLK_INDEX_MASK) >> CURR_SCLK_INDEX_SHIFT; + if (tmp == levels) + break; + udelay(1); + } + } + if ((!pi->mclk_dpm_key_disabled) && + pi->dpm_level_enable_mask.mclk_dpm_enable_mask) { + levels = ci_get_lowest_enabled_level(rdev, + pi->dpm_level_enable_mask.mclk_dpm_enable_mask); + ret = ci_dpm_force_state_mclk(rdev, levels); + if (ret) + return ret; + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) & + CURR_MCLK_INDEX_MASK) >> CURR_MCLK_INDEX_SHIFT; + if (tmp == levels) + break; + udelay(1); + } + } + if ((!pi->pcie_dpm_key_disabled) && + pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { + levels = ci_get_lowest_enabled_level(rdev, + pi->dpm_level_enable_mask.pcie_dpm_enable_mask); + ret = ci_dpm_force_state_pcie(rdev, levels); + if (ret) + return ret; + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX_1) & + CURR_PCIE_INDEX_MASK) >> CURR_PCIE_INDEX_SHIFT; + if (tmp == levels) + break; + udelay(1); + } + } + } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) { + if (!pi->pcie_dpm_key_disabled) { + PPSMC_Result smc_result; + + smc_result = ci_send_msg_to_smc(rdev, + PPSMC_MSG_PCIeDPM_UnForceLevel); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } + ret = ci_upload_dpm_level_enable_mask(rdev); + if (ret) + return ret; + } + + rdev->pm.dpm.forced_level = level; + + return 0; +} + +static int ci_set_mc_special_registers(struct radeon_device *rdev, + struct ci_mc_reg_table *table) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u8 i, j, k; + u32 temp_reg; + + for (i = 0, j = table->last; i < table->last; i++) { + if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + switch(table->mc_reg_address[i].s1 << 2) { + case MC_SEQ_MISC1: + temp_reg = RREG32(MC_PMG_CMD_EMRS); + table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2; + for (k = 0; k < table->num_entries; k++) { + table->mc_reg_table_entry[k].mc_data[j] = + ((temp_reg & 0xffff0000)) | ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16); + } + j++; + if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + + temp_reg = RREG32(MC_PMG_CMD_MRS); + table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2; + for (k = 0; k < table->num_entries; k++) { + table->mc_reg_table_entry[k].mc_data[j] = + (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); + if (!pi->mem_gddr5) + table->mc_reg_table_entry[k].mc_data[j] |= 0x100; + } + j++; + if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + + if (!pi->mem_gddr5) { + table->mc_reg_address[j].s1 = MC_PMG_AUTO_CMD >> 2; + table->mc_reg_address[j].s0 = MC_PMG_AUTO_CMD >> 2; + for (k = 0; k < table->num_entries; k++) { + table->mc_reg_table_entry[k].mc_data[j] = + (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16; + } + j++; + if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + } + break; + case MC_SEQ_RESERVE_M: + temp_reg = RREG32(MC_PMG_CMD_MRS1); + table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2; + for (k = 0; k < table->num_entries; k++) { + table->mc_reg_table_entry[k].mc_data[j] = + (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); + } + j++; + if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + break; + default: + break; + } + + } + + table->last = j; + + return 0; +} + +static bool ci_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg) +{ + bool result = true; + + switch(in_reg) { + case MC_SEQ_RAS_TIMING >> 2: + *out_reg = MC_SEQ_RAS_TIMING_LP >> 2; + break; + case MC_SEQ_DLL_STBY >> 2: + *out_reg = MC_SEQ_DLL_STBY_LP >> 2; + break; + case MC_SEQ_G5PDX_CMD0 >> 2: + *out_reg = MC_SEQ_G5PDX_CMD0_LP >> 2; + break; + case MC_SEQ_G5PDX_CMD1 >> 2: + *out_reg = MC_SEQ_G5PDX_CMD1_LP >> 2; + break; + case MC_SEQ_G5PDX_CTRL >> 2: + *out_reg = MC_SEQ_G5PDX_CTRL_LP >> 2; + break; + case MC_SEQ_CAS_TIMING >> 2: + *out_reg = MC_SEQ_CAS_TIMING_LP >> 2; + break; + case MC_SEQ_MISC_TIMING >> 2: + *out_reg = MC_SEQ_MISC_TIMING_LP >> 2; + break; + case MC_SEQ_MISC_TIMING2 >> 2: + *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2; + break; + case MC_SEQ_PMG_DVS_CMD >> 2: + *out_reg = MC_SEQ_PMG_DVS_CMD_LP >> 2; + break; + case MC_SEQ_PMG_DVS_CTL >> 2: + *out_reg = MC_SEQ_PMG_DVS_CTL_LP >> 2; + break; + case MC_SEQ_RD_CTL_D0 >> 2: + *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2; + break; + case MC_SEQ_RD_CTL_D1 >> 2: + *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2; + break; + case MC_SEQ_WR_CTL_D0 >> 2: + *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2; + break; + case MC_SEQ_WR_CTL_D1 >> 2: + *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2; + break; + case MC_PMG_CMD_EMRS >> 2: + *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2; + break; + case MC_PMG_CMD_MRS >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2; + break; + case MC_PMG_CMD_MRS1 >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2; + break; + case MC_SEQ_PMG_TIMING >> 2: + *out_reg = MC_SEQ_PMG_TIMING_LP >> 2; + break; + case MC_PMG_CMD_MRS2 >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2; + break; + case MC_SEQ_WR_CTL_2 >> 2: + *out_reg = MC_SEQ_WR_CTL_2_LP >> 2; + break; + default: + result = false; + break; + } + + return result; +} + +static void ci_set_valid_flag(struct ci_mc_reg_table *table) +{ + u8 i, j; + + for (i = 0; i < table->last; i++) { + for (j = 1; j < table->num_entries; j++) { + if (table->mc_reg_table_entry[j-1].mc_data[i] != + table->mc_reg_table_entry[j].mc_data[i]) { + table->valid_flag |= 1 << i; + break; + } + } + } +} + +static void ci_set_s0_mc_reg_index(struct ci_mc_reg_table *table) +{ + u32 i; + u16 address; + + for (i = 0; i < table->last; i++) { + table->mc_reg_address[i].s0 = + ci_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ? + address : table->mc_reg_address[i].s1; + } +} + +static int ci_copy_vbios_mc_reg_table(const struct atom_mc_reg_table *table, + struct ci_mc_reg_table *ci_table) +{ + u8 i, j; + + if (table->last > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + if (table->num_entries > MAX_AC_TIMING_ENTRIES) + return -EINVAL; + + for (i = 0; i < table->last; i++) + ci_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1; + + ci_table->last = table->last; + + for (i = 0; i < table->num_entries; i++) { + ci_table->mc_reg_table_entry[i].mclk_max = + table->mc_reg_table_entry[i].mclk_max; + for (j = 0; j < table->last; j++) + ci_table->mc_reg_table_entry[i].mc_data[j] = + table->mc_reg_table_entry[i].mc_data[j]; + } + ci_table->num_entries = table->num_entries; + + return 0; +} + +static int ci_register_patching_mc_seq(struct radeon_device *rdev, + struct ci_mc_reg_table *table) +{ + u8 i, k; + u32 tmp; + bool patch; + + tmp = RREG32(MC_SEQ_MISC0); + patch = ((tmp & 0x0000f00) == 0x300) ? true : false; + + if (patch && + ((rdev->pdev->device == 0x67B0) || + (rdev->pdev->device == 0x67B1))) { + for (i = 0; i < table->last; i++) { + if (table->last >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + switch(table->mc_reg_address[i].s1 >> 2) { + case MC_SEQ_MISC1: + for (k = 0; k < table->num_entries; k++) { + if ((table->mc_reg_table_entry[k].mclk_max == 125000) || + (table->mc_reg_table_entry[k].mclk_max == 137500)) + table->mc_reg_table_entry[k].mc_data[i] = + (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFF8) | + 0x00000007; + } + break; + case MC_SEQ_WR_CTL_D0: + for (k = 0; k < table->num_entries; k++) { + if ((table->mc_reg_table_entry[k].mclk_max == 125000) || + (table->mc_reg_table_entry[k].mclk_max == 137500)) + table->mc_reg_table_entry[k].mc_data[i] = + (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) | + 0x0000D0DD; + } + break; + case MC_SEQ_WR_CTL_D1: + for (k = 0; k < table->num_entries; k++) { + if ((table->mc_reg_table_entry[k].mclk_max == 125000) || + (table->mc_reg_table_entry[k].mclk_max == 137500)) + table->mc_reg_table_entry[k].mc_data[i] = + (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) | + 0x0000D0DD; + } + break; + case MC_SEQ_WR_CTL_2: + for (k = 0; k < table->num_entries; k++) { + if ((table->mc_reg_table_entry[k].mclk_max == 125000) || + (table->mc_reg_table_entry[k].mclk_max == 137500)) + table->mc_reg_table_entry[k].mc_data[i] = 0; + } + break; + case MC_SEQ_CAS_TIMING: + for (k = 0; k < table->num_entries; k++) { + if (table->mc_reg_table_entry[k].mclk_max == 125000) + table->mc_reg_table_entry[k].mc_data[i] = + (table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) | + 0x000C0140; + else if (table->mc_reg_table_entry[k].mclk_max == 137500) + table->mc_reg_table_entry[k].mc_data[i] = + (table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) | + 0x000C0150; + } + break; + case MC_SEQ_MISC_TIMING: + for (k = 0; k < table->num_entries; k++) { + if (table->mc_reg_table_entry[k].mclk_max == 125000) + table->mc_reg_table_entry[k].mc_data[i] = + (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) | + 0x00000030; + else if (table->mc_reg_table_entry[k].mclk_max == 137500) + table->mc_reg_table_entry[k].mc_data[i] = + (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) | + 0x00000035; + } + break; + default: + break; + } + } + + WREG32(MC_SEQ_IO_DEBUG_INDEX, 3); + tmp = RREG32(MC_SEQ_IO_DEBUG_DATA); + tmp = (tmp & 0xFFF8FFFF) | (1 << 16); + WREG32(MC_SEQ_IO_DEBUG_INDEX, 3); + WREG32(MC_SEQ_IO_DEBUG_DATA, tmp); + } + + return 0; +} + +static int ci_initialize_mc_reg_table(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct atom_mc_reg_table *table; + struct ci_mc_reg_table *ci_table = &pi->mc_reg_table; + u8 module_index = rv770_get_memory_module_index(rdev); + int ret; + + table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING)); + WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING)); + WREG32(MC_SEQ_DLL_STBY_LP, RREG32(MC_SEQ_DLL_STBY)); + WREG32(MC_SEQ_G5PDX_CMD0_LP, RREG32(MC_SEQ_G5PDX_CMD0)); + WREG32(MC_SEQ_G5PDX_CMD1_LP, RREG32(MC_SEQ_G5PDX_CMD1)); + WREG32(MC_SEQ_G5PDX_CTRL_LP, RREG32(MC_SEQ_G5PDX_CTRL)); + WREG32(MC_SEQ_PMG_DVS_CMD_LP, RREG32(MC_SEQ_PMG_DVS_CMD)); + WREG32(MC_SEQ_PMG_DVS_CTL_LP, RREG32(MC_SEQ_PMG_DVS_CTL)); + WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING)); + WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2)); + WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS)); + WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS)); + WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1)); + WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0)); + WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1)); + WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0)); + WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1)); + WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING)); + WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2)); + WREG32(MC_SEQ_WR_CTL_2_LP, RREG32(MC_SEQ_WR_CTL_2)); + + ret = radeon_atom_init_mc_reg_table(rdev, module_index, table); + if (ret) + goto init_mc_done; + + ret = ci_copy_vbios_mc_reg_table(table, ci_table); + if (ret) + goto init_mc_done; + + ci_set_s0_mc_reg_index(ci_table); + + ret = ci_register_patching_mc_seq(rdev, ci_table); + if (ret) + goto init_mc_done; + + ret = ci_set_mc_special_registers(rdev, ci_table); + if (ret) + goto init_mc_done; + + ci_set_valid_flag(ci_table); + +init_mc_done: + kfree(table); + + return ret; +} + +static int ci_populate_mc_reg_addresses(struct radeon_device *rdev, + SMU7_Discrete_MCRegisters *mc_reg_table) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 i, j; + + for (i = 0, j = 0; j < pi->mc_reg_table.last; j++) { + if (pi->mc_reg_table.valid_flag & (1 << j)) { + if (i >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + mc_reg_table->address[i].s0 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s0); + mc_reg_table->address[i].s1 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s1); + i++; + } + } + + mc_reg_table->last = (u8)i; + + return 0; +} + +static void ci_convert_mc_registers(const struct ci_mc_reg_entry *entry, + SMU7_Discrete_MCRegisterSet *data, + u32 num_entries, u32 valid_flag) +{ + u32 i, j; + + for (i = 0, j = 0; j < num_entries; j++) { + if (valid_flag & (1 << j)) { + data->value[i] = cpu_to_be32(entry->mc_data[j]); + i++; + } + } +} + +static void ci_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev, + const u32 memory_clock, + SMU7_Discrete_MCRegisterSet *mc_reg_table_data) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 i = 0; + + for(i = 0; i < pi->mc_reg_table.num_entries; i++) { + if (memory_clock <= pi->mc_reg_table.mc_reg_table_entry[i].mclk_max) + break; + } + + if ((i == pi->mc_reg_table.num_entries) && (i > 0)) + --i; + + ci_convert_mc_registers(&pi->mc_reg_table.mc_reg_table_entry[i], + mc_reg_table_data, pi->mc_reg_table.last, + pi->mc_reg_table.valid_flag); +} + +static void ci_convert_mc_reg_table_to_smc(struct radeon_device *rdev, + SMU7_Discrete_MCRegisters *mc_reg_table) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 i; + + for (i = 0; i < pi->dpm_table.mclk_table.count; i++) + ci_convert_mc_reg_table_entry_to_smc(rdev, + pi->dpm_table.mclk_table.dpm_levels[i].value, + &mc_reg_table->data[i]); +} + +static int ci_populate_initial_mc_reg_table(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + int ret; + + memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters)); + + ret = ci_populate_mc_reg_addresses(rdev, &pi->smc_mc_reg_table); + if (ret) + return ret; + ci_convert_mc_reg_table_to_smc(rdev, &pi->smc_mc_reg_table); + + return ci_copy_bytes_to_smc(rdev, + pi->mc_reg_table_start, + (u8 *)&pi->smc_mc_reg_table, + sizeof(SMU7_Discrete_MCRegisters), + pi->sram_end); +} + +static int ci_update_and_upload_mc_reg_table(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if (!(pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) + return 0; + + memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters)); + + ci_convert_mc_reg_table_to_smc(rdev, &pi->smc_mc_reg_table); + + return ci_copy_bytes_to_smc(rdev, + pi->mc_reg_table_start + + offsetof(SMU7_Discrete_MCRegisters, data[0]), + (u8 *)&pi->smc_mc_reg_table.data[0], + sizeof(SMU7_Discrete_MCRegisterSet) * + pi->dpm_table.mclk_table.count, + pi->sram_end); +} + +static void ci_enable_voltage_control(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(GENERAL_PWRMGT); + + tmp |= VOLT_PWRMGT_EN; + WREG32_SMC(GENERAL_PWRMGT, tmp); +} + +static enum radeon_pcie_gen ci_get_maximum_link_speed(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct ci_ps *state = ci_get_ps(radeon_state); + int i; + u16 pcie_speed, max_speed = 0; + + for (i = 0; i < state->performance_level_count; i++) { + pcie_speed = state->performance_levels[i].pcie_gen; + if (max_speed < pcie_speed) + max_speed = pcie_speed; + } + + return max_speed; +} + +static u16 ci_get_current_pcie_speed(struct radeon_device *rdev) +{ + u32 speed_cntl = 0; + + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE_MASK; + speed_cntl >>= LC_CURRENT_DATA_RATE_SHIFT; + + return (u16)speed_cntl; +} + +static int ci_get_current_pcie_lane_number(struct radeon_device *rdev) +{ + u32 link_width = 0; + + link_width = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL) & LC_LINK_WIDTH_RD_MASK; + link_width >>= LC_LINK_WIDTH_RD_SHIFT; + + switch (link_width) { + case RADEON_PCIE_LC_LINK_WIDTH_X1: + return 1; + case RADEON_PCIE_LC_LINK_WIDTH_X2: + return 2; + case RADEON_PCIE_LC_LINK_WIDTH_X4: + return 4; + case RADEON_PCIE_LC_LINK_WIDTH_X8: + return 8; + case RADEON_PCIE_LC_LINK_WIDTH_X12: + /* not actually supported */ + return 12; + case RADEON_PCIE_LC_LINK_WIDTH_X0: + case RADEON_PCIE_LC_LINK_WIDTH_X16: + default: + return 16; + } +} + +static void ci_request_link_speed_change_before_state_change(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + enum radeon_pcie_gen target_link_speed = + ci_get_maximum_link_speed(rdev, radeon_new_state); + enum radeon_pcie_gen current_link_speed; + + if (pi->force_pcie_gen == RADEON_PCIE_GEN_INVALID) + current_link_speed = ci_get_maximum_link_speed(rdev, radeon_current_state); + else + current_link_speed = pi->force_pcie_gen; + + pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID; + pi->pspp_notify_required = false; + if (target_link_speed > current_link_speed) { + switch (target_link_speed) { +#ifdef CONFIG_ACPI + case RADEON_PCIE_GEN3: + if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN3, false) == 0) + break; + pi->force_pcie_gen = RADEON_PCIE_GEN2; + if (current_link_speed == RADEON_PCIE_GEN2) + break; + case RADEON_PCIE_GEN2: + if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) + break; +#endif + default: + pi->force_pcie_gen = ci_get_current_pcie_speed(rdev); + break; + } + } else { + if (target_link_speed < current_link_speed) + pi->pspp_notify_required = true; + } +} + +static void ci_notify_link_speed_change_after_state_change(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + enum radeon_pcie_gen target_link_speed = + ci_get_maximum_link_speed(rdev, radeon_new_state); + u8 request; + + if (pi->pspp_notify_required) { + if (target_link_speed == RADEON_PCIE_GEN3) + request = PCIE_PERF_REQ_PECI_GEN3; + else if (target_link_speed == RADEON_PCIE_GEN2) + request = PCIE_PERF_REQ_PECI_GEN2; + else + request = PCIE_PERF_REQ_PECI_GEN1; + + if ((request == PCIE_PERF_REQ_PECI_GEN1) && + (ci_get_current_pcie_speed(rdev) > 0)) + return; + +#ifdef CONFIG_ACPI + radeon_acpi_pcie_performance_request(rdev, request, false); +#endif + } +} + +static int ci_set_private_data_variables_based_on_pptable(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_clock_voltage_dependency_table *allowed_sclk_vddc_table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + struct radeon_clock_voltage_dependency_table *allowed_mclk_vddc_table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk; + struct radeon_clock_voltage_dependency_table *allowed_mclk_vddci_table = + &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk; + + if (allowed_sclk_vddc_table == NULL) + return -EINVAL; + if (allowed_sclk_vddc_table->count < 1) + return -EINVAL; + if (allowed_mclk_vddc_table == NULL) + return -EINVAL; + if (allowed_mclk_vddc_table->count < 1) + return -EINVAL; + if (allowed_mclk_vddci_table == NULL) + return -EINVAL; + if (allowed_mclk_vddci_table->count < 1) + return -EINVAL; + + pi->min_vddc_in_pp_table = allowed_sclk_vddc_table->entries[0].v; + pi->max_vddc_in_pp_table = + allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v; + + pi->min_vddci_in_pp_table = allowed_mclk_vddci_table->entries[0].v; + pi->max_vddci_in_pp_table = + allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v; + + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = + allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = + allowed_mclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = + allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = + allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v; + + return 0; +} + +static void ci_patch_with_vddc_leakage(struct radeon_device *rdev, u16 *vddc) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_leakage_voltage *leakage_table = &pi->vddc_leakage; + u32 leakage_index; + + for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) { + if (leakage_table->leakage_id[leakage_index] == *vddc) { + *vddc = leakage_table->actual_voltage[leakage_index]; + break; + } + } +} + +static void ci_patch_with_vddci_leakage(struct radeon_device *rdev, u16 *vddci) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_leakage_voltage *leakage_table = &pi->vddci_leakage; + u32 leakage_index; + + for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) { + if (leakage_table->leakage_id[leakage_index] == *vddci) { + *vddci = leakage_table->actual_voltage[leakage_index]; + break; + } + } +} + +static void ci_patch_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device *rdev, + struct radeon_clock_voltage_dependency_table *table) +{ + u32 i; + + if (table) { + for (i = 0; i < table->count; i++) + ci_patch_with_vddc_leakage(rdev, &table->entries[i].v); + } +} + +static void ci_patch_clock_voltage_dependency_table_with_vddci_leakage(struct radeon_device *rdev, + struct radeon_clock_voltage_dependency_table *table) +{ + u32 i; + + if (table) { + for (i = 0; i < table->count; i++) + ci_patch_with_vddci_leakage(rdev, &table->entries[i].v); + } +} + +static void ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device *rdev, + struct radeon_vce_clock_voltage_dependency_table *table) +{ + u32 i; + + if (table) { + for (i = 0; i < table->count; i++) + ci_patch_with_vddc_leakage(rdev, &table->entries[i].v); + } +} + +static void ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device *rdev, + struct radeon_uvd_clock_voltage_dependency_table *table) +{ + u32 i; + + if (table) { + for (i = 0; i < table->count; i++) + ci_patch_with_vddc_leakage(rdev, &table->entries[i].v); + } +} + +static void ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(struct radeon_device *rdev, + struct radeon_phase_shedding_limits_table *table) +{ + u32 i; + + if (table) { + for (i = 0; i < table->count; i++) + ci_patch_with_vddc_leakage(rdev, &table->entries[i].voltage); + } +} + +static void ci_patch_clock_voltage_limits_with_vddc_leakage(struct radeon_device *rdev, + struct radeon_clock_and_voltage_limits *table) +{ + if (table) { + ci_patch_with_vddc_leakage(rdev, (u16 *)&table->vddc); + ci_patch_with_vddci_leakage(rdev, (u16 *)&table->vddci); + } +} + +static void ci_patch_cac_leakage_table_with_vddc_leakage(struct radeon_device *rdev, + struct radeon_cac_leakage_table *table) +{ + u32 i; + + if (table) { + for (i = 0; i < table->count; i++) + ci_patch_with_vddc_leakage(rdev, &table->entries[i].vddc); + } +} + +static void ci_patch_dependency_tables_with_leakage(struct radeon_device *rdev) +{ + + ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk); + ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk); + ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk); + ci_patch_clock_voltage_dependency_table_with_vddci_leakage(rdev, + &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk); + ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table); + ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table); + ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table); + ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table); + ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.phase_shedding_limits_table); + ci_patch_clock_voltage_limits_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac); + ci_patch_clock_voltage_limits_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc); + ci_patch_cac_leakage_table_with_vddc_leakage(rdev, + &rdev->pm.dpm.dyn_state.cac_leakage_table); + +} + +static void ci_get_memory_type(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + + tmp = RREG32(MC_SEQ_MISC0); + + if (((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT) == + MC_SEQ_MISC0_GDDR5_VALUE) + pi->mem_gddr5 = true; + else + pi->mem_gddr5 = false; + +} + +static void ci_update_current_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct ci_ps *new_ps = ci_get_ps(rps); + struct ci_power_info *pi = ci_get_pi(rdev); + + pi->current_rps = *rps; + pi->current_ps = *new_ps; + pi->current_rps.ps_priv = &pi->current_ps; +} + +static void ci_update_requested_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct ci_ps *new_ps = ci_get_ps(rps); + struct ci_power_info *pi = ci_get_pi(rdev); + + pi->requested_rps = *rps; + pi->requested_ps = *new_ps; + pi->requested_rps.ps_priv = &pi->requested_ps; +} + +int ci_dpm_pre_set_power_state(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps; + struct radeon_ps *new_ps = &requested_ps; + + ci_update_requested_ps(rdev, new_ps); + + ci_apply_state_adjust_rules(rdev, &pi->requested_rps); + + return 0; +} + +void ci_dpm_post_set_power_state(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_ps *new_ps = &pi->requested_rps; + + ci_update_current_ps(rdev, new_ps); +} + + +void ci_dpm_setup_asic(struct radeon_device *rdev) +{ + int r; + + r = ci_mc_load_microcode(rdev); + if (r) + DRM_ERROR("Failed to load MC firmware!\n"); + ci_read_clock_registers(rdev); + ci_get_memory_type(rdev); + ci_enable_acpi_power_management(rdev); + ci_init_sclk_t(rdev); +} + +int ci_dpm_enable(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + int ret; + + if (ci_is_smc_running(rdev)) + return -EINVAL; + if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE) { + ci_enable_voltage_control(rdev); + ret = ci_construct_voltage_tables(rdev); + if (ret) { + DRM_ERROR("ci_construct_voltage_tables failed\n"); + return ret; + } + } + if (pi->caps_dynamic_ac_timing) { + ret = ci_initialize_mc_reg_table(rdev); + if (ret) + pi->caps_dynamic_ac_timing = false; + } + if (pi->dynamic_ss) + ci_enable_spread_spectrum(rdev, true); + if (pi->thermal_protection) + ci_enable_thermal_protection(rdev, true); + ci_program_sstp(rdev); + ci_enable_display_gap(rdev); + ci_program_vc(rdev); + ret = ci_upload_firmware(rdev); + if (ret) { + DRM_ERROR("ci_upload_firmware failed\n"); + return ret; + } + ret = ci_process_firmware_header(rdev); + if (ret) { + DRM_ERROR("ci_process_firmware_header failed\n"); + return ret; + } + ret = ci_initial_switch_from_arb_f0_to_f1(rdev); + if (ret) { + DRM_ERROR("ci_initial_switch_from_arb_f0_to_f1 failed\n"); + return ret; + } + ret = ci_init_smc_table(rdev); + if (ret) { + DRM_ERROR("ci_init_smc_table failed\n"); + return ret; + } + ret = ci_init_arb_table_index(rdev); + if (ret) { + DRM_ERROR("ci_init_arb_table_index failed\n"); + return ret; + } + if (pi->caps_dynamic_ac_timing) { + ret = ci_populate_initial_mc_reg_table(rdev); + if (ret) { + DRM_ERROR("ci_populate_initial_mc_reg_table failed\n"); + return ret; + } + } + ret = ci_populate_pm_base(rdev); + if (ret) { + DRM_ERROR("ci_populate_pm_base failed\n"); + return ret; + } + ci_dpm_start_smc(rdev); + ci_enable_vr_hot_gpio_interrupt(rdev); + ret = ci_notify_smc_display_change(rdev, false); + if (ret) { + DRM_ERROR("ci_notify_smc_display_change failed\n"); + return ret; + } + ci_enable_sclk_control(rdev, true); + ret = ci_enable_ulv(rdev, true); + if (ret) { + DRM_ERROR("ci_enable_ulv failed\n"); + return ret; + } + ret = ci_enable_ds_master_switch(rdev, true); + if (ret) { + DRM_ERROR("ci_enable_ds_master_switch failed\n"); + return ret; + } + ret = ci_start_dpm(rdev); + if (ret) { + DRM_ERROR("ci_start_dpm failed\n"); + return ret; + } + ret = ci_enable_didt(rdev, true); + if (ret) { + DRM_ERROR("ci_enable_didt failed\n"); + return ret; + } + ret = ci_enable_smc_cac(rdev, true); + if (ret) { + DRM_ERROR("ci_enable_smc_cac failed\n"); + return ret; + } + ret = ci_enable_power_containment(rdev, true); + if (ret) { + DRM_ERROR("ci_enable_power_containment failed\n"); + return ret; + } + + ret = ci_power_control_set_level(rdev); + if (ret) { + DRM_ERROR("ci_power_control_set_level failed\n"); + return ret; + } + + ci_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + + ret = ci_enable_thermal_based_sclk_dpm(rdev, true); + if (ret) { + DRM_ERROR("ci_enable_thermal_based_sclk_dpm failed\n"); + return ret; + } + + ci_thermal_start_thermal_controller(rdev); + + ci_update_current_ps(rdev, boot_ps); + + return 0; +} + +static int ci_set_temperature_range(struct radeon_device *rdev) +{ + int ret; + + ret = ci_thermal_enable_alert(rdev, false); + if (ret) + return ret; + ret = ci_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) + return ret; + ret = ci_thermal_enable_alert(rdev, true); + if (ret) + return ret; + + return ret; +} + +int ci_dpm_late_enable(struct radeon_device *rdev) +{ + int ret; + + ret = ci_set_temperature_range(rdev); + if (ret) + return ret; + + ci_dpm_powergate_uvd(rdev, true); + + return 0; +} + +void ci_dpm_disable(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + + ci_dpm_powergate_uvd(rdev, false); + + if (!ci_is_smc_running(rdev)) + return; + + ci_thermal_stop_thermal_controller(rdev); + + if (pi->thermal_protection) + ci_enable_thermal_protection(rdev, false); + ci_enable_power_containment(rdev, false); + ci_enable_smc_cac(rdev, false); + ci_enable_didt(rdev, false); + ci_enable_spread_spectrum(rdev, false); + ci_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false); + ci_stop_dpm(rdev); + ci_enable_ds_master_switch(rdev, false); + ci_enable_ulv(rdev, false); + ci_clear_vc(rdev); + ci_reset_to_default(rdev); + ci_dpm_stop_smc(rdev); + ci_force_switch_to_arb_f0(rdev); + ci_enable_thermal_based_sclk_dpm(rdev, false); + + ci_update_current_ps(rdev, boot_ps); +} + +int ci_dpm_set_power_state(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_ps *new_ps = &pi->requested_rps; + struct radeon_ps *old_ps = &pi->current_rps; + int ret; + + ci_find_dpm_states_clocks_in_dpm_table(rdev, new_ps); + if (pi->pcie_performance_request) + ci_request_link_speed_change_before_state_change(rdev, new_ps, old_ps); + ret = ci_freeze_sclk_mclk_dpm(rdev); + if (ret) { + DRM_ERROR("ci_freeze_sclk_mclk_dpm failed\n"); + return ret; + } + ret = ci_populate_and_upload_sclk_mclk_dpm_levels(rdev, new_ps); + if (ret) { + DRM_ERROR("ci_populate_and_upload_sclk_mclk_dpm_levels failed\n"); + return ret; + } + ret = ci_generate_dpm_level_enable_mask(rdev, new_ps); + if (ret) { + DRM_ERROR("ci_generate_dpm_level_enable_mask failed\n"); + return ret; + } + + ret = ci_update_vce_dpm(rdev, new_ps, old_ps); + if (ret) { + DRM_ERROR("ci_update_vce_dpm failed\n"); + return ret; + } + + ret = ci_update_sclk_t(rdev); + if (ret) { + DRM_ERROR("ci_update_sclk_t failed\n"); + return ret; + } + if (pi->caps_dynamic_ac_timing) { + ret = ci_update_and_upload_mc_reg_table(rdev); + if (ret) { + DRM_ERROR("ci_update_and_upload_mc_reg_table failed\n"); + return ret; + } + } + ret = ci_program_memory_timing_parameters(rdev); + if (ret) { + DRM_ERROR("ci_program_memory_timing_parameters failed\n"); + return ret; + } + ret = ci_unfreeze_sclk_mclk_dpm(rdev); + if (ret) { + DRM_ERROR("ci_unfreeze_sclk_mclk_dpm failed\n"); + return ret; + } + ret = ci_upload_dpm_level_enable_mask(rdev); + if (ret) { + DRM_ERROR("ci_upload_dpm_level_enable_mask failed\n"); + return ret; + } + if (pi->pcie_performance_request) + ci_notify_link_speed_change_after_state_change(rdev, new_ps, old_ps); + + return 0; +} + +#if 0 +void ci_dpm_reset_asic(struct radeon_device *rdev) +{ + ci_set_boot_state(rdev); +} +#endif + +void ci_dpm_display_configuration_changed(struct radeon_device *rdev) +{ + ci_program_display_gap(rdev); +} + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; +}; + +union pplib_clock_info { + struct _ATOM_PPLIB_R600_CLOCK_INFO r600; + struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; + struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; + struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; + struct _ATOM_PPLIB_SI_CLOCK_INFO si; + struct _ATOM_PPLIB_CI_CLOCK_INFO ci; +}; + +union pplib_power_state { + struct _ATOM_PPLIB_STATE v1; + struct _ATOM_PPLIB_STATE_V2 v2; +}; + +static void ci_parse_pplib_non_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info, + u8 table_rev) +{ + rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); + rps->class = le16_to_cpu(non_clock_info->usClassification); + rps->class2 = le16_to_cpu(non_clock_info->usClassification2); + + if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) { + rps->vclk = le32_to_cpu(non_clock_info->ulVCLK); + rps->dclk = le32_to_cpu(non_clock_info->ulDCLK); + } else { + rps->vclk = 0; + rps->dclk = 0; + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) + rdev->pm.dpm.boot_ps = rps; + if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + rdev->pm.dpm.uvd_ps = rps; +} + +static void ci_parse_pplib_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, int index, + union pplib_clock_info *clock_info) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_ps *ps = ci_get_ps(rps); + struct ci_pl *pl = &ps->performance_levels[index]; + + ps->performance_level_count = index + 1; + + pl->sclk = le16_to_cpu(clock_info->ci.usEngineClockLow); + pl->sclk |= clock_info->ci.ucEngineClockHigh << 16; + pl->mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow); + pl->mclk |= clock_info->ci.ucMemoryClockHigh << 16; + + pl->pcie_gen = r600_get_pcie_gen_support(rdev, + pi->sys_pcie_mask, + pi->vbios_boot_state.pcie_gen_bootup_value, + clock_info->ci.ucPCIEGen); + pl->pcie_lane = r600_get_pcie_lane_support(rdev, + pi->vbios_boot_state.pcie_lane_bootup_value, + le16_to_cpu(clock_info->ci.usPCIELane)); + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) { + pi->acpi_pcie_gen = pl->pcie_gen; + } + + if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) { + pi->ulv.supported = true; + pi->ulv.pl = *pl; + pi->ulv.cg_ulv_parameter = CISLANDS_CGULVPARAMETER_DFLT; + } + + /* patch up boot state */ + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { + pl->mclk = pi->vbios_boot_state.mclk_bootup_value; + pl->sclk = pi->vbios_boot_state.sclk_bootup_value; + pl->pcie_gen = pi->vbios_boot_state.pcie_gen_bootup_value; + pl->pcie_lane = pi->vbios_boot_state.pcie_lane_bootup_value; + } + + switch (rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) { + case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY: + pi->use_pcie_powersaving_levels = true; + if (pi->pcie_gen_powersaving.max < pl->pcie_gen) + pi->pcie_gen_powersaving.max = pl->pcie_gen; + if (pi->pcie_gen_powersaving.min > pl->pcie_gen) + pi->pcie_gen_powersaving.min = pl->pcie_gen; + if (pi->pcie_lane_powersaving.max < pl->pcie_lane) + pi->pcie_lane_powersaving.max = pl->pcie_lane; + if (pi->pcie_lane_powersaving.min > pl->pcie_lane) + pi->pcie_lane_powersaving.min = pl->pcie_lane; + break; + case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE: + pi->use_pcie_performance_levels = true; + if (pi->pcie_gen_performance.max < pl->pcie_gen) + pi->pcie_gen_performance.max = pl->pcie_gen; + if (pi->pcie_gen_performance.min > pl->pcie_gen) + pi->pcie_gen_performance.min = pl->pcie_gen; + if (pi->pcie_lane_performance.max < pl->pcie_lane) + pi->pcie_lane_performance.max = pl->pcie_lane; + if (pi->pcie_lane_performance.min > pl->pcie_lane) + pi->pcie_lane_performance.min = pl->pcie_lane; + break; + default: + break; + } +} + +static int ci_parse_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; + union pplib_power_state *power_state; + int i, j, k, non_clock_array_index, clock_array_index; + union pplib_clock_info *clock_info; + struct _StateArray *state_array; + struct _ClockInfoArray *clock_info_array; + struct _NonClockInfoArray *non_clock_info_array; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + u8 *power_state_offset; + struct ci_ps *ps; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + state_array = (struct _StateArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usStateArrayOffset)); + clock_info_array = (struct _ClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usClockInfoArrayOffset)); + non_clock_info_array = (struct _NonClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset)); + + rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) * + state_array->ucNumEntries, GFP_KERNEL); + if (!rdev->pm.dpm.ps) + return -ENOMEM; + power_state_offset = (u8 *)state_array->states; + for (i = 0; i < state_array->ucNumEntries; i++) { + u8 *idx; + power_state = (union pplib_power_state *)power_state_offset; + non_clock_array_index = power_state->v2.nonClockInfoIndex; + non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) + &non_clock_info_array->nonClockInfo[non_clock_array_index]; + if (!rdev->pm.power_state[i].clock_info) + return -EINVAL; + ps = kzalloc(sizeof(struct ci_ps), GFP_KERNEL); + if (ps == NULL) { + kfree(rdev->pm.dpm.ps); + return -ENOMEM; + } + rdev->pm.dpm.ps[i].ps_priv = ps; + ci_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i], + non_clock_info, + non_clock_info_array->ucEntrySize); + k = 0; + idx = (u8 *)&power_state->v2.clockInfoIndex[0]; + for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) { + clock_array_index = idx[j]; + if (clock_array_index >= clock_info_array->ucNumEntries) + continue; + if (k >= CISLANDS_MAX_HARDWARE_POWERLEVELS) + break; + clock_info = (union pplib_clock_info *) + ((u8 *)&clock_info_array->clockInfo[0] + + (clock_array_index * clock_info_array->ucEntrySize)); + ci_parse_pplib_clock_info(rdev, + &rdev->pm.dpm.ps[i], k, + clock_info); + k++; + } + power_state_offset += 2 + power_state->v2.ucNumDPMLevels; + } + rdev->pm.dpm.num_ps = state_array->ucNumEntries; + + /* fill in the vce power states */ + for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) { + u32 sclk, mclk; + clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx; + clock_info = (union pplib_clock_info *) + &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize]; + sclk = le16_to_cpu(clock_info->ci.usEngineClockLow); + sclk |= clock_info->ci.ucEngineClockHigh << 16; + mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow); + mclk |= clock_info->ci.ucMemoryClockHigh << 16; + rdev->pm.dpm.vce_states[i].sclk = sclk; + rdev->pm.dpm.vce_states[i].mclk = mclk; + } + + return 0; +} + +static int ci_get_vbios_boot_values(struct radeon_device *rdev, + struct ci_vbios_boot_state *boot_state) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + int index = GetIndexIntoMasterTable(DATA, FirmwareInfo); + ATOM_FIRMWARE_INFO_V2_2 *firmware_info; + u8 frev, crev; + u16 data_offset; + + if (atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + firmware_info = + (ATOM_FIRMWARE_INFO_V2_2 *)(mode_info->atom_context->bios + + data_offset); + boot_state->mvdd_bootup_value = le16_to_cpu(firmware_info->usBootUpMVDDCVoltage); + boot_state->vddc_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCVoltage); + boot_state->vddci_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCIVoltage); + boot_state->pcie_gen_bootup_value = ci_get_current_pcie_speed(rdev); + boot_state->pcie_lane_bootup_value = ci_get_current_pcie_lane_number(rdev); + boot_state->sclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultEngineClock); + boot_state->mclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultMemoryClock); + + return 0; + } + return -EINVAL; +} + +void ci_dpm_fini(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); + kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries); + r600_free_extended_power_table(rdev); +} + +int ci_dpm_init(struct radeon_device *rdev) +{ + int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info); + SMU7_Discrete_DpmTable *dpm_table; + struct radeon_gpio_rec gpio; + u16 data_offset, size; + u8 frev, crev; + struct ci_power_info *pi; + int ret; + u32 mask; + + pi = kzalloc(sizeof(struct ci_power_info), GFP_KERNEL); + if (pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = pi; + + ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); + if (ret) + pi->sys_pcie_mask = 0; + else + pi->sys_pcie_mask = mask; + pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID; + + pi->pcie_gen_performance.max = RADEON_PCIE_GEN1; + pi->pcie_gen_performance.min = RADEON_PCIE_GEN3; + pi->pcie_gen_powersaving.max = RADEON_PCIE_GEN1; + pi->pcie_gen_powersaving.min = RADEON_PCIE_GEN3; + + pi->pcie_lane_performance.max = 0; + pi->pcie_lane_performance.min = 16; + pi->pcie_lane_powersaving.max = 0; + pi->pcie_lane_powersaving.min = 16; + + ret = ci_get_vbios_boot_values(rdev, &pi->vbios_boot_state); + if (ret) { + ci_dpm_fini(rdev); + return ret; + } + + ret = r600_get_platform_caps(rdev); + if (ret) { + ci_dpm_fini(rdev); + return ret; + } + + ret = r600_parse_extended_power_table(rdev); + if (ret) { + ci_dpm_fini(rdev); + return ret; + } + + ret = ci_parse_power_table(rdev); + if (ret) { + ci_dpm_fini(rdev); + return ret; + } + + pi->dll_default_on = false; + pi->sram_end = SMC_RAM_END; + + pi->activity_target[0] = CISLAND_TARGETACTIVITY_DFLT; + pi->activity_target[1] = CISLAND_TARGETACTIVITY_DFLT; + pi->activity_target[2] = CISLAND_TARGETACTIVITY_DFLT; + pi->activity_target[3] = CISLAND_TARGETACTIVITY_DFLT; + pi->activity_target[4] = CISLAND_TARGETACTIVITY_DFLT; + pi->activity_target[5] = CISLAND_TARGETACTIVITY_DFLT; + pi->activity_target[6] = CISLAND_TARGETACTIVITY_DFLT; + pi->activity_target[7] = CISLAND_TARGETACTIVITY_DFLT; + + pi->mclk_activity_target = CISLAND_MCLK_TARGETACTIVITY_DFLT; + + pi->sclk_dpm_key_disabled = 0; + pi->mclk_dpm_key_disabled = 0; + pi->pcie_dpm_key_disabled = 0; + pi->thermal_sclk_dpm_enabled = 0; + + /* mclk dpm is unstable on some R7 260X cards with the old mc ucode */ + if ((rdev->pdev->device == 0x6658) && + (rdev->mc_fw->size == (BONAIRE_MC_UCODE_SIZE * 4))) { + pi->mclk_dpm_key_disabled = 1; + } + + pi->caps_sclk_ds = true; + + pi->mclk_strobe_mode_threshold = 40000; + pi->mclk_stutter_mode_threshold = 40000; + pi->mclk_edc_enable_threshold = 40000; + pi->mclk_edc_wr_enable_threshold = 40000; + + ci_initialize_powertune_defaults(rdev); + + pi->caps_fps = false; + + pi->caps_sclk_throttle_low_notification = false; + + pi->caps_uvd_dpm = true; + pi->caps_vce_dpm = true; + + ci_get_leakage_voltages(rdev); + ci_patch_dependency_tables_with_leakage(rdev); + ci_set_private_data_variables_based_on_pptable(rdev); + + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries = + kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) { + ci_dpm_fini(rdev); + return -ENOMEM; + } + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900; + + rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 4; + rdev->pm.dpm.dyn_state.sclk_mclk_delta = 15000; + rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200; + + rdev->pm.dpm.dyn_state.valid_sclk_values.count = 0; + rdev->pm.dpm.dyn_state.valid_sclk_values.values = NULL; + rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0; + rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL; + + if (rdev->family == CHIP_HAWAII) { + pi->thermal_temp_setting.temperature_low = 94500; + pi->thermal_temp_setting.temperature_high = 95000; + pi->thermal_temp_setting.temperature_shutdown = 104000; + } else { + pi->thermal_temp_setting.temperature_low = 99500; + pi->thermal_temp_setting.temperature_high = 100000; + pi->thermal_temp_setting.temperature_shutdown = 104000; + } + + pi->uvd_enabled = false; + + dpm_table = &pi->smc_state_table; + + gpio = radeon_atombios_lookup_gpio(rdev, VDDC_VRHOT_GPIO_PINID); + if (gpio.valid) { + dpm_table->VRHotGpio = gpio.shift; + rdev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_REGULATOR_HOT; + } else { + dpm_table->VRHotGpio = CISLANDS_UNUSED_GPIO_PIN; + rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_REGULATOR_HOT; + } + + gpio = radeon_atombios_lookup_gpio(rdev, PP_AC_DC_SWITCH_GPIO_PINID); + if (gpio.valid) { + dpm_table->AcDcGpio = gpio.shift; + rdev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_HARDWAREDC; + } else { + dpm_table->AcDcGpio = CISLANDS_UNUSED_GPIO_PIN; + rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_HARDWAREDC; + } + + gpio = radeon_atombios_lookup_gpio(rdev, VDDC_PCC_GPIO_PINID); + if (gpio.valid) { + u32 tmp = RREG32_SMC(CNB_PWRMGT_CNTL); + + switch (gpio.shift) { + case 0: + tmp &= ~GNB_SLOW_MODE_MASK; + tmp |= GNB_SLOW_MODE(1); + break; + case 1: + tmp &= ~GNB_SLOW_MODE_MASK; + tmp |= GNB_SLOW_MODE(2); + break; + case 2: + tmp |= GNB_SLOW; + break; + case 3: + tmp |= FORCE_NB_PS1; + break; + case 4: + tmp |= DPM_ENABLED; + break; + default: + DRM_DEBUG("Invalid PCC GPIO: %u!\n", gpio.shift); + break; + } + WREG32_SMC(CNB_PWRMGT_CNTL, tmp); + } + + pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_NONE; + pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_NONE; + pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_NONE; + if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_GPIO_LUT)) + pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO; + else if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2)) + pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL) { + if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT)) + pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO; + else if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2)) + pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2; + else + rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL; + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_MVDDCONTROL) { + if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT)) + pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO; + else if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_SVID2)) + pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2; + else + rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_MVDDCONTROL; + } + + pi->vddc_phase_shed_control = true; + +#if defined(CONFIG_ACPI) + pi->pcie_performance_request = + radeon_acpi_is_pcie_performance_request_supported(rdev); +#else + pi->pcie_performance_request = false; +#endif + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + pi->caps_sclk_ss_support = true; + pi->caps_mclk_ss_support = true; + pi->dynamic_ss = true; + } else { + pi->caps_sclk_ss_support = false; + pi->caps_mclk_ss_support = false; + pi->dynamic_ss = true; + } + + if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE) + pi->thermal_protection = true; + else + pi->thermal_protection = false; + + pi->caps_dynamic_ac_timing = true; + + pi->uvd_power_gated = false; + + /* make sure dc limits are valid */ + if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) || + (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0)) + rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc = + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + + pi->fan_ctrl_is_in_default_mode = true; + + return 0; +} + +void ci_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct radeon_ps *rps = &pi->current_rps; + u32 sclk = ci_get_average_sclk_freq(rdev); + u32 mclk = ci_get_average_mclk_freq(rdev); + + seq_printf(m, "uvd %sabled\n", pi->uvd_enabled ? "en" : "dis"); + seq_printf(m, "vce %sabled\n", rps->vce_active ? "en" : "dis"); + seq_printf(m, "power level avg sclk: %u mclk: %u\n", + sclk, mclk); +} + +void ci_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct ci_ps *ps = ci_get_ps(rps); + struct ci_pl *pl; + int i; + + r600_dpm_print_class_info(rps->class, rps->class2); + r600_dpm_print_cap_info(rps->caps); + printk("\tuvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + for (i = 0; i < ps->performance_level_count; i++) { + pl = &ps->performance_levels[i]; + printk("\t\tpower level %d sclk: %u mclk: %u pcie gen: %u pcie lanes: %u\n", + i, pl->sclk, pl->mclk, pl->pcie_gen + 1, pl->pcie_lane); + } + r600_dpm_print_ps_status(rdev, rps); +} + +u32 ci_dpm_get_current_sclk(struct radeon_device *rdev) +{ + u32 sclk = ci_get_average_sclk_freq(rdev); + + return sclk; +} + +u32 ci_dpm_get_current_mclk(struct radeon_device *rdev) +{ + u32 mclk = ci_get_average_mclk_freq(rdev); + + return mclk; +} + +u32 ci_dpm_get_sclk(struct radeon_device *rdev, bool low) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps); + + if (low) + return requested_state->performance_levels[0].sclk; + else + return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk; +} + +u32 ci_dpm_get_mclk(struct radeon_device *rdev, bool low) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps); + + if (low) + return requested_state->performance_levels[0].mclk; + else + return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk; +} diff --git a/sys/dev/pci/drm/radeon/ci_dpm.h b/sys/dev/pci/drm/radeon/ci_dpm.h new file mode 100644 index 00000000000..723220ffbea --- /dev/null +++ b/sys/dev/pci/drm/radeon/ci_dpm.h @@ -0,0 +1,341 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __CI_DPM_H__ +#define __CI_DPM_H__ + +#include "ppsmc.h" + +#define SMU__NUM_SCLK_DPM_STATE 8 +#define SMU__NUM_MCLK_DPM_LEVELS 6 +#define SMU__NUM_LCLK_DPM_LEVELS 8 +#define SMU__NUM_PCIE_DPM_LEVELS 8 +#include "smu7_discrete.h" + +#define CISLANDS_MAX_HARDWARE_POWERLEVELS 2 + +#define CISLANDS_UNUSED_GPIO_PIN 0x7F + +struct ci_pl { + u32 mclk; + u32 sclk; + enum radeon_pcie_gen pcie_gen; + u16 pcie_lane; +}; + +struct ci_ps { + u16 performance_level_count; + bool dc_compatible; + u32 sclk_t; + struct ci_pl performance_levels[CISLANDS_MAX_HARDWARE_POWERLEVELS]; +}; + +struct ci_dpm_level { + bool enabled; + u32 value; + u32 param1; +}; + +#define CISLAND_MAX_DEEPSLEEP_DIVIDER_ID 5 +#define MAX_REGULAR_DPM_NUMBER 8 +#define CISLAND_MINIMUM_ENGINE_CLOCK 800 + +struct ci_single_dpm_table { + u32 count; + struct ci_dpm_level dpm_levels[MAX_REGULAR_DPM_NUMBER]; +}; + +struct ci_dpm_table { + struct ci_single_dpm_table sclk_table; + struct ci_single_dpm_table mclk_table; + struct ci_single_dpm_table pcie_speed_table; + struct ci_single_dpm_table vddc_table; + struct ci_single_dpm_table vddci_table; + struct ci_single_dpm_table mvdd_table; +}; + +struct ci_mc_reg_entry { + u32 mclk_max; + u32 mc_data[SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE]; +}; + +struct ci_mc_reg_table { + u8 last; + u8 num_entries; + u16 valid_flag; + struct ci_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES]; + SMU7_Discrete_MCRegisterAddress mc_reg_address[SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE]; +}; + +struct ci_ulv_parm +{ + bool supported; + u32 cg_ulv_parameter; + u32 volt_change_delay; + struct ci_pl pl; +}; + +#define CISLANDS_MAX_LEAKAGE_COUNT 8 + +struct ci_leakage_voltage { + u16 count; + u16 leakage_id[CISLANDS_MAX_LEAKAGE_COUNT]; + u16 actual_voltage[CISLANDS_MAX_LEAKAGE_COUNT]; +}; + +struct ci_dpm_level_enable_mask { + u32 uvd_dpm_enable_mask; + u32 vce_dpm_enable_mask; + u32 acp_dpm_enable_mask; + u32 samu_dpm_enable_mask; + u32 sclk_dpm_enable_mask; + u32 mclk_dpm_enable_mask; + u32 pcie_dpm_enable_mask; +}; + +struct ci_vbios_boot_state +{ + u16 mvdd_bootup_value; + u16 vddc_bootup_value; + u16 vddci_bootup_value; + u32 sclk_bootup_value; + u32 mclk_bootup_value; + u16 pcie_gen_bootup_value; + u16 pcie_lane_bootup_value; +}; + +struct ci_clock_registers { + u32 cg_spll_func_cntl; + u32 cg_spll_func_cntl_2; + u32 cg_spll_func_cntl_3; + u32 cg_spll_func_cntl_4; + u32 cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2; + u32 dll_cntl; + u32 mclk_pwrmgt_cntl; + u32 mpll_ad_func_cntl; + u32 mpll_dq_func_cntl; + u32 mpll_func_cntl; + u32 mpll_func_cntl_1; + u32 mpll_func_cntl_2; + u32 mpll_ss1; + u32 mpll_ss2; +}; + +struct ci_thermal_temperature_setting { + s32 temperature_low; + s32 temperature_high; + s32 temperature_shutdown; +}; + +struct ci_pcie_perf_range { + u16 max; + u16 min; +}; + +enum ci_pt_config_reg_type { + CISLANDS_CONFIGREG_MMR = 0, + CISLANDS_CONFIGREG_SMC_IND, + CISLANDS_CONFIGREG_DIDT_IND, + CISLANDS_CONFIGREG_CACHE, + CISLANDS_CONFIGREG_MAX +}; + +#define POWERCONTAINMENT_FEATURE_BAPM 0x00000001 +#define POWERCONTAINMENT_FEATURE_TDCLimit 0x00000002 +#define POWERCONTAINMENT_FEATURE_PkgPwrLimit 0x00000004 + +struct ci_pt_config_reg { + u32 offset; + u32 mask; + u32 shift; + u32 value; + enum ci_pt_config_reg_type type; +}; + +struct ci_pt_defaults { + u8 svi_load_line_en; + u8 svi_load_line_vddc; + u8 tdc_vddc_throttle_release_limit_perc; + u8 tdc_mawt; + u8 tdc_waterfall_ctl; + u8 dte_ambient_temp_base; + u32 display_cac; + u32 bapm_temp_gradient; + u16 bapmti_r[SMU7_DTE_ITERATIONS * SMU7_DTE_SOURCES * SMU7_DTE_SINKS]; + u16 bapmti_rc[SMU7_DTE_ITERATIONS * SMU7_DTE_SOURCES * SMU7_DTE_SINKS]; +}; + +#define DPMTABLE_OD_UPDATE_SCLK 0x00000001 +#define DPMTABLE_OD_UPDATE_MCLK 0x00000002 +#define DPMTABLE_UPDATE_SCLK 0x00000004 +#define DPMTABLE_UPDATE_MCLK 0x00000008 + +struct ci_power_info { + struct ci_dpm_table dpm_table; + u32 voltage_control; + u32 mvdd_control; + u32 vddci_control; + u32 active_auto_throttle_sources; + struct ci_clock_registers clock_registers; + u16 acpi_vddc; + u16 acpi_vddci; + enum radeon_pcie_gen force_pcie_gen; + enum radeon_pcie_gen acpi_pcie_gen; + struct ci_leakage_voltage vddc_leakage; + struct ci_leakage_voltage vddci_leakage; + u16 max_vddc_in_pp_table; + u16 min_vddc_in_pp_table; + u16 max_vddci_in_pp_table; + u16 min_vddci_in_pp_table; + u32 mclk_strobe_mode_threshold; + u32 mclk_stutter_mode_threshold; + u32 mclk_edc_enable_threshold; + u32 mclk_edc_wr_enable_threshold; + struct ci_vbios_boot_state vbios_boot_state; + /* smc offsets */ + u32 sram_end; + u32 dpm_table_start; + u32 soft_regs_start; + u32 mc_reg_table_start; + u32 fan_table_start; + u32 arb_table_start; + /* smc tables */ + SMU7_Discrete_DpmTable smc_state_table; + SMU7_Discrete_MCRegisters smc_mc_reg_table; + SMU7_Discrete_PmFuses smc_powertune_table; + /* other stuff */ + struct ci_mc_reg_table mc_reg_table; + struct atom_voltage_table vddc_voltage_table; + struct atom_voltage_table vddci_voltage_table; + struct atom_voltage_table mvdd_voltage_table; + struct ci_ulv_parm ulv; + u32 power_containment_features; + const struct ci_pt_defaults *powertune_defaults; + u32 dte_tj_offset; + bool vddc_phase_shed_control; + struct ci_thermal_temperature_setting thermal_temp_setting; + struct ci_dpm_level_enable_mask dpm_level_enable_mask; + u32 need_update_smu7_dpm_table; + u32 sclk_dpm_key_disabled; + u32 mclk_dpm_key_disabled; + u32 pcie_dpm_key_disabled; + u32 thermal_sclk_dpm_enabled; + struct ci_pcie_perf_range pcie_gen_performance; + struct ci_pcie_perf_range pcie_lane_performance; + struct ci_pcie_perf_range pcie_gen_powersaving; + struct ci_pcie_perf_range pcie_lane_powersaving; + u32 activity_target[SMU7_MAX_LEVELS_GRAPHICS]; + u32 mclk_activity_target; + u32 low_sclk_interrupt_t; + u32 last_mclk_dpm_enable_mask; + u32 sys_pcie_mask; + /* caps */ + bool caps_power_containment; + bool caps_cac; + bool caps_sq_ramping; + bool caps_db_ramping; + bool caps_td_ramping; + bool caps_tcp_ramping; + bool caps_fps; + bool caps_sclk_ds; + bool caps_sclk_ss_support; + bool caps_mclk_ss_support; + bool caps_uvd_dpm; + bool caps_vce_dpm; + bool caps_samu_dpm; + bool caps_acp_dpm; + bool caps_automatic_dc_transition; + bool caps_sclk_throttle_low_notification; + bool caps_dynamic_ac_timing; + bool caps_od_fuzzy_fan_control_support; + /* flags */ + bool thermal_protection; + bool pcie_performance_request; + bool dynamic_ss; + bool dll_default_on; + bool cac_enabled; + bool uvd_enabled; + bool battery_state; + bool pspp_notify_required; + bool mem_gddr5; + bool enable_bapm_feature; + bool enable_tdc_limit_feature; + bool enable_pkg_pwr_tracking_feature; + bool use_pcie_performance_levels; + bool use_pcie_powersaving_levels; + bool uvd_power_gated; + /* driver states */ + struct radeon_ps current_rps; + struct ci_ps current_ps; + struct radeon_ps requested_rps; + struct ci_ps requested_ps; + /* fan control */ + bool fan_ctrl_is_in_default_mode; + bool fan_is_controlled_by_smc; + u32 t_min; + u32 fan_ctrl_default_mode; +}; + +#define CISLANDS_VOLTAGE_CONTROL_NONE 0x0 +#define CISLANDS_VOLTAGE_CONTROL_BY_GPIO 0x1 +#define CISLANDS_VOLTAGE_CONTROL_BY_SVID2 0x2 + +#define CISLANDS_Q88_FORMAT_CONVERSION_UNIT 256 + +#define CISLANDS_VRC_DFLT0 0x3FFFC000 +#define CISLANDS_VRC_DFLT1 0x000400 +#define CISLANDS_VRC_DFLT2 0xC00080 +#define CISLANDS_VRC_DFLT3 0xC00200 +#define CISLANDS_VRC_DFLT4 0xC01680 +#define CISLANDS_VRC_DFLT5 0xC00033 +#define CISLANDS_VRC_DFLT6 0xC00033 +#define CISLANDS_VRC_DFLT7 0x3FFFC000 + +#define CISLANDS_CGULVPARAMETER_DFLT 0x00040035 +#define CISLAND_TARGETACTIVITY_DFLT 30 +#define CISLAND_MCLK_TARGETACTIVITY_DFLT 10 + +#define PCIE_PERF_REQ_REMOVE_REGISTRY 0 +#define PCIE_PERF_REQ_FORCE_LOWPOWER 1 +#define PCIE_PERF_REQ_PECI_GEN1 2 +#define PCIE_PERF_REQ_PECI_GEN2 3 +#define PCIE_PERF_REQ_PECI_GEN3 4 + +int ci_copy_bytes_to_smc(struct radeon_device *rdev, + u32 smc_start_address, + const u8 *src, u32 byte_count, u32 limit); +void ci_start_smc(struct radeon_device *rdev); +void ci_reset_smc(struct radeon_device *rdev); +int ci_program_jump_on_start(struct radeon_device *rdev); +void ci_stop_smc_clock(struct radeon_device *rdev); +void ci_start_smc_clock(struct radeon_device *rdev); +bool ci_is_smc_running(struct radeon_device *rdev); +PPSMC_Result ci_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg); +PPSMC_Result ci_wait_for_smc_inactive(struct radeon_device *rdev); +int ci_load_smc_ucode(struct radeon_device *rdev, u32 limit); +int ci_read_smc_sram_dword(struct radeon_device *rdev, + u32 smc_address, u32 *value, u32 limit); +int ci_write_smc_sram_dword(struct radeon_device *rdev, + u32 smc_address, u32 value, u32 limit); + +#endif diff --git a/sys/dev/pci/drm/radeon/ci_smc.c b/sys/dev/pci/drm/radeon/ci_smc.c new file mode 100644 index 00000000000..32fe3d07e34 --- /dev/null +++ b/sys/dev/pci/drm/radeon/ci_smc.c @@ -0,0 +1,294 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "cikd.h" +#include "ppsmc.h" +#include "radeon_ucode.h" +#include "ci_dpm.h" + +static int ci_set_smc_sram_address(struct radeon_device *rdev, + u32 smc_address, u32 limit) +{ + if (smc_address & 3) + return -EINVAL; + if ((smc_address + 3) > limit) + return -EINVAL; + + WREG32(SMC_IND_INDEX_0, smc_address); + WREG32_P(SMC_IND_ACCESS_CNTL, 0, ~AUTO_INCREMENT_IND_0); + + return 0; +} + +int ci_copy_bytes_to_smc(struct radeon_device *rdev, + u32 smc_start_address, + const u8 *src, u32 byte_count, u32 limit) +{ + unsigned long flags; + u32 data, original_data; + u32 addr; + u32 extra_shift; + int ret = 0; + + if (smc_start_address & 3) + return -EINVAL; + if ((smc_start_address + byte_count) > limit) + return -EINVAL; + + addr = smc_start_address; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + while (byte_count >= 4) { + /* SMC address space is BE */ + data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3]; + + ret = ci_set_smc_sram_address(rdev, addr, limit); + if (ret) + goto done; + + WREG32(SMC_IND_DATA_0, data); + + src += 4; + byte_count -= 4; + addr += 4; + } + + /* RMW for the final bytes */ + if (byte_count > 0) { + data = 0; + + ret = ci_set_smc_sram_address(rdev, addr, limit); + if (ret) + goto done; + + original_data = RREG32(SMC_IND_DATA_0); + + extra_shift = 8 * (4 - byte_count); + + while (byte_count > 0) { + data = (data << 8) + *src++; + byte_count--; + } + + data <<= extra_shift; + + data |= (original_data & ~((~0UL) << extra_shift)); + + ret = ci_set_smc_sram_address(rdev, addr, limit); + if (ret) + goto done; + + WREG32(SMC_IND_DATA_0, data); + } + +done: + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return ret; +} + +void ci_start_smc(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(SMC_SYSCON_RESET_CNTL); + + tmp &= ~RST_REG; + WREG32_SMC(SMC_SYSCON_RESET_CNTL, tmp); +} + +void ci_reset_smc(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(SMC_SYSCON_RESET_CNTL); + + tmp |= RST_REG; + WREG32_SMC(SMC_SYSCON_RESET_CNTL, tmp); +} + +int ci_program_jump_on_start(struct radeon_device *rdev) +{ + static const u8 data[] = { 0xE0, 0x00, 0x80, 0x40 }; + + return ci_copy_bytes_to_smc(rdev, 0x0, data, 4, sizeof(data)+1); +} + +void ci_stop_smc_clock(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0); + + tmp |= CK_DISABLE; + + WREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0, tmp); +} + +void ci_start_smc_clock(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0); + + tmp &= ~CK_DISABLE; + + WREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0, tmp); +} + +bool ci_is_smc_running(struct radeon_device *rdev) +{ + u32 clk = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0); + u32 pc_c = RREG32_SMC(SMC_PC_C); + + if (!(clk & CK_DISABLE) && (0x20100 <= pc_c)) + return true; + + return false; +} + +PPSMC_Result ci_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg) +{ + u32 tmp; + int i; + + if (!ci_is_smc_running(rdev)) + return PPSMC_Result_Failed; + + WREG32(SMC_MESSAGE_0, msg); + + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = RREG32(SMC_RESP_0); + if (tmp != 0) + break; + udelay(1); + } + tmp = RREG32(SMC_RESP_0); + + return (PPSMC_Result)tmp; +} + +#if 0 +PPSMC_Result ci_wait_for_smc_inactive(struct radeon_device *rdev) +{ + u32 tmp; + int i; + + if (!ci_is_smc_running(rdev)) + return PPSMC_Result_OK; + + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0); + if ((tmp & CKEN) == 0) + break; + udelay(1); + } + + return PPSMC_Result_OK; +} +#endif + +int ci_load_smc_ucode(struct radeon_device *rdev, u32 limit) +{ + unsigned long flags; + u32 ucode_start_address; + u32 ucode_size; + const u8 *src; + u32 data; + + if (!rdev->smc_fw) + return -EINVAL; + + if (rdev->new_fw) { + const struct smc_firmware_header_v1_0 *hdr = + (const struct smc_firmware_header_v1_0 *)rdev->smc_fw->data; + + radeon_ucode_print_smc_hdr(&hdr->header); + + ucode_start_address = le32_to_cpu(hdr->ucode_start_addr); + ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes); + src = (const u8 *) + (rdev->smc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + } else { + switch (rdev->family) { + case CHIP_BONAIRE: + ucode_start_address = BONAIRE_SMC_UCODE_START; + ucode_size = BONAIRE_SMC_UCODE_SIZE; + break; + case CHIP_HAWAII: + ucode_start_address = HAWAII_SMC_UCODE_START; + ucode_size = HAWAII_SMC_UCODE_SIZE; + break; + default: + DRM_ERROR("unknown asic in smc ucode loader\n"); + BUG(); + } + + src = (const u8 *)rdev->smc_fw->data; + } + + if (ucode_size & 3) + return -EINVAL; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + WREG32(SMC_IND_INDEX_0, ucode_start_address); + WREG32_P(SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, ~AUTO_INCREMENT_IND_0); + while (ucode_size >= 4) { + /* SMC address space is BE */ + data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3]; + + WREG32(SMC_IND_DATA_0, data); + + src += 4; + ucode_size -= 4; + } + WREG32_P(SMC_IND_ACCESS_CNTL, 0, ~AUTO_INCREMENT_IND_0); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return 0; +} + +int ci_read_smc_sram_dword(struct radeon_device *rdev, + u32 smc_address, u32 *value, u32 limit) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + ret = ci_set_smc_sram_address(rdev, smc_address, limit); + if (ret == 0) + *value = RREG32(SMC_IND_DATA_0); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return ret; +} + +int ci_write_smc_sram_dword(struct radeon_device *rdev, + u32 smc_address, u32 value, u32 limit) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + ret = ci_set_smc_sram_address(rdev, smc_address, limit); + if (ret == 0) + WREG32(SMC_IND_DATA_0, value); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return ret; +} diff --git a/sys/dev/pci/drm/radeon/cik.c b/sys/dev/pci/drm/radeon/cik.c new file mode 100644 index 00000000000..5ce8951c2bc --- /dev/null +++ b/sys/dev/pci/drm/radeon/cik.c @@ -0,0 +1,10078 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "radeon_audio.h" +#include "cikd.h" +#include "atom.h" +#include "cik_blit_shaders.h" +#include "radeon_ucode.h" +#include "clearstate_ci.h" +#include "radeon_kfd.h" + +MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); +MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); +MODULE_FIRMWARE("radeon/BONAIRE_ce.bin"); +MODULE_FIRMWARE("radeon/BONAIRE_mec.bin"); +MODULE_FIRMWARE("radeon/BONAIRE_mc.bin"); +MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin"); +MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin"); +MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin"); +MODULE_FIRMWARE("radeon/BONAIRE_smc.bin"); + +MODULE_FIRMWARE("radeon/bonaire_pfp.bin"); +MODULE_FIRMWARE("radeon/bonaire_me.bin"); +MODULE_FIRMWARE("radeon/bonaire_ce.bin"); +MODULE_FIRMWARE("radeon/bonaire_mec.bin"); +MODULE_FIRMWARE("radeon/bonaire_mc.bin"); +MODULE_FIRMWARE("radeon/bonaire_rlc.bin"); +MODULE_FIRMWARE("radeon/bonaire_sdma.bin"); +MODULE_FIRMWARE("radeon/bonaire_smc.bin"); + +MODULE_FIRMWARE("radeon/HAWAII_pfp.bin"); +MODULE_FIRMWARE("radeon/HAWAII_me.bin"); +MODULE_FIRMWARE("radeon/HAWAII_ce.bin"); +MODULE_FIRMWARE("radeon/HAWAII_mec.bin"); +MODULE_FIRMWARE("radeon/HAWAII_mc.bin"); +MODULE_FIRMWARE("radeon/HAWAII_mc2.bin"); +MODULE_FIRMWARE("radeon/HAWAII_rlc.bin"); +MODULE_FIRMWARE("radeon/HAWAII_sdma.bin"); +MODULE_FIRMWARE("radeon/HAWAII_smc.bin"); + +MODULE_FIRMWARE("radeon/hawaii_pfp.bin"); +MODULE_FIRMWARE("radeon/hawaii_me.bin"); +MODULE_FIRMWARE("radeon/hawaii_ce.bin"); +MODULE_FIRMWARE("radeon/hawaii_mec.bin"); +MODULE_FIRMWARE("radeon/hawaii_mc.bin"); +MODULE_FIRMWARE("radeon/hawaii_rlc.bin"); +MODULE_FIRMWARE("radeon/hawaii_sdma.bin"); +MODULE_FIRMWARE("radeon/hawaii_smc.bin"); + +MODULE_FIRMWARE("radeon/KAVERI_pfp.bin"); +MODULE_FIRMWARE("radeon/KAVERI_me.bin"); +MODULE_FIRMWARE("radeon/KAVERI_ce.bin"); +MODULE_FIRMWARE("radeon/KAVERI_mec.bin"); +MODULE_FIRMWARE("radeon/KAVERI_rlc.bin"); +MODULE_FIRMWARE("radeon/KAVERI_sdma.bin"); + +MODULE_FIRMWARE("radeon/kaveri_pfp.bin"); +MODULE_FIRMWARE("radeon/kaveri_me.bin"); +MODULE_FIRMWARE("radeon/kaveri_ce.bin"); +MODULE_FIRMWARE("radeon/kaveri_mec.bin"); +MODULE_FIRMWARE("radeon/kaveri_mec2.bin"); +MODULE_FIRMWARE("radeon/kaveri_rlc.bin"); +MODULE_FIRMWARE("radeon/kaveri_sdma.bin"); + +MODULE_FIRMWARE("radeon/KABINI_pfp.bin"); +MODULE_FIRMWARE("radeon/KABINI_me.bin"); +MODULE_FIRMWARE("radeon/KABINI_ce.bin"); +MODULE_FIRMWARE("radeon/KABINI_mec.bin"); +MODULE_FIRMWARE("radeon/KABINI_rlc.bin"); +MODULE_FIRMWARE("radeon/KABINI_sdma.bin"); + +MODULE_FIRMWARE("radeon/kabini_pfp.bin"); +MODULE_FIRMWARE("radeon/kabini_me.bin"); +MODULE_FIRMWARE("radeon/kabini_ce.bin"); +MODULE_FIRMWARE("radeon/kabini_mec.bin"); +MODULE_FIRMWARE("radeon/kabini_rlc.bin"); +MODULE_FIRMWARE("radeon/kabini_sdma.bin"); + +MODULE_FIRMWARE("radeon/MULLINS_pfp.bin"); +MODULE_FIRMWARE("radeon/MULLINS_me.bin"); +MODULE_FIRMWARE("radeon/MULLINS_ce.bin"); +MODULE_FIRMWARE("radeon/MULLINS_mec.bin"); +MODULE_FIRMWARE("radeon/MULLINS_rlc.bin"); +MODULE_FIRMWARE("radeon/MULLINS_sdma.bin"); + +MODULE_FIRMWARE("radeon/mullins_pfp.bin"); +MODULE_FIRMWARE("radeon/mullins_me.bin"); +MODULE_FIRMWARE("radeon/mullins_ce.bin"); +MODULE_FIRMWARE("radeon/mullins_mec.bin"); +MODULE_FIRMWARE("radeon/mullins_rlc.bin"); +MODULE_FIRMWARE("radeon/mullins_sdma.bin"); + +extern int r600_ih_ring_alloc(struct radeon_device *rdev); +extern void r600_ih_ring_fini(struct radeon_device *rdev); +extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); +extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); +extern bool evergreen_is_display_hung(struct radeon_device *rdev); +extern void sumo_rlc_fini(struct radeon_device *rdev); +extern int sumo_rlc_init(struct radeon_device *rdev); +extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc); +extern void si_rlc_reset(struct radeon_device *rdev); +extern void si_init_uvd_internal_cg(struct radeon_device *rdev); +static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); +extern int cik_sdma_resume(struct radeon_device *rdev); +extern void cik_sdma_enable(struct radeon_device *rdev, bool enable); +extern void cik_sdma_fini(struct radeon_device *rdev); +extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable); +static void cik_rlc_stop(struct radeon_device *rdev); +static void cik_pcie_gen3_enable(struct radeon_device *rdev); +static void cik_program_aspm(struct radeon_device *rdev); +static void cik_init_pg(struct radeon_device *rdev); +static void cik_init_cg(struct radeon_device *rdev); +static void cik_fini_pg(struct radeon_device *rdev); +static void cik_fini_cg(struct radeon_device *rdev); +static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, + bool enable); + +/** + * cik_get_allowed_info_register - fetch the register for the info ioctl + * + * @rdev: radeon_device pointer + * @reg: register offset in bytes + * @val: register value + * + * Returns 0 for success or -EINVAL for an invalid register + * + */ +int cik_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val) +{ + switch (reg) { + case GRBM_STATUS: + case GRBM_STATUS2: + case GRBM_STATUS_SE0: + case GRBM_STATUS_SE1: + case GRBM_STATUS_SE2: + case GRBM_STATUS_SE3: + case SRBM_STATUS: + case SRBM_STATUS2: + case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET): + case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET): + case UVD_STATUS: + /* TODO VCE */ + *val = RREG32(reg); + return 0; + default: + return -EINVAL; + } +} + +/* + * Indirect registers accessor + */ +u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->didt_idx_lock, flags); + WREG32(CIK_DIDT_IND_INDEX, (reg)); + r = RREG32(CIK_DIDT_IND_DATA); + spin_unlock_irqrestore(&rdev->didt_idx_lock, flags); + return r; +} + +void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->didt_idx_lock, flags); + WREG32(CIK_DIDT_IND_INDEX, (reg)); + WREG32(CIK_DIDT_IND_DATA, (v)); + spin_unlock_irqrestore(&rdev->didt_idx_lock, flags); +} + +/* get temperature in millidegrees */ +int ci_get_temp(struct radeon_device *rdev) +{ + u32 temp; + int actual_temp = 0; + + temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> + CTF_TEMP_SHIFT; + + if (temp & 0x200) + actual_temp = 255; + else + actual_temp = temp & 0x1ff; + + actual_temp = actual_temp * 1000; + + return actual_temp; +} + +/* get temperature in millidegrees */ +int kv_get_temp(struct radeon_device *rdev) +{ + u32 temp; + int actual_temp = 0; + + temp = RREG32_SMC(0xC0300E0C); + + if (temp) + actual_temp = (temp / 8) - 49; + else + actual_temp = 0; + + actual_temp = actual_temp * 1000; + + return actual_temp; +} + +/* + * Indirect registers accessor + */ +u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->pciep_idx_lock, flags); + WREG32(PCIE_INDEX, reg); + (void)RREG32(PCIE_INDEX); + r = RREG32(PCIE_DATA); + spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); + return r; +} + +void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->pciep_idx_lock, flags); + WREG32(PCIE_INDEX, reg); + (void)RREG32(PCIE_INDEX); + WREG32(PCIE_DATA, v); + (void)RREG32(PCIE_DATA); + spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); +} + +static const u32 spectre_rlc_save_restore_register_list[] = +{ + (0x0e00 << 16) | (0xc12c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc140 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc150 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc15c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc168 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc170 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc178 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc204 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2b4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2b8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2bc >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2c0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8228 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x829c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x869c >> 2), + 0x00000000, + (0x0600 << 16) | (0x98f4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x98f8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9900 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc260 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x90e8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c000 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c00c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c1c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9700 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x4e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x8e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x9e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0xae00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0xbe00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x89bc >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8900 >> 2), + 0x00000000, + 0x3, + (0x0e00 << 16) | (0xc130 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc134 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc1fc >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc208 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc264 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc268 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc26c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc270 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc274 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc278 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc27c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc280 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc284 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc288 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc28c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc290 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc294 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc298 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc29c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2a0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2a4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2a8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2ac >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2b0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x301d0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30238 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30250 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30254 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30258 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3025c >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x8e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x9e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0xae00 << 16) | (0xc900 >> 2), + 0x00000000, + (0xbe00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x8e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x9e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0xae00 << 16) | (0xc904 >> 2), + 0x00000000, + (0xbe00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x8e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x9e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0xae00 << 16) | (0xc908 >> 2), + 0x00000000, + (0xbe00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x8e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x9e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0xae00 << 16) | (0xc90c >> 2), + 0x00000000, + (0xbe00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x8e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x9e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0xae00 << 16) | (0xc910 >> 2), + 0x00000000, + (0xbe00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc99c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9834 >> 2), + 0x00000000, + (0x0000 << 16) | (0x30f00 >> 2), + 0x00000000, + (0x0001 << 16) | (0x30f00 >> 2), + 0x00000000, + (0x0000 << 16) | (0x30f04 >> 2), + 0x00000000, + (0x0001 << 16) | (0x30f04 >> 2), + 0x00000000, + (0x0000 << 16) | (0x30f08 >> 2), + 0x00000000, + (0x0001 << 16) | (0x30f08 >> 2), + 0x00000000, + (0x0000 << 16) | (0x30f0c >> 2), + 0x00000000, + (0x0001 << 16) | (0x30f0c >> 2), + 0x00000000, + (0x0600 << 16) | (0x9b7c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8a14 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8a18 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a00 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8bf0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8bcc >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8b24 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30a04 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a10 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a14 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a18 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a2c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc700 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc704 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc708 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc768 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc770 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc774 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc778 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc77c >> 2), + 0x00000000, + (0x0400 << 16) | (0xc780 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc784 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc788 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc78c >> 2), + 0x00000000, + (0x0400 << 16) | (0xc798 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc79c >> 2), + 0x00000000, + (0x0400 << 16) | (0xc7a0 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc7a4 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc7a8 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc7ac >> 2), + 0x00000000, + (0x0400 << 16) | (0xc7b0 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc7b4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9100 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c010 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92a8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92ac >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92b4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92b8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92bc >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92c0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92c4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92c8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92cc >> 2), + 0x00000000, + (0x0e00 << 16) | (0x92d0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c00 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c04 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c20 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c38 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c3c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xae00 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9604 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac08 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac0c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac10 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac14 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac58 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac68 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac6c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac70 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac74 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac78 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac7c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac80 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac84 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac88 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac8c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x970c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9714 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9718 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x971c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x4e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x5e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x6e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x7e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x8e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x9e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0xae00 << 16) | (0x31068 >> 2), + 0x00000000, + (0xbe00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xcd10 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xcd14 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88b0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88b4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88b8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88bc >> 2), + 0x00000000, + (0x0400 << 16) | (0x89c0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88c4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88c8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88d0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88d4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88d8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8980 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30938 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3093c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30940 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x89a0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30900 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30904 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x89b4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c210 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c214 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c218 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8904 >> 2), + 0x00000000, + 0x5, + (0x0e00 << 16) | (0x8c28 >> 2), + (0x0e00 << 16) | (0x8c2c >> 2), + (0x0e00 << 16) | (0x8c30 >> 2), + (0x0e00 << 16) | (0x8c34 >> 2), + (0x0e00 << 16) | (0x9600 >> 2), +}; + +static const u32 kalindi_rlc_save_restore_register_list[] = +{ + (0x0e00 << 16) | (0xc12c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc140 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc150 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc15c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc168 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc170 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc204 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2b4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2b8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2bc >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2c0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8228 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x829c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x869c >> 2), + 0x00000000, + (0x0600 << 16) | (0x98f4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x98f8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9900 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc260 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x90e8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c000 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c00c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c1c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9700 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x4e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xcd20 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x89bc >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8900 >> 2), + 0x00000000, + 0x3, + (0x0e00 << 16) | (0xc130 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc134 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc1fc >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc208 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc264 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc268 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc26c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc270 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc274 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc28c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc290 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc294 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc298 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2a0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2a4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2a8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc2ac >> 2), + 0x00000000, + (0x0e00 << 16) | (0x301d0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30238 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30250 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30254 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30258 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3025c >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc900 >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc904 >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc908 >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc90c >> 2), + 0x00000000, + (0x4e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x5e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x6e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x7e00 << 16) | (0xc910 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc99c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9834 >> 2), + 0x00000000, + (0x0000 << 16) | (0x30f00 >> 2), + 0x00000000, + (0x0000 << 16) | (0x30f04 >> 2), + 0x00000000, + (0x0000 << 16) | (0x30f08 >> 2), + 0x00000000, + (0x0000 << 16) | (0x30f0c >> 2), + 0x00000000, + (0x0600 << 16) | (0x9b7c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8a14 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8a18 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a00 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8bf0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8bcc >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8b24 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30a04 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a10 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a14 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a18 >> 2), + 0x00000000, + (0x0600 << 16) | (0x30a2c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc700 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc704 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc708 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xc768 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc770 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc774 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc798 >> 2), + 0x00000000, + (0x0400 << 16) | (0xc79c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9100 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c010 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c00 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c04 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c20 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c38 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8c3c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xae00 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9604 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac08 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac0c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac10 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac14 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac58 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac68 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac6c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac70 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac74 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac78 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac7c >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac80 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac84 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac88 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xac8c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x970c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9714 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x9718 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x971c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x4e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x5e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x6e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x7e00 << 16) | (0x31068 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xcd10 >> 2), + 0x00000000, + (0x0e00 << 16) | (0xcd14 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88b0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88b4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88b8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88bc >> 2), + 0x00000000, + (0x0400 << 16) | (0x89c0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88c4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88c8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88d0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88d4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x88d8 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8980 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30938 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3093c >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30940 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x89a0 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30900 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x30904 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x89b4 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3e1fc >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c210 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c214 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x3c218 >> 2), + 0x00000000, + (0x0e00 << 16) | (0x8904 >> 2), + 0x00000000, + 0x5, + (0x0e00 << 16) | (0x8c28 >> 2), + (0x0e00 << 16) | (0x8c2c >> 2), + (0x0e00 << 16) | (0x8c30 >> 2), + (0x0e00 << 16) | (0x8c34 >> 2), + (0x0e00 << 16) | (0x9600 >> 2), +}; + +static const u32 bonaire_golden_spm_registers[] = +{ + 0x30800, 0xe0ffffff, 0xe0000000 +}; + +static const u32 bonaire_golden_common_registers[] = +{ + 0xc770, 0xffffffff, 0x00000800, + 0xc774, 0xffffffff, 0x00000800, + 0xc798, 0xffffffff, 0x00007fbf, + 0xc79c, 0xffffffff, 0x00007faf +}; + +static const u32 bonaire_golden_registers[] = +{ + 0x3354, 0x00000333, 0x00000333, + 0x3350, 0x000c0fc0, 0x00040200, + 0x9a10, 0x00010000, 0x00058208, + 0x3c000, 0xffff1fff, 0x00140000, + 0x3c200, 0xfdfc0fff, 0x00000100, + 0x3c234, 0x40000000, 0x40000200, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0x0002021c, 0x00020200, + 0xc78, 0x00000080, 0x00000000, + 0x5bb0, 0x000000f0, 0x00000070, + 0x5bc0, 0xf0311fff, 0x80300000, + 0x98f8, 0x73773777, 0x12010001, + 0x350c, 0x00810000, 0x408af000, + 0x7030, 0x31000111, 0x00000011, + 0x2f48, 0x73773777, 0x12010001, + 0x220c, 0x00007fb6, 0x0021a1b1, + 0x2210, 0x00007fb6, 0x002021b1, + 0x2180, 0x00007fb6, 0x00002191, + 0x2218, 0x00007fb6, 0x002121b1, + 0x221c, 0x00007fb6, 0x002021b1, + 0x21dc, 0x00007fb6, 0x00002191, + 0x21e0, 0x00007fb6, 0x00002191, + 0x3628, 0x0000003f, 0x0000000a, + 0x362c, 0x0000003f, 0x0000000a, + 0x2ae4, 0x00073ffe, 0x000022a2, + 0x240c, 0x000007ff, 0x00000000, + 0x8a14, 0xf000003f, 0x00000007, + 0x8bf0, 0x00002001, 0x00000001, + 0x8b24, 0xffffffff, 0x00ffffff, + 0x30a04, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x4d8, 0x00000fff, 0x00000100, + 0x3e78, 0x00000001, 0x00000002, + 0x9100, 0x03000000, 0x0362c688, + 0x8c00, 0x000000ff, 0x00000001, + 0xe40, 0x00001fff, 0x00001fff, + 0x9060, 0x0000007f, 0x00000020, + 0x9508, 0x00010000, 0x00010000, + 0xac14, 0x000003ff, 0x000000f3, + 0xac0c, 0xffffffff, 0x00001032 +}; + +static const u32 bonaire_mgcg_cgcg_init[] = +{ + 0xc420, 0xffffffff, 0xfffffffc, + 0x30800, 0xffffffff, 0xe0000000, + 0x3c2a0, 0xffffffff, 0x00000100, + 0x3c208, 0xffffffff, 0x00000100, + 0x3c2c0, 0xffffffff, 0xc0000100, + 0x3c2c8, 0xffffffff, 0xc0000100, + 0x3c2c4, 0xffffffff, 0xc0000100, + 0x55e4, 0xffffffff, 0x00600100, + 0x3c280, 0xffffffff, 0x00000100, + 0x3c214, 0xffffffff, 0x06000100, + 0x3c220, 0xffffffff, 0x00000100, + 0x3c218, 0xffffffff, 0x06000100, + 0x3c204, 0xffffffff, 0x00000100, + 0x3c2e0, 0xffffffff, 0x00000100, + 0x3c224, 0xffffffff, 0x00000100, + 0x3c200, 0xffffffff, 0x00000100, + 0x3c230, 0xffffffff, 0x00000100, + 0x3c234, 0xffffffff, 0x00000100, + 0x3c250, 0xffffffff, 0x00000100, + 0x3c254, 0xffffffff, 0x00000100, + 0x3c258, 0xffffffff, 0x00000100, + 0x3c25c, 0xffffffff, 0x00000100, + 0x3c260, 0xffffffff, 0x00000100, + 0x3c27c, 0xffffffff, 0x00000100, + 0x3c278, 0xffffffff, 0x00000100, + 0x3c210, 0xffffffff, 0x06000100, + 0x3c290, 0xffffffff, 0x00000100, + 0x3c274, 0xffffffff, 0x00000100, + 0x3c2b4, 0xffffffff, 0x00000100, + 0x3c2b0, 0xffffffff, 0x00000100, + 0x3c270, 0xffffffff, 0x00000100, + 0x30800, 0xffffffff, 0xe0000000, + 0x3c020, 0xffffffff, 0x00010000, + 0x3c024, 0xffffffff, 0x00030002, + 0x3c028, 0xffffffff, 0x00040007, + 0x3c02c, 0xffffffff, 0x00060005, + 0x3c030, 0xffffffff, 0x00090008, + 0x3c034, 0xffffffff, 0x00010000, + 0x3c038, 0xffffffff, 0x00030002, + 0x3c03c, 0xffffffff, 0x00040007, + 0x3c040, 0xffffffff, 0x00060005, + 0x3c044, 0xffffffff, 0x00090008, + 0x3c048, 0xffffffff, 0x00010000, + 0x3c04c, 0xffffffff, 0x00030002, + 0x3c050, 0xffffffff, 0x00040007, + 0x3c054, 0xffffffff, 0x00060005, + 0x3c058, 0xffffffff, 0x00090008, + 0x3c05c, 0xffffffff, 0x00010000, + 0x3c060, 0xffffffff, 0x00030002, + 0x3c064, 0xffffffff, 0x00040007, + 0x3c068, 0xffffffff, 0x00060005, + 0x3c06c, 0xffffffff, 0x00090008, + 0x3c070, 0xffffffff, 0x00010000, + 0x3c074, 0xffffffff, 0x00030002, + 0x3c078, 0xffffffff, 0x00040007, + 0x3c07c, 0xffffffff, 0x00060005, + 0x3c080, 0xffffffff, 0x00090008, + 0x3c084, 0xffffffff, 0x00010000, + 0x3c088, 0xffffffff, 0x00030002, + 0x3c08c, 0xffffffff, 0x00040007, + 0x3c090, 0xffffffff, 0x00060005, + 0x3c094, 0xffffffff, 0x00090008, + 0x3c098, 0xffffffff, 0x00010000, + 0x3c09c, 0xffffffff, 0x00030002, + 0x3c0a0, 0xffffffff, 0x00040007, + 0x3c0a4, 0xffffffff, 0x00060005, + 0x3c0a8, 0xffffffff, 0x00090008, + 0x3c000, 0xffffffff, 0x96e00200, + 0x8708, 0xffffffff, 0x00900100, + 0xc424, 0xffffffff, 0x0020003f, + 0x38, 0xffffffff, 0x0140001c, + 0x3c, 0x000f0000, 0x000f0000, + 0x220, 0xffffffff, 0xC060000C, + 0x224, 0xc0000fff, 0x00000100, + 0xf90, 0xffffffff, 0x00000100, + 0xf98, 0x00000101, 0x00000000, + 0x20a8, 0xffffffff, 0x00000104, + 0x55e4, 0xff000fff, 0x00000100, + 0x30cc, 0xc0000fff, 0x00000104, + 0xc1e4, 0x00000001, 0x00000001, + 0xd00c, 0xff000ff0, 0x00000100, + 0xd80c, 0xff000ff0, 0x00000100 +}; + +static const u32 spectre_golden_spm_registers[] = +{ + 0x30800, 0xe0ffffff, 0xe0000000 +}; + +static const u32 spectre_golden_common_registers[] = +{ + 0xc770, 0xffffffff, 0x00000800, + 0xc774, 0xffffffff, 0x00000800, + 0xc798, 0xffffffff, 0x00007fbf, + 0xc79c, 0xffffffff, 0x00007faf +}; + +static const u32 spectre_golden_registers[] = +{ + 0x3c000, 0xffff1fff, 0x96940200, + 0x3c00c, 0xffff0001, 0xff000000, + 0x3c200, 0xfffc0fff, 0x00000100, + 0x6ed8, 0x00010101, 0x00010000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0xfffffffc, 0x00020200, + 0x5bb0, 0x000000f0, 0x00000070, + 0x5bc0, 0xf0311fff, 0x80300000, + 0x98f8, 0x73773777, 0x12010001, + 0x9b7c, 0x00ff0000, 0x00fc0000, + 0x2f48, 0x73773777, 0x12010001, + 0x8a14, 0xf000003f, 0x00000007, + 0x8b24, 0xffffffff, 0x00ffffff, + 0x28350, 0x3f3f3fff, 0x00000082, + 0x28354, 0x0000003f, 0x00000000, + 0x3e78, 0x00000001, 0x00000002, + 0x913c, 0xffff03df, 0x00000004, + 0xc768, 0x00000008, 0x00000008, + 0x8c00, 0x000008ff, 0x00000800, + 0x9508, 0x00010000, 0x00010000, + 0xac0c, 0xffffffff, 0x54763210, + 0x214f8, 0x01ff01ff, 0x00000002, + 0x21498, 0x007ff800, 0x00200000, + 0x2015c, 0xffffffff, 0x00000f40, + 0x30934, 0xffffffff, 0x00000001 +}; + +static const u32 spectre_mgcg_cgcg_init[] = +{ + 0xc420, 0xffffffff, 0xfffffffc, + 0x30800, 0xffffffff, 0xe0000000, + 0x3c2a0, 0xffffffff, 0x00000100, + 0x3c208, 0xffffffff, 0x00000100, + 0x3c2c0, 0xffffffff, 0x00000100, + 0x3c2c8, 0xffffffff, 0x00000100, + 0x3c2c4, 0xffffffff, 0x00000100, + 0x55e4, 0xffffffff, 0x00600100, + 0x3c280, 0xffffffff, 0x00000100, + 0x3c214, 0xffffffff, 0x06000100, + 0x3c220, 0xffffffff, 0x00000100, + 0x3c218, 0xffffffff, 0x06000100, + 0x3c204, 0xffffffff, 0x00000100, + 0x3c2e0, 0xffffffff, 0x00000100, + 0x3c224, 0xffffffff, 0x00000100, + 0x3c200, 0xffffffff, 0x00000100, + 0x3c230, 0xffffffff, 0x00000100, + 0x3c234, 0xffffffff, 0x00000100, + 0x3c250, 0xffffffff, 0x00000100, + 0x3c254, 0xffffffff, 0x00000100, + 0x3c258, 0xffffffff, 0x00000100, + 0x3c25c, 0xffffffff, 0x00000100, + 0x3c260, 0xffffffff, 0x00000100, + 0x3c27c, 0xffffffff, 0x00000100, + 0x3c278, 0xffffffff, 0x00000100, + 0x3c210, 0xffffffff, 0x06000100, + 0x3c290, 0xffffffff, 0x00000100, + 0x3c274, 0xffffffff, 0x00000100, + 0x3c2b4, 0xffffffff, 0x00000100, + 0x3c2b0, 0xffffffff, 0x00000100, + 0x3c270, 0xffffffff, 0x00000100, + 0x30800, 0xffffffff, 0xe0000000, + 0x3c020, 0xffffffff, 0x00010000, + 0x3c024, 0xffffffff, 0x00030002, + 0x3c028, 0xffffffff, 0x00040007, + 0x3c02c, 0xffffffff, 0x00060005, + 0x3c030, 0xffffffff, 0x00090008, + 0x3c034, 0xffffffff, 0x00010000, + 0x3c038, 0xffffffff, 0x00030002, + 0x3c03c, 0xffffffff, 0x00040007, + 0x3c040, 0xffffffff, 0x00060005, + 0x3c044, 0xffffffff, 0x00090008, + 0x3c048, 0xffffffff, 0x00010000, + 0x3c04c, 0xffffffff, 0x00030002, + 0x3c050, 0xffffffff, 0x00040007, + 0x3c054, 0xffffffff, 0x00060005, + 0x3c058, 0xffffffff, 0x00090008, + 0x3c05c, 0xffffffff, 0x00010000, + 0x3c060, 0xffffffff, 0x00030002, + 0x3c064, 0xffffffff, 0x00040007, + 0x3c068, 0xffffffff, 0x00060005, + 0x3c06c, 0xffffffff, 0x00090008, + 0x3c070, 0xffffffff, 0x00010000, + 0x3c074, 0xffffffff, 0x00030002, + 0x3c078, 0xffffffff, 0x00040007, + 0x3c07c, 0xffffffff, 0x00060005, + 0x3c080, 0xffffffff, 0x00090008, + 0x3c084, 0xffffffff, 0x00010000, + 0x3c088, 0xffffffff, 0x00030002, + 0x3c08c, 0xffffffff, 0x00040007, + 0x3c090, 0xffffffff, 0x00060005, + 0x3c094, 0xffffffff, 0x00090008, + 0x3c098, 0xffffffff, 0x00010000, + 0x3c09c, 0xffffffff, 0x00030002, + 0x3c0a0, 0xffffffff, 0x00040007, + 0x3c0a4, 0xffffffff, 0x00060005, + 0x3c0a8, 0xffffffff, 0x00090008, + 0x3c0ac, 0xffffffff, 0x00010000, + 0x3c0b0, 0xffffffff, 0x00030002, + 0x3c0b4, 0xffffffff, 0x00040007, + 0x3c0b8, 0xffffffff, 0x00060005, + 0x3c0bc, 0xffffffff, 0x00090008, + 0x3c000, 0xffffffff, 0x96e00200, + 0x8708, 0xffffffff, 0x00900100, + 0xc424, 0xffffffff, 0x0020003f, + 0x38, 0xffffffff, 0x0140001c, + 0x3c, 0x000f0000, 0x000f0000, + 0x220, 0xffffffff, 0xC060000C, + 0x224, 0xc0000fff, 0x00000100, + 0xf90, 0xffffffff, 0x00000100, + 0xf98, 0x00000101, 0x00000000, + 0x20a8, 0xffffffff, 0x00000104, + 0x55e4, 0xff000fff, 0x00000100, + 0x30cc, 0xc0000fff, 0x00000104, + 0xc1e4, 0x00000001, 0x00000001, + 0xd00c, 0xff000ff0, 0x00000100, + 0xd80c, 0xff000ff0, 0x00000100 +}; + +static const u32 kalindi_golden_spm_registers[] = +{ + 0x30800, 0xe0ffffff, 0xe0000000 +}; + +static const u32 kalindi_golden_common_registers[] = +{ + 0xc770, 0xffffffff, 0x00000800, + 0xc774, 0xffffffff, 0x00000800, + 0xc798, 0xffffffff, 0x00007fbf, + 0xc79c, 0xffffffff, 0x00007faf +}; + +static const u32 kalindi_golden_registers[] = +{ + 0x3c000, 0xffffdfff, 0x6e944040, + 0x55e4, 0xff607fff, 0xfc000100, + 0x3c220, 0xff000fff, 0x00000100, + 0x3c224, 0xff000fff, 0x00000100, + 0x3c200, 0xfffc0fff, 0x00000100, + 0x6ed8, 0x00010101, 0x00010000, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x5bb0, 0x000000f0, 0x00000070, + 0x5bc0, 0xf0311fff, 0x80300000, + 0x98f8, 0x73773777, 0x12010001, + 0x98fc, 0xffffffff, 0x00000010, + 0x9b7c, 0x00ff0000, 0x00fc0000, + 0x8030, 0x00001f0f, 0x0000100a, + 0x2f48, 0x73773777, 0x12010001, + 0x2408, 0x000fffff, 0x000c007f, + 0x8a14, 0xf000003f, 0x00000007, + 0x8b24, 0x3fff3fff, 0x00ffcfff, + 0x30a04, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x4d8, 0x00000fff, 0x00000100, + 0x3e78, 0x00000001, 0x00000002, + 0xc768, 0x00000008, 0x00000008, + 0x8c00, 0x000000ff, 0x00000003, + 0x214f8, 0x01ff01ff, 0x00000002, + 0x21498, 0x007ff800, 0x00200000, + 0x2015c, 0xffffffff, 0x00000f40, + 0x88c4, 0x001f3ae3, 0x00000082, + 0x88d4, 0x0000001f, 0x00000010, + 0x30934, 0xffffffff, 0x00000000 +}; + +static const u32 kalindi_mgcg_cgcg_init[] = +{ + 0xc420, 0xffffffff, 0xfffffffc, + 0x30800, 0xffffffff, 0xe0000000, + 0x3c2a0, 0xffffffff, 0x00000100, + 0x3c208, 0xffffffff, 0x00000100, + 0x3c2c0, 0xffffffff, 0x00000100, + 0x3c2c8, 0xffffffff, 0x00000100, + 0x3c2c4, 0xffffffff, 0x00000100, + 0x55e4, 0xffffffff, 0x00600100, + 0x3c280, 0xffffffff, 0x00000100, + 0x3c214, 0xffffffff, 0x06000100, + 0x3c220, 0xffffffff, 0x00000100, + 0x3c218, 0xffffffff, 0x06000100, + 0x3c204, 0xffffffff, 0x00000100, + 0x3c2e0, 0xffffffff, 0x00000100, + 0x3c224, 0xffffffff, 0x00000100, + 0x3c200, 0xffffffff, 0x00000100, + 0x3c230, 0xffffffff, 0x00000100, + 0x3c234, 0xffffffff, 0x00000100, + 0x3c250, 0xffffffff, 0x00000100, + 0x3c254, 0xffffffff, 0x00000100, + 0x3c258, 0xffffffff, 0x00000100, + 0x3c25c, 0xffffffff, 0x00000100, + 0x3c260, 0xffffffff, 0x00000100, + 0x3c27c, 0xffffffff, 0x00000100, + 0x3c278, 0xffffffff, 0x00000100, + 0x3c210, 0xffffffff, 0x06000100, + 0x3c290, 0xffffffff, 0x00000100, + 0x3c274, 0xffffffff, 0x00000100, + 0x3c2b4, 0xffffffff, 0x00000100, + 0x3c2b0, 0xffffffff, 0x00000100, + 0x3c270, 0xffffffff, 0x00000100, + 0x30800, 0xffffffff, 0xe0000000, + 0x3c020, 0xffffffff, 0x00010000, + 0x3c024, 0xffffffff, 0x00030002, + 0x3c028, 0xffffffff, 0x00040007, + 0x3c02c, 0xffffffff, 0x00060005, + 0x3c030, 0xffffffff, 0x00090008, + 0x3c034, 0xffffffff, 0x00010000, + 0x3c038, 0xffffffff, 0x00030002, + 0x3c03c, 0xffffffff, 0x00040007, + 0x3c040, 0xffffffff, 0x00060005, + 0x3c044, 0xffffffff, 0x00090008, + 0x3c000, 0xffffffff, 0x96e00200, + 0x8708, 0xffffffff, 0x00900100, + 0xc424, 0xffffffff, 0x0020003f, + 0x38, 0xffffffff, 0x0140001c, + 0x3c, 0x000f0000, 0x000f0000, + 0x220, 0xffffffff, 0xC060000C, + 0x224, 0xc0000fff, 0x00000100, + 0x20a8, 0xffffffff, 0x00000104, + 0x55e4, 0xff000fff, 0x00000100, + 0x30cc, 0xc0000fff, 0x00000104, + 0xc1e4, 0x00000001, 0x00000001, + 0xd00c, 0xff000ff0, 0x00000100, + 0xd80c, 0xff000ff0, 0x00000100 +}; + +static const u32 hawaii_golden_spm_registers[] = +{ + 0x30800, 0xe0ffffff, 0xe0000000 +}; + +static const u32 hawaii_golden_common_registers[] = +{ + 0x30800, 0xffffffff, 0xe0000000, + 0x28350, 0xffffffff, 0x3a00161a, + 0x28354, 0xffffffff, 0x0000002e, + 0x9a10, 0xffffffff, 0x00018208, + 0x98f8, 0xffffffff, 0x12011003 +}; + +static const u32 hawaii_golden_registers[] = +{ + 0x3354, 0x00000333, 0x00000333, + 0x9a10, 0x00010000, 0x00058208, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0x0002021c, 0x00020200, + 0xc78, 0x00000080, 0x00000000, + 0x5bb0, 0x000000f0, 0x00000070, + 0x5bc0, 0xf0311fff, 0x80300000, + 0x350c, 0x00810000, 0x408af000, + 0x7030, 0x31000111, 0x00000011, + 0x2f48, 0x73773777, 0x12010001, + 0x2120, 0x0000007f, 0x0000001b, + 0x21dc, 0x00007fb6, 0x00002191, + 0x3628, 0x0000003f, 0x0000000a, + 0x362c, 0x0000003f, 0x0000000a, + 0x2ae4, 0x00073ffe, 0x000022a2, + 0x240c, 0x000007ff, 0x00000000, + 0x8bf0, 0x00002001, 0x00000001, + 0x8b24, 0xffffffff, 0x00ffffff, + 0x30a04, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x3e78, 0x00000001, 0x00000002, + 0xc768, 0x00000008, 0x00000008, + 0xc770, 0x00000f00, 0x00000800, + 0xc774, 0x00000f00, 0x00000800, + 0xc798, 0x00ffffff, 0x00ff7fbf, + 0xc79c, 0x00ffffff, 0x00ff7faf, + 0x8c00, 0x000000ff, 0x00000800, + 0xe40, 0x00001fff, 0x00001fff, + 0x9060, 0x0000007f, 0x00000020, + 0x9508, 0x00010000, 0x00010000, + 0xae00, 0x00100000, 0x000ff07c, + 0xac14, 0x000003ff, 0x0000000f, + 0xac10, 0xffffffff, 0x7564fdec, + 0xac0c, 0xffffffff, 0x3120b9a8, + 0xac08, 0x20000000, 0x0f9c0000 +}; + +static const u32 hawaii_mgcg_cgcg_init[] = +{ + 0xc420, 0xffffffff, 0xfffffffd, + 0x30800, 0xffffffff, 0xe0000000, + 0x3c2a0, 0xffffffff, 0x00000100, + 0x3c208, 0xffffffff, 0x00000100, + 0x3c2c0, 0xffffffff, 0x00000100, + 0x3c2c8, 0xffffffff, 0x00000100, + 0x3c2c4, 0xffffffff, 0x00000100, + 0x55e4, 0xffffffff, 0x00200100, + 0x3c280, 0xffffffff, 0x00000100, + 0x3c214, 0xffffffff, 0x06000100, + 0x3c220, 0xffffffff, 0x00000100, + 0x3c218, 0xffffffff, 0x06000100, + 0x3c204, 0xffffffff, 0x00000100, + 0x3c2e0, 0xffffffff, 0x00000100, + 0x3c224, 0xffffffff, 0x00000100, + 0x3c200, 0xffffffff, 0x00000100, + 0x3c230, 0xffffffff, 0x00000100, + 0x3c234, 0xffffffff, 0x00000100, + 0x3c250, 0xffffffff, 0x00000100, + 0x3c254, 0xffffffff, 0x00000100, + 0x3c258, 0xffffffff, 0x00000100, + 0x3c25c, 0xffffffff, 0x00000100, + 0x3c260, 0xffffffff, 0x00000100, + 0x3c27c, 0xffffffff, 0x00000100, + 0x3c278, 0xffffffff, 0x00000100, + 0x3c210, 0xffffffff, 0x06000100, + 0x3c290, 0xffffffff, 0x00000100, + 0x3c274, 0xffffffff, 0x00000100, + 0x3c2b4, 0xffffffff, 0x00000100, + 0x3c2b0, 0xffffffff, 0x00000100, + 0x3c270, 0xffffffff, 0x00000100, + 0x30800, 0xffffffff, 0xe0000000, + 0x3c020, 0xffffffff, 0x00010000, + 0x3c024, 0xffffffff, 0x00030002, + 0x3c028, 0xffffffff, 0x00040007, + 0x3c02c, 0xffffffff, 0x00060005, + 0x3c030, 0xffffffff, 0x00090008, + 0x3c034, 0xffffffff, 0x00010000, + 0x3c038, 0xffffffff, 0x00030002, + 0x3c03c, 0xffffffff, 0x00040007, + 0x3c040, 0xffffffff, 0x00060005, + 0x3c044, 0xffffffff, 0x00090008, + 0x3c048, 0xffffffff, 0x00010000, + 0x3c04c, 0xffffffff, 0x00030002, + 0x3c050, 0xffffffff, 0x00040007, + 0x3c054, 0xffffffff, 0x00060005, + 0x3c058, 0xffffffff, 0x00090008, + 0x3c05c, 0xffffffff, 0x00010000, + 0x3c060, 0xffffffff, 0x00030002, + 0x3c064, 0xffffffff, 0x00040007, + 0x3c068, 0xffffffff, 0x00060005, + 0x3c06c, 0xffffffff, 0x00090008, + 0x3c070, 0xffffffff, 0x00010000, + 0x3c074, 0xffffffff, 0x00030002, + 0x3c078, 0xffffffff, 0x00040007, + 0x3c07c, 0xffffffff, 0x00060005, + 0x3c080, 0xffffffff, 0x00090008, + 0x3c084, 0xffffffff, 0x00010000, + 0x3c088, 0xffffffff, 0x00030002, + 0x3c08c, 0xffffffff, 0x00040007, + 0x3c090, 0xffffffff, 0x00060005, + 0x3c094, 0xffffffff, 0x00090008, + 0x3c098, 0xffffffff, 0x00010000, + 0x3c09c, 0xffffffff, 0x00030002, + 0x3c0a0, 0xffffffff, 0x00040007, + 0x3c0a4, 0xffffffff, 0x00060005, + 0x3c0a8, 0xffffffff, 0x00090008, + 0x3c0ac, 0xffffffff, 0x00010000, + 0x3c0b0, 0xffffffff, 0x00030002, + 0x3c0b4, 0xffffffff, 0x00040007, + 0x3c0b8, 0xffffffff, 0x00060005, + 0x3c0bc, 0xffffffff, 0x00090008, + 0x3c0c0, 0xffffffff, 0x00010000, + 0x3c0c4, 0xffffffff, 0x00030002, + 0x3c0c8, 0xffffffff, 0x00040007, + 0x3c0cc, 0xffffffff, 0x00060005, + 0x3c0d0, 0xffffffff, 0x00090008, + 0x3c0d4, 0xffffffff, 0x00010000, + 0x3c0d8, 0xffffffff, 0x00030002, + 0x3c0dc, 0xffffffff, 0x00040007, + 0x3c0e0, 0xffffffff, 0x00060005, + 0x3c0e4, 0xffffffff, 0x00090008, + 0x3c0e8, 0xffffffff, 0x00010000, + 0x3c0ec, 0xffffffff, 0x00030002, + 0x3c0f0, 0xffffffff, 0x00040007, + 0x3c0f4, 0xffffffff, 0x00060005, + 0x3c0f8, 0xffffffff, 0x00090008, + 0xc318, 0xffffffff, 0x00020200, + 0x3350, 0xffffffff, 0x00000200, + 0x15c0, 0xffffffff, 0x00000400, + 0x55e8, 0xffffffff, 0x00000000, + 0x2f50, 0xffffffff, 0x00000902, + 0x3c000, 0xffffffff, 0x96940200, + 0x8708, 0xffffffff, 0x00900100, + 0xc424, 0xffffffff, 0x0020003f, + 0x38, 0xffffffff, 0x0140001c, + 0x3c, 0x000f0000, 0x000f0000, + 0x220, 0xffffffff, 0xc060000c, + 0x224, 0xc0000fff, 0x00000100, + 0xf90, 0xffffffff, 0x00000100, + 0xf98, 0x00000101, 0x00000000, + 0x20a8, 0xffffffff, 0x00000104, + 0x55e4, 0xff000fff, 0x00000100, + 0x30cc, 0xc0000fff, 0x00000104, + 0xc1e4, 0x00000001, 0x00000001, + 0xd00c, 0xff000ff0, 0x00000100, + 0xd80c, 0xff000ff0, 0x00000100 +}; + +static const u32 godavari_golden_registers[] = +{ + 0x55e4, 0xff607fff, 0xfc000100, + 0x6ed8, 0x00010101, 0x00010000, + 0x9830, 0xffffffff, 0x00000000, + 0x98302, 0xf00fffff, 0x00000400, + 0x6130, 0xffffffff, 0x00010000, + 0x5bb0, 0x000000f0, 0x00000070, + 0x5bc0, 0xf0311fff, 0x80300000, + 0x98f8, 0x73773777, 0x12010001, + 0x98fc, 0xffffffff, 0x00000010, + 0x8030, 0x00001f0f, 0x0000100a, + 0x2f48, 0x73773777, 0x12010001, + 0x2408, 0x000fffff, 0x000c007f, + 0x8a14, 0xf000003f, 0x00000007, + 0x8b24, 0xffffffff, 0x00ff0fff, + 0x30a04, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x4d8, 0x00000fff, 0x00000100, + 0xd014, 0x00010000, 0x00810001, + 0xd814, 0x00010000, 0x00810001, + 0x3e78, 0x00000001, 0x00000002, + 0xc768, 0x00000008, 0x00000008, + 0xc770, 0x00000f00, 0x00000800, + 0xc774, 0x00000f00, 0x00000800, + 0xc798, 0x00ffffff, 0x00ff7fbf, + 0xc79c, 0x00ffffff, 0x00ff7faf, + 0x8c00, 0x000000ff, 0x00000001, + 0x214f8, 0x01ff01ff, 0x00000002, + 0x21498, 0x007ff800, 0x00200000, + 0x2015c, 0xffffffff, 0x00000f40, + 0x88c4, 0x001f3ae3, 0x00000082, + 0x88d4, 0x0000001f, 0x00000010, + 0x30934, 0xffffffff, 0x00000000 +}; + + +static void cik_init_golden_registers(struct radeon_device *rdev) +{ + /* Some of the registers might be dependent on GRBM_GFX_INDEX */ + mutex_lock(&rdev->grbm_idx_mutex); + switch (rdev->family) { + case CHIP_BONAIRE: + radeon_program_register_sequence(rdev, + bonaire_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init)); + radeon_program_register_sequence(rdev, + bonaire_golden_registers, + (const u32)ARRAY_SIZE(bonaire_golden_registers)); + radeon_program_register_sequence(rdev, + bonaire_golden_common_registers, + (const u32)ARRAY_SIZE(bonaire_golden_common_registers)); + radeon_program_register_sequence(rdev, + bonaire_golden_spm_registers, + (const u32)ARRAY_SIZE(bonaire_golden_spm_registers)); + break; + case CHIP_KABINI: + radeon_program_register_sequence(rdev, + kalindi_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); + radeon_program_register_sequence(rdev, + kalindi_golden_registers, + (const u32)ARRAY_SIZE(kalindi_golden_registers)); + radeon_program_register_sequence(rdev, + kalindi_golden_common_registers, + (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); + radeon_program_register_sequence(rdev, + kalindi_golden_spm_registers, + (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); + break; + case CHIP_MULLINS: + radeon_program_register_sequence(rdev, + kalindi_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); + radeon_program_register_sequence(rdev, + godavari_golden_registers, + (const u32)ARRAY_SIZE(godavari_golden_registers)); + radeon_program_register_sequence(rdev, + kalindi_golden_common_registers, + (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); + radeon_program_register_sequence(rdev, + kalindi_golden_spm_registers, + (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); + break; + case CHIP_KAVERI: + radeon_program_register_sequence(rdev, + spectre_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init)); + radeon_program_register_sequence(rdev, + spectre_golden_registers, + (const u32)ARRAY_SIZE(spectre_golden_registers)); + radeon_program_register_sequence(rdev, + spectre_golden_common_registers, + (const u32)ARRAY_SIZE(spectre_golden_common_registers)); + radeon_program_register_sequence(rdev, + spectre_golden_spm_registers, + (const u32)ARRAY_SIZE(spectre_golden_spm_registers)); + break; + case CHIP_HAWAII: + radeon_program_register_sequence(rdev, + hawaii_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init)); + radeon_program_register_sequence(rdev, + hawaii_golden_registers, + (const u32)ARRAY_SIZE(hawaii_golden_registers)); + radeon_program_register_sequence(rdev, + hawaii_golden_common_registers, + (const u32)ARRAY_SIZE(hawaii_golden_common_registers)); + radeon_program_register_sequence(rdev, + hawaii_golden_spm_registers, + (const u32)ARRAY_SIZE(hawaii_golden_spm_registers)); + break; + default: + break; + } + mutex_unlock(&rdev->grbm_idx_mutex); +} + +/** + * cik_get_xclk - get the xclk + * + * @rdev: radeon_device pointer + * + * Returns the reference clock used by the gfx engine + * (CIK). + */ +u32 cik_get_xclk(struct radeon_device *rdev) +{ + u32 reference_clock = rdev->clock.spll.reference_freq; + + if (rdev->flags & RADEON_IS_IGP) { + if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK) + return reference_clock / 2; + } else { + if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE) + return reference_clock / 4; + } + return reference_clock; +} + +/** + * cik_mm_rdoorbell - read a doorbell dword + * + * @rdev: radeon_device pointer + * @index: doorbell index + * + * Returns the value in the doorbell aperture at the + * requested doorbell index (CIK). + */ +u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) +{ + if (index < rdev->doorbell.num_doorbells) { + return bus_space_read_4(rdev->memt, rdev->doorbell.bsh, index); + } else { + DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); + return 0; + } +} + +/** + * cik_mm_wdoorbell - write a doorbell dword + * + * @rdev: radeon_device pointer + * @index: doorbell index + * @v: value to write + * + * Writes @v to the doorbell aperture at the + * requested doorbell index (CIK). + */ +void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) +{ + if (index < rdev->doorbell.num_doorbells) { + bus_space_write_4(rdev->memt, rdev->doorbell.bsh, index, v); + } else { + DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); + } +} + +#define BONAIRE_IO_MC_REGS_SIZE 36 + +static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] = +{ + {0x00000070, 0x04400000}, + {0x00000071, 0x80c01803}, + {0x00000072, 0x00004004}, + {0x00000073, 0x00000100}, + {0x00000074, 0x00ff0000}, + {0x00000075, 0x34000000}, + {0x00000076, 0x08000014}, + {0x00000077, 0x00cc08ec}, + {0x00000078, 0x00000400}, + {0x00000079, 0x00000000}, + {0x0000007a, 0x04090000}, + {0x0000007c, 0x00000000}, + {0x0000007e, 0x4408a8e8}, + {0x0000007f, 0x00000304}, + {0x00000080, 0x00000000}, + {0x00000082, 0x00000001}, + {0x00000083, 0x00000002}, + {0x00000084, 0xf3e4f400}, + {0x00000085, 0x052024e3}, + {0x00000087, 0x00000000}, + {0x00000088, 0x01000000}, + {0x0000008a, 0x1c0a0000}, + {0x0000008b, 0xff010000}, + {0x0000008d, 0xffffefff}, + {0x0000008e, 0xfff3efff}, + {0x0000008f, 0xfff3efbf}, + {0x00000092, 0xf7ffffff}, + {0x00000093, 0xffffff7f}, + {0x00000095, 0x00101101}, + {0x00000096, 0x00000fff}, + {0x00000097, 0x00116fff}, + {0x00000098, 0x60010000}, + {0x00000099, 0x10010000}, + {0x0000009a, 0x00006000}, + {0x0000009b, 0x00001000}, + {0x0000009f, 0x00b48000} +}; + +#define HAWAII_IO_MC_REGS_SIZE 22 + +static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] = +{ + {0x0000007d, 0x40000000}, + {0x0000007e, 0x40180304}, + {0x0000007f, 0x0000ff00}, + {0x00000081, 0x00000000}, + {0x00000083, 0x00000800}, + {0x00000086, 0x00000000}, + {0x00000087, 0x00000100}, + {0x00000088, 0x00020100}, + {0x00000089, 0x00000000}, + {0x0000008b, 0x00040000}, + {0x0000008c, 0x00000100}, + {0x0000008e, 0xff010000}, + {0x00000090, 0xffffefff}, + {0x00000091, 0xfff3efff}, + {0x00000092, 0xfff3efbf}, + {0x00000093, 0xf7ffffff}, + {0x00000094, 0xffffff7f}, + {0x00000095, 0x00000fff}, + {0x00000096, 0x00116fff}, + {0x00000097, 0x60010000}, + {0x00000098, 0x10010000}, + {0x0000009f, 0x00c79000} +}; + + +/** + * cik_srbm_select - select specific register instances + * + * @rdev: radeon_device pointer + * @me: selected ME (micro engine) + * @pipe: pipe + * @queue: queue + * @vmid: VMID + * + * Switches the currently active registers instances. Some + * registers are instanced per VMID, others are instanced per + * me/pipe/queue combination. + */ +static void cik_srbm_select(struct radeon_device *rdev, + u32 me, u32 pipe, u32 queue, u32 vmid) +{ + u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) | + MEID(me & 0x3) | + VMID(vmid & 0xf) | + QUEUEID(queue & 0x7)); + WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl); +} + +/* ucode loading */ +/** + * ci_mc_load_microcode - load MC ucode into the hw + * + * @rdev: radeon_device pointer + * + * Load the GDDR MC ucode into the hw (CIK). + * Returns 0 on success, error on failure. + */ +int ci_mc_load_microcode(struct radeon_device *rdev) +{ + const __be32 *fw_data = NULL; + const __le32 *new_fw_data = NULL; + u32 running, blackout = 0, tmp; + u32 *io_mc_regs = NULL; + const __le32 *new_io_mc_regs = NULL; + int i, regs_size, ucode_size; + + if (!rdev->mc_fw) + return -EINVAL; + + if (rdev->new_fw) { + const struct mc_firmware_header_v1_0 *hdr = + (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; + + radeon_ucode_print_mc_hdr(&hdr->header); + + regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); + new_io_mc_regs = (const __le32 *) + (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); + ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; + new_fw_data = (const __le32 *) + (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + } else { + ucode_size = rdev->mc_fw->size / 4; + + switch (rdev->family) { + case CHIP_BONAIRE: + io_mc_regs = (u32 *)&bonaire_io_mc_regs; + regs_size = BONAIRE_IO_MC_REGS_SIZE; + break; + case CHIP_HAWAII: + io_mc_regs = (u32 *)&hawaii_io_mc_regs; + regs_size = HAWAII_IO_MC_REGS_SIZE; + break; + default: + return -EINVAL; + } + fw_data = (const __be32 *)rdev->mc_fw->data; + } + + running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; + + if (running == 0) { + if (running) { + blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); + WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); + } + + /* reset the engine and set to writable */ + WREG32(MC_SEQ_SUP_CNTL, 0x00000008); + WREG32(MC_SEQ_SUP_CNTL, 0x00000010); + + /* load mc io regs */ + for (i = 0; i < regs_size; i++) { + if (rdev->new_fw) { + WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++)); + WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++)); + } else { + WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); + WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); + } + } + + tmp = RREG32(MC_SEQ_MISC0); + if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) { + WREG32(MC_SEQ_IO_DEBUG_INDEX, 5); + WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023); + WREG32(MC_SEQ_IO_DEBUG_INDEX, 9); + WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0); + } + + /* load the MC ucode */ + for (i = 0; i < ucode_size; i++) { + if (rdev->new_fw) + WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++)); + else + WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); + } + + /* put the engine back into the active state */ + WREG32(MC_SEQ_SUP_CNTL, 0x00000008); + WREG32(MC_SEQ_SUP_CNTL, 0x00000004); + WREG32(MC_SEQ_SUP_CNTL, 0x00000001); + + /* wait for training to complete */ + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) + break; + udelay(1); + } + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) + break; + udelay(1); + } + + if (running) + WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); + } + + return 0; +} + +/** + * cik_init_microcode - load ucode images from disk + * + * @rdev: radeon_device pointer + * + * Use the firmware interface to load the ucode images into + * the driver (not loaded into hw). + * Returns 0 on success, error on failure. + */ +static int cik_init_microcode(struct radeon_device *rdev) +{ + const char *chip_name; + const char *new_chip_name; + size_t pfp_req_size, me_req_size, ce_req_size, + mec_req_size, rlc_req_size, mc_req_size = 0, + sdma_req_size, smc_req_size = 0, mc2_req_size = 0; + char fw_name[30]; + int new_fw = 0; + int err; + int num_fw; + + DRM_DEBUG("\n"); + + switch (rdev->family) { + case CHIP_BONAIRE: + chip_name = "BONAIRE"; + new_chip_name = "bonaire"; + pfp_req_size = CIK_PFP_UCODE_SIZE * 4; + me_req_size = CIK_ME_UCODE_SIZE * 4; + ce_req_size = CIK_CE_UCODE_SIZE * 4; + mec_req_size = CIK_MEC_UCODE_SIZE * 4; + rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; + mc_req_size = BONAIRE_MC_UCODE_SIZE * 4; + mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4; + sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; + smc_req_size = roundup2(BONAIRE_SMC_UCODE_SIZE, 4); + num_fw = 8; + break; + case CHIP_HAWAII: + chip_name = "HAWAII"; + new_chip_name = "hawaii"; + pfp_req_size = CIK_PFP_UCODE_SIZE * 4; + me_req_size = CIK_ME_UCODE_SIZE * 4; + ce_req_size = CIK_CE_UCODE_SIZE * 4; + mec_req_size = CIK_MEC_UCODE_SIZE * 4; + rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; + mc_req_size = HAWAII_MC_UCODE_SIZE * 4; + mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4; + sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; + smc_req_size = roundup2(HAWAII_SMC_UCODE_SIZE, 4); + num_fw = 8; + break; + case CHIP_KAVERI: + chip_name = "KAVERI"; + new_chip_name = "kaveri"; + pfp_req_size = CIK_PFP_UCODE_SIZE * 4; + me_req_size = CIK_ME_UCODE_SIZE * 4; + ce_req_size = CIK_CE_UCODE_SIZE * 4; + mec_req_size = CIK_MEC_UCODE_SIZE * 4; + rlc_req_size = KV_RLC_UCODE_SIZE * 4; + sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; + num_fw = 7; + break; + case CHIP_KABINI: + chip_name = "KABINI"; + new_chip_name = "kabini"; + pfp_req_size = CIK_PFP_UCODE_SIZE * 4; + me_req_size = CIK_ME_UCODE_SIZE * 4; + ce_req_size = CIK_CE_UCODE_SIZE * 4; + mec_req_size = CIK_MEC_UCODE_SIZE * 4; + rlc_req_size = KB_RLC_UCODE_SIZE * 4; + sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; + num_fw = 6; + break; + case CHIP_MULLINS: + chip_name = "MULLINS"; + new_chip_name = "mullins"; + pfp_req_size = CIK_PFP_UCODE_SIZE * 4; + me_req_size = CIK_ME_UCODE_SIZE * 4; + ce_req_size = CIK_CE_UCODE_SIZE * 4; + mec_req_size = CIK_MEC_UCODE_SIZE * 4; + rlc_req_size = ML_RLC_UCODE_SIZE * 4; + sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; + num_fw = 6; + break; + default: BUG(); + } + + DRM_INFO("Loading %s Microcode\n", new_chip_name); + + snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name); + err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); + err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->pfp_fw->size != pfp_req_size) { + printk(KERN_ERR + "cik_cp: Bogus length %zu in firmware \"%s\"\n", + rdev->pfp_fw->size, fw_name); + err = -EINVAL; + goto out; + } + } else { + err = radeon_ucode_validate(rdev->pfp_fw); + if (err) { + printk(KERN_ERR + "cik_fw: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } + } + + snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name); + err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); + err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->me_fw->size != me_req_size) { + printk(KERN_ERR + "cik_cp: Bogus length %zu in firmware \"%s\"\n", + rdev->me_fw->size, fw_name); + err = -EINVAL; + } + } else { + err = radeon_ucode_validate(rdev->me_fw); + if (err) { + printk(KERN_ERR + "cik_fw: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } + } + + snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name); + err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); + err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->ce_fw->size != ce_req_size) { + printk(KERN_ERR + "cik_cp: Bogus length %zu in firmware \"%s\"\n", + rdev->ce_fw->size, fw_name); + err = -EINVAL; + } + } else { + err = radeon_ucode_validate(rdev->ce_fw); + if (err) { + printk(KERN_ERR + "cik_fw: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } + } + + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name); + err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name); + err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->mec_fw->size != mec_req_size) { + printk(KERN_ERR + "cik_cp: Bogus length %zu in firmware \"%s\"\n", + rdev->mec_fw->size, fw_name); + err = -EINVAL; + } + } else { + err = radeon_ucode_validate(rdev->mec_fw); + if (err) { + printk(KERN_ERR + "cik_fw: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } + } + + if (rdev->family == CHIP_KAVERI) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name); + err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev); + if (err) { + goto out; + } else { + err = radeon_ucode_validate(rdev->mec2_fw); + if (err) { + goto out; + } else { + new_fw++; + } + } + } + + snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name); + err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name); + err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->rlc_fw->size != rlc_req_size) { + printk(KERN_ERR + "cik_rlc: Bogus length %zu in firmware \"%s\"\n", + rdev->rlc_fw->size, fw_name); + err = -EINVAL; + } + } else { + err = radeon_ucode_validate(rdev->rlc_fw); + if (err) { + printk(KERN_ERR + "cik_fw: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } + } + + snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name); + err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name); + err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->sdma_fw->size != sdma_req_size) { + printk(KERN_ERR + "cik_sdma: Bogus length %zu in firmware \"%s\"\n", + rdev->sdma_fw->size, fw_name); + err = -EINVAL; + } + } else { + err = radeon_ucode_validate(rdev->sdma_fw); + if (err) { + printk(KERN_ERR + "cik_fw: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } + } + + /* No SMC, MC ucode on APUs */ + if (!(rdev->flags & RADEON_IS_IGP)) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name); + err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name); + err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); + err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); + if (err) + goto out; + } + if ((rdev->mc_fw->size != mc_req_size) && + (rdev->mc_fw->size != mc2_req_size)){ + printk(KERN_ERR + "cik_mc: Bogus length %zu in firmware \"%s\"\n", + rdev->mc_fw->size, fw_name); + err = -EINVAL; + } + DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); + } else { + err = radeon_ucode_validate(rdev->mc_fw); + if (err) { + printk(KERN_ERR + "cik_fw: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } + } + + snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name); + err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); + err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); + if (err) { + printk(KERN_ERR + "smc: error loading firmware \"%s\"\n", + fw_name); + release_firmware(rdev->smc_fw); + rdev->smc_fw = NULL; + err = 0; + } else if (rdev->smc_fw->size != smc_req_size) { + printk(KERN_ERR + "cik_smc: Bogus length %zu in firmware \"%s\"\n", + rdev->smc_fw->size, fw_name); + err = -EINVAL; + } + } else { + err = radeon_ucode_validate(rdev->smc_fw); + if (err) { + printk(KERN_ERR + "cik_fw: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } + } + } + + if (new_fw == 0) { + rdev->new_fw = false; + } else if (new_fw < num_fw) { + printk(KERN_ERR "ci_fw: mixing new and old firmware!\n"); + err = -EINVAL; + } else { + rdev->new_fw = true; + } + +out: + if (err) { + if (err != -EINVAL) + printk(KERN_ERR + "cik_cp: Failed to load firmware \"%s\"\n", + fw_name); + release_firmware(rdev->pfp_fw); + rdev->pfp_fw = NULL; + release_firmware(rdev->me_fw); + rdev->me_fw = NULL; + release_firmware(rdev->ce_fw); + rdev->ce_fw = NULL; + release_firmware(rdev->mec_fw); + rdev->mec_fw = NULL; + release_firmware(rdev->mec2_fw); + rdev->mec2_fw = NULL; + release_firmware(rdev->rlc_fw); + rdev->rlc_fw = NULL; + release_firmware(rdev->sdma_fw); + rdev->sdma_fw = NULL; + release_firmware(rdev->mc_fw); + rdev->mc_fw = NULL; + release_firmware(rdev->smc_fw); + rdev->smc_fw = NULL; + } + return err; +} + +/* + * Core functions + */ +/** + * cik_tiling_mode_table_init - init the hw tiling table + * + * @rdev: radeon_device pointer + * + * Starting with SI, the tiling setup is done globally in a + * set of 32 tiling modes. Rather than selecting each set of + * parameters per surface as on older asics, we just select + * which index in the tiling table we want to use, and the + * surface uses those parameters (CIK). + */ +static void cik_tiling_mode_table_init(struct radeon_device *rdev) +{ + const u32 num_tile_mode_states = 32; + const u32 num_secondary_tile_mode_states = 16; + u32 reg_offset, gb_tile_moden, split_equal_to_row_size; + u32 num_pipe_configs; + u32 num_rbs = rdev->config.cik.max_backends_per_se * + rdev->config.cik.max_shader_engines; + + switch (rdev->config.cik.mem_row_size_in_kb) { + case 1: + split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; + break; + case 2: + default: + split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; + break; + case 4: + split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; + break; + } + + num_pipe_configs = rdev->config.cik.max_tile_pipes; + if (num_pipe_configs > 8) + num_pipe_configs = 16; + + if (num_pipe_configs == 16) { + for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { + switch (reg_offset) { + case 0: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); + break; + case 1: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); + break; + case 2: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 3: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); + break; + case 4: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 5: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); + break; + case 6: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 7: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 8: + gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); + break; + case 9: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); + break; + case 10: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 11: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 12: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 13: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); + break; + case 14: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 16: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 17: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 27: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); + break; + case 28: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 29: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 30: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + default: + gb_tile_moden = 0; + break; + } + rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; + WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); + } + for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { + switch (reg_offset) { + case 0: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 1: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 2: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 3: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 4: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_8_BANK)); + break; + case 5: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_4_BANK)); + break; + case 6: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_2_BANK)); + break; + case 8: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 9: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 10: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 11: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_8_BANK)); + break; + case 12: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_4_BANK)); + break; + case 13: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_2_BANK)); + break; + case 14: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_2_BANK)); + break; + default: + gb_tile_moden = 0; + break; + } + rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; + WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); + } + } else if (num_pipe_configs == 8) { + for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { + switch (reg_offset) { + case 0: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); + break; + case 1: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); + break; + case 2: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 3: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); + break; + case 4: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 5: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); + break; + case 6: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 7: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 8: + gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); + break; + case 9: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); + break; + case 10: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 11: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 12: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 13: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); + break; + case 14: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 16: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 17: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 27: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); + break; + case 28: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 29: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 30: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + default: + gb_tile_moden = 0; + break; + } + rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; + WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); + } + for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { + switch (reg_offset) { + case 0: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 1: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 2: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 3: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 4: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_8_BANK)); + break; + case 5: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_4_BANK)); + break; + case 6: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_2_BANK)); + break; + case 8: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 9: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 10: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 11: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 12: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_8_BANK)); + break; + case 13: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_4_BANK)); + break; + case 14: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_2_BANK)); + break; + default: + gb_tile_moden = 0; + break; + } + rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; + WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); + } + } else if (num_pipe_configs == 4) { + if (num_rbs == 4) { + for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { + switch (reg_offset) { + case 0: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); + break; + case 1: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); + break; + case 2: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 3: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); + break; + case 4: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 5: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); + break; + case 6: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 7: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 8: + gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | + PIPE_CONFIG(ADDR_SURF_P4_16x16)); + break; + case 9: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); + break; + case 10: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 11: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 12: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 13: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); + break; + case 14: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 16: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 17: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 27: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); + break; + case 28: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 29: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 30: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_16x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + default: + gb_tile_moden = 0; + break; + } + rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; + WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); + } + } else if (num_rbs < 4) { + for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { + switch (reg_offset) { + case 0: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); + break; + case 1: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); + break; + case 2: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 3: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); + break; + case 4: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 5: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); + break; + case 6: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 7: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 8: + gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | + PIPE_CONFIG(ADDR_SURF_P4_8x16)); + break; + case 9: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); + break; + case 10: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 11: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 12: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 13: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); + break; + case 14: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 16: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 17: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 27: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); + break; + case 28: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 29: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 30: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P4_8x16) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + default: + gb_tile_moden = 0; + break; + } + rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; + WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); + } + } + for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { + switch (reg_offset) { + case 0: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 1: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 2: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 3: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 4: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 5: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_8_BANK)); + break; + case 6: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_4_BANK)); + break; + case 8: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 9: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 10: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 11: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 12: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 13: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_8_BANK)); + break; + case 14: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | + NUM_BANKS(ADDR_SURF_4_BANK)); + break; + default: + gb_tile_moden = 0; + break; + } + rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; + WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); + } + } else if (num_pipe_configs == 2) { + for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { + switch (reg_offset) { + case 0: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); + break; + case 1: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); + break; + case 2: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 3: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); + break; + case 4: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 5: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P2) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); + break; + case 6: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); + break; + case 7: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + TILE_SPLIT(split_equal_to_row_size)); + break; + case 8: + gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | + PIPE_CONFIG(ADDR_SURF_P2); + break; + case 9: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2)); + break; + case 10: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 11: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 12: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 13: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + PIPE_CONFIG(ADDR_SURF_P2) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); + break; + case 14: + gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 16: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 17: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 27: + gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2)); + break; + case 28: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 29: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + case 30: + gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | + MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | + PIPE_CONFIG(ADDR_SURF_P2) | + SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); + break; + default: + gb_tile_moden = 0; + break; + } + rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; + WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); + } + for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { + switch (reg_offset) { + case 0: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 1: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 2: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 3: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 4: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 5: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 6: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_8_BANK)); + break; + case 8: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 9: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 10: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 11: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 12: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 13: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | + NUM_BANKS(ADDR_SURF_16_BANK)); + break; + case 14: + gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | + BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | + MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | + NUM_BANKS(ADDR_SURF_8_BANK)); + break; + default: + gb_tile_moden = 0; + break; + } + rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; + WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); + } + } else + DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs); +} + +/** + * cik_select_se_sh - select which SE, SH to address + * + * @rdev: radeon_device pointer + * @se_num: shader engine to address + * @sh_num: sh block to address + * + * Select which SE, SH combinations to address. Certain + * registers are instanced per SE or SH. 0xffffffff means + * broadcast to all SEs or SHs (CIK). + */ +static void cik_select_se_sh(struct radeon_device *rdev, + u32 se_num, u32 sh_num) +{ + u32 data = INSTANCE_BROADCAST_WRITES; + + if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) + data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; + else if (se_num == 0xffffffff) + data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); + else if (sh_num == 0xffffffff) + data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); + else + data |= SH_INDEX(sh_num) | SE_INDEX(se_num); + WREG32(GRBM_GFX_INDEX, data); +} + +/** + * cik_create_bitmask - create a bitmask + * + * @bit_width: length of the mask + * + * create a variable length bit mask (CIK). + * Returns the bitmask. + */ +static u32 cik_create_bitmask(u32 bit_width) +{ + u32 i, mask = 0; + + for (i = 0; i < bit_width; i++) { + mask <<= 1; + mask |= 1; + } + return mask; +} + +/** + * cik_get_rb_disabled - computes the mask of disabled RBs + * + * @rdev: radeon_device pointer + * @max_rb_num: max RBs (render backends) for the asic + * @se_num: number of SEs (shader engines) for the asic + * @sh_per_se: number of SH blocks per SE for the asic + * + * Calculates the bitmask of disabled RBs (CIK). + * Returns the disabled RB bitmask. + */ +static u32 cik_get_rb_disabled(struct radeon_device *rdev, + u32 max_rb_num_per_se, + u32 sh_per_se) +{ + u32 data, mask; + + data = RREG32(CC_RB_BACKEND_DISABLE); + if (data & 1) + data &= BACKEND_DISABLE_MASK; + else + data = 0; + data |= RREG32(GC_USER_RB_BACKEND_DISABLE); + + data >>= BACKEND_DISABLE_SHIFT; + + mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se); + + return data & mask; +} + +/** + * cik_setup_rb - setup the RBs on the asic + * + * @rdev: radeon_device pointer + * @se_num: number of SEs (shader engines) for the asic + * @sh_per_se: number of SH blocks per SE for the asic + * @max_rb_num: max RBs (render backends) for the asic + * + * Configures per-SE/SH RB registers (CIK). + */ +static void cik_setup_rb(struct radeon_device *rdev, + u32 se_num, u32 sh_per_se, + u32 max_rb_num_per_se) +{ + int i, j; + u32 data, mask; + u32 disabled_rbs = 0; + u32 enabled_rbs = 0; + + mutex_lock(&rdev->grbm_idx_mutex); + for (i = 0; i < se_num; i++) { + for (j = 0; j < sh_per_se; j++) { + cik_select_se_sh(rdev, i, j); + data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); + if (rdev->family == CHIP_HAWAII) + disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH); + else + disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH); + } + } + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + mutex_unlock(&rdev->grbm_idx_mutex); + + mask = 1; + for (i = 0; i < max_rb_num_per_se * se_num; i++) { + if (!(disabled_rbs & mask)) + enabled_rbs |= mask; + mask <<= 1; + } + + rdev->config.cik.backend_enable_mask = enabled_rbs; + + mutex_lock(&rdev->grbm_idx_mutex); + for (i = 0; i < se_num; i++) { + cik_select_se_sh(rdev, i, 0xffffffff); + data = 0; + for (j = 0; j < sh_per_se; j++) { + switch (enabled_rbs & 3) { + case 0: + if (j == 0) + data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3); + else + data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0); + break; + case 1: + data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); + break; + case 2: + data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); + break; + case 3: + default: + data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); + break; + } + enabled_rbs >>= 2; + } + WREG32(PA_SC_RASTER_CONFIG, data); + } + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + mutex_unlock(&rdev->grbm_idx_mutex); +} + +/** + * cik_gpu_init - setup the 3D engine + * + * @rdev: radeon_device pointer + * + * Configures the 3D engine and tiling configuration + * registers so that the 3D engine is usable. + */ +static void cik_gpu_init(struct radeon_device *rdev) +{ + u32 gb_addr_config = RREG32(GB_ADDR_CONFIG); + u32 mc_shared_chmap, mc_arb_ramcfg; + u32 hdp_host_path_cntl; + u32 tmp; + int i, j; + + switch (rdev->family) { + case CHIP_BONAIRE: + rdev->config.cik.max_shader_engines = 2; + rdev->config.cik.max_tile_pipes = 4; + rdev->config.cik.max_cu_per_sh = 7; + rdev->config.cik.max_sh_per_se = 1; + rdev->config.cik.max_backends_per_se = 2; + rdev->config.cik.max_texture_channel_caches = 4; + rdev->config.cik.max_gprs = 256; + rdev->config.cik.max_gs_threads = 32; + rdev->config.cik.max_hw_contexts = 8; + + rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; + rdev->config.cik.sc_prim_fifo_size_backend = 0x100; + rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; + rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; + gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; + break; + case CHIP_HAWAII: + rdev->config.cik.max_shader_engines = 4; + rdev->config.cik.max_tile_pipes = 16; + rdev->config.cik.max_cu_per_sh = 11; + rdev->config.cik.max_sh_per_se = 1; + rdev->config.cik.max_backends_per_se = 4; + rdev->config.cik.max_texture_channel_caches = 16; + rdev->config.cik.max_gprs = 256; + rdev->config.cik.max_gs_threads = 32; + rdev->config.cik.max_hw_contexts = 8; + + rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; + rdev->config.cik.sc_prim_fifo_size_backend = 0x100; + rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; + rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; + gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN; + break; + case CHIP_KAVERI: + rdev->config.cik.max_shader_engines = 1; + rdev->config.cik.max_tile_pipes = 4; + rdev->config.cik.max_cu_per_sh = 8; + rdev->config.cik.max_backends_per_se = 2; + rdev->config.cik.max_sh_per_se = 1; + rdev->config.cik.max_texture_channel_caches = 4; + rdev->config.cik.max_gprs = 256; + rdev->config.cik.max_gs_threads = 16; + rdev->config.cik.max_hw_contexts = 8; + + rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; + rdev->config.cik.sc_prim_fifo_size_backend = 0x100; + rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; + rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; + gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; + break; + case CHIP_KABINI: + case CHIP_MULLINS: + default: + rdev->config.cik.max_shader_engines = 1; + rdev->config.cik.max_tile_pipes = 2; + rdev->config.cik.max_cu_per_sh = 2; + rdev->config.cik.max_sh_per_se = 1; + rdev->config.cik.max_backends_per_se = 1; + rdev->config.cik.max_texture_channel_caches = 2; + rdev->config.cik.max_gprs = 256; + rdev->config.cik.max_gs_threads = 16; + rdev->config.cik.max_hw_contexts = 8; + + rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; + rdev->config.cik.sc_prim_fifo_size_backend = 0x100; + rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; + rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; + gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; + break; + } + + /* Initialize HDP */ + for (i = 0, j = 0; i < 32; i++, j += 0x18) { + WREG32((0x2c14 + j), 0x00000000); + WREG32((0x2c18 + j), 0x00000000); + WREG32((0x2c1c + j), 0x00000000); + WREG32((0x2c20 + j), 0x00000000); + WREG32((0x2c24 + j), 0x00000000); + } + + WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); + WREG32(SRBM_INT_CNTL, 0x1); + WREG32(SRBM_INT_ACK, 0x1); + + WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); + + mc_shared_chmap = RREG32(MC_SHARED_CHMAP); + mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); + + rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; + rdev->config.cik.mem_max_burst_length_bytes = 256; + tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; + rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; + if (rdev->config.cik.mem_row_size_in_kb > 4) + rdev->config.cik.mem_row_size_in_kb = 4; + /* XXX use MC settings? */ + rdev->config.cik.shader_engine_tile_size = 32; + rdev->config.cik.num_gpus = 1; + rdev->config.cik.multi_gpu_tile_size = 64; + + /* fix up row size */ + gb_addr_config &= ~ROW_SIZE_MASK; + switch (rdev->config.cik.mem_row_size_in_kb) { + case 1: + default: + gb_addr_config |= ROW_SIZE(0); + break; + case 2: + gb_addr_config |= ROW_SIZE(1); + break; + case 4: + gb_addr_config |= ROW_SIZE(2); + break; + } + + /* setup tiling info dword. gb_addr_config is not adequate since it does + * not have bank info, so create a custom tiling dword. + * bits 3:0 num_pipes + * bits 7:4 num_banks + * bits 11:8 group_size + * bits 15:12 row_size + */ + rdev->config.cik.tile_config = 0; + switch (rdev->config.cik.num_tile_pipes) { + case 1: + rdev->config.cik.tile_config |= (0 << 0); + break; + case 2: + rdev->config.cik.tile_config |= (1 << 0); + break; + case 4: + rdev->config.cik.tile_config |= (2 << 0); + break; + case 8: + default: + /* XXX what about 12? */ + rdev->config.cik.tile_config |= (3 << 0); + break; + } + rdev->config.cik.tile_config |= + ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4; + rdev->config.cik.tile_config |= + ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; + rdev->config.cik.tile_config |= + ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; + + WREG32(GB_ADDR_CONFIG, gb_addr_config); + WREG32(HDP_ADDR_CONFIG, gb_addr_config); + WREG32(DMIF_ADDR_CALC, gb_addr_config); + WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70); + WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70); + WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); + WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); + WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); + + cik_tiling_mode_table_init(rdev); + + cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, + rdev->config.cik.max_sh_per_se, + rdev->config.cik.max_backends_per_se); + + rdev->config.cik.active_cus = 0; + for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { + for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { + rdev->config.cik.active_cus += + hweight32(cik_get_cu_active_bitmap(rdev, i, j)); + } + } + + /* set HW defaults for 3D engine */ + WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); + + mutex_lock(&rdev->grbm_idx_mutex); + /* + * making sure that the following register writes will be broadcasted + * to all the shaders + */ + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + WREG32(SX_DEBUG_1, 0x20); + + WREG32(TA_CNTL_AUX, 0x00010000); + + tmp = RREG32(SPI_CONFIG_CNTL); + tmp |= 0x03000000; + WREG32(SPI_CONFIG_CNTL, tmp); + + WREG32(SQ_CONFIG, 1); + + WREG32(DB_DEBUG, 0); + + tmp = RREG32(DB_DEBUG2) & ~0xf00fffff; + tmp |= 0x00000400; + WREG32(DB_DEBUG2, tmp); + + tmp = RREG32(DB_DEBUG3) & ~0x0002021c; + tmp |= 0x00020200; + WREG32(DB_DEBUG3, tmp); + + tmp = RREG32(CB_HW_CONTROL) & ~0x00010000; + tmp |= 0x00018208; + WREG32(CB_HW_CONTROL, tmp); + + WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); + + WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | + SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | + SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | + SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); + + WREG32(VGT_NUM_INSTANCES, 1); + + WREG32(CP_PERFMON_CNTL, 0); + + WREG32(SQ_CONFIG, 0); + + WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | + FORCE_EOV_MAX_REZ_CNT(255))); + + WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | + AUTO_INVLD_EN(ES_AND_GS_AUTO)); + + WREG32(VGT_GS_VERTEX_REUSE, 16); + WREG32(PA_SC_LINE_STIPPLE_STATE, 0); + + tmp = RREG32(HDP_MISC_CNTL); + tmp |= HDP_FLUSH_INVALIDATE_CACHE; + WREG32(HDP_MISC_CNTL, tmp); + + hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); + WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); + + WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); + WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); + mutex_unlock(&rdev->grbm_idx_mutex); + + udelay(50); +} + +/* + * GPU scratch registers helpers function. + */ +/** + * cik_scratch_init - setup driver info for CP scratch regs + * + * @rdev: radeon_device pointer + * + * Set up the number and offset of the CP scratch registers. + * NOTE: use of CP scratch registers is a legacy inferface and + * is not used by default on newer asics (r6xx+). On newer asics, + * memory buffers are used for fences rather than scratch regs. + */ +static void cik_scratch_init(struct radeon_device *rdev) +{ + int i; + + rdev->scratch.num_reg = 7; + rdev->scratch.reg_base = SCRATCH_REG0; + for (i = 0; i < rdev->scratch.num_reg; i++) { + rdev->scratch.free[i] = true; + rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); + } +} + +/** + * cik_ring_test - basic gfx ring test + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Allocate a scratch register and write to it using the gfx ring (CIK). + * Provides a basic gfx ring test to verify that the ring is working. + * Used by cik_cp_gfx_resume(); + * Returns 0 on success, error on failure. + */ +int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) +{ + uint32_t scratch; + uint32_t tmp = 0; + unsigned i; + int r; + + r = radeon_scratch_get(rdev, &scratch); + if (r) { + DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r); + return r; + } + WREG32(scratch, 0xCAFEDEAD); + r = radeon_ring_lock(rdev, ring, 3); + if (r) { + DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r); + radeon_scratch_free(rdev, scratch); + return r; + } + radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); + radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2)); + radeon_ring_write(ring, 0xDEADBEEF); + radeon_ring_unlock_commit(rdev, ring, false); + + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = RREG32(scratch); + if (tmp == 0xDEADBEEF) + break; + DRM_UDELAY(1); + } + if (i < rdev->usec_timeout) { + DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); + } else { + DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n", + ring->idx, scratch, tmp); + r = -EINVAL; + } + radeon_scratch_free(rdev, scratch); + return r; +} + +/** + * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp + * + * @rdev: radeon_device pointer + * @ridx: radeon ring index + * + * Emits an hdp flush on the cp. + */ +static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, + int ridx) +{ + struct radeon_ring *ring = &rdev->ring[ridx]; + u32 ref_and_mask; + + switch (ring->idx) { + case CAYMAN_RING_TYPE_CP1_INDEX: + case CAYMAN_RING_TYPE_CP2_INDEX: + default: + switch (ring->me) { + case 0: + ref_and_mask = CP2 << ring->pipe; + break; + case 1: + ref_and_mask = CP6 << ring->pipe; + break; + default: + return; + } + break; + case RADEON_RING_TYPE_GFX_INDEX: + ref_and_mask = CP0; + break; + } + + radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); + radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ + WAIT_REG_MEM_FUNCTION(3) | /* == */ + WAIT_REG_MEM_ENGINE(1))); /* pfp */ + radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2); + radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2); + radeon_ring_write(ring, ref_and_mask); + radeon_ring_write(ring, ref_and_mask); + radeon_ring_write(ring, 0x20); /* poll interval */ +} + +/** + * cik_fence_gfx_ring_emit - emit a fence on the gfx ring + * + * @rdev: radeon_device pointer + * @fence: radeon fence object + * + * Emits a fence sequnce number on the gfx ring and flushes + * GPU caches. + */ +void cik_fence_gfx_ring_emit(struct radeon_device *rdev, + struct radeon_fence *fence) +{ + struct radeon_ring *ring = &rdev->ring[fence->ring]; + u64 addr = rdev->fence_drv[fence->ring].gpu_addr; + + /* Workaround for cache flush problems. First send a dummy EOP + * event down the pipe with seq one below. + */ + radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); + radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | + EOP_TC_ACTION_EN | + EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | + EVENT_INDEX(5))); + radeon_ring_write(ring, addr & 0xfffffffc); + radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | + DATA_SEL(1) | INT_SEL(0)); + radeon_ring_write(ring, fence->seq - 1); + radeon_ring_write(ring, 0); + + /* Then send the real EOP event down the pipe. */ + radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); + radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | + EOP_TC_ACTION_EN | + EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | + EVENT_INDEX(5))); + radeon_ring_write(ring, addr & 0xfffffffc); + radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2)); + radeon_ring_write(ring, fence->seq); + radeon_ring_write(ring, 0); +} + +/** + * cik_fence_compute_ring_emit - emit a fence on the compute ring + * + * @rdev: radeon_device pointer + * @fence: radeon fence object + * + * Emits a fence sequnce number on the compute ring and flushes + * GPU caches. + */ +void cik_fence_compute_ring_emit(struct radeon_device *rdev, + struct radeon_fence *fence) +{ + struct radeon_ring *ring = &rdev->ring[fence->ring]; + u64 addr = rdev->fence_drv[fence->ring].gpu_addr; + + /* RELEASE_MEM - flush caches, send int */ + radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); + radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | + EOP_TC_ACTION_EN | + EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | + EVENT_INDEX(5))); + radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2)); + radeon_ring_write(ring, addr & 0xfffffffc); + radeon_ring_write(ring, upper_32_bits(addr)); + radeon_ring_write(ring, fence->seq); + radeon_ring_write(ring, 0); +} + +/** + * cik_semaphore_ring_emit - emit a semaphore on the CP ring + * + * @rdev: radeon_device pointer + * @ring: radeon ring buffer object + * @semaphore: radeon semaphore object + * @emit_wait: Is this a sempahore wait? + * + * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP + * from running ahead of semaphore waits. + */ +bool cik_semaphore_ring_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait) +{ + uint64_t addr = semaphore->gpu_addr; + unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL; + + radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1)); + radeon_ring_write(ring, lower_32_bits(addr)); + radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel); + + if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) { + /* Prevent the PFP from running ahead of the semaphore wait */ + radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); + radeon_ring_write(ring, 0x0); + } + + return true; +} + +/** + * cik_copy_cpdma - copy pages using the CP DMA engine + * + * @rdev: radeon_device pointer + * @src_offset: src GPU address + * @dst_offset: dst GPU address + * @num_gpu_pages: number of GPU pages to xfer + * @resv: reservation object to sync to + * + * Copy GPU paging using the CP DMA engine (CIK+). + * Used by the radeon ttm implementation to move pages if + * registered as the asic copy callback. + */ +struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv) +{ + struct radeon_fence *fence; + struct radeon_sync sync; + int ring_index = rdev->asic->copy.blit_ring_index; + struct radeon_ring *ring = &rdev->ring[ring_index]; + u32 size_in_bytes, cur_size_in_bytes, control; + int i, num_loops; + int r = 0; + + radeon_sync_create(&sync); + + size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); + num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); + r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); + if (r) { + DRM_ERROR("radeon: moving bo (%d).\n", r); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); + + for (i = 0; i < num_loops; i++) { + cur_size_in_bytes = size_in_bytes; + if (cur_size_in_bytes > 0x1fffff) + cur_size_in_bytes = 0x1fffff; + size_in_bytes -= cur_size_in_bytes; + control = 0; + if (size_in_bytes == 0) + control |= PACKET3_DMA_DATA_CP_SYNC; + radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); + radeon_ring_write(ring, control); + radeon_ring_write(ring, lower_32_bits(src_offset)); + radeon_ring_write(ring, upper_32_bits(src_offset)); + radeon_ring_write(ring, lower_32_bits(dst_offset)); + radeon_ring_write(ring, upper_32_bits(dst_offset)); + radeon_ring_write(ring, cur_size_in_bytes); + src_offset += cur_size_in_bytes; + dst_offset += cur_size_in_bytes; + } + + r = radeon_fence_emit(rdev, &fence, ring->idx); + if (r) { + radeon_ring_unlock_undo(rdev, ring); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_ring_unlock_commit(rdev, ring, false); + radeon_sync_free(rdev, &sync, fence); + + return fence; +} + +/* + * IB stuff + */ +/** + * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring + * + * @rdev: radeon_device pointer + * @ib: radeon indirect buffer object + * + * Emits an DE (drawing engine) or CE (constant engine) IB + * on the gfx ring. IBs are usually generated by userspace + * acceleration drivers and submitted to the kernel for + * sheduling on the ring. This function schedules the IB + * on the gfx ring for execution by the GPU. + */ +void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) +{ + struct radeon_ring *ring = &rdev->ring[ib->ring]; + unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; + u32 header, control = INDIRECT_BUFFER_VALID; + + if (ib->is_const_ib) { + /* set switch buffer packet before const IB */ + radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); + radeon_ring_write(ring, 0); + + header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); + } else { + u32 next_rptr; + if (ring->rptr_save_reg) { + next_rptr = ring->wptr + 3 + 4; + radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); + radeon_ring_write(ring, ((ring->rptr_save_reg - + PACKET3_SET_UCONFIG_REG_START) >> 2)); + radeon_ring_write(ring, next_rptr); + } else if (rdev->wb.enabled) { + next_rptr = ring->wptr + 5 + 4; + radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + radeon_ring_write(ring, WRITE_DATA_DST_SEL(1)); + radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); + radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); + radeon_ring_write(ring, next_rptr); + } + + header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); + } + + control |= ib->length_dw | (vm_id << 24); + + radeon_ring_write(ring, header); + radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC)); + radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); + radeon_ring_write(ring, control); +} + +/** + * cik_ib_test - basic gfx ring IB test + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Allocate an IB and execute it on the gfx ring (CIK). + * Provides a basic gfx ring test to verify that IBs are working. + * Returns 0 on success, error on failure. + */ +int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) +{ + struct radeon_ib ib; + uint32_t scratch; + uint32_t tmp = 0; + unsigned i; + int r; + + r = radeon_scratch_get(rdev, &scratch); + if (r) { + DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r); + return r; + } + WREG32(scratch, 0xCAFEDEAD); + r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); + if (r) { + DRM_ERROR("radeon: failed to get ib (%d).\n", r); + radeon_scratch_free(rdev, scratch); + return r; + } + ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); + ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2); + ib.ptr[2] = 0xDEADBEEF; + ib.length_dw = 3; + r = radeon_ib_schedule(rdev, &ib, NULL, false); + if (r) { + radeon_scratch_free(rdev, scratch); + radeon_ib_free(rdev, &ib); + DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); + return r; + } + r = radeon_fence_wait(ib.fence, false); + if (r) { + DRM_ERROR("radeon: fence wait failed (%d).\n", r); + radeon_scratch_free(rdev, scratch); + radeon_ib_free(rdev, &ib); + return r; + } + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = RREG32(scratch); + if (tmp == 0xDEADBEEF) + break; + DRM_UDELAY(1); + } + if (i < rdev->usec_timeout) { + DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); + } else { + DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", + scratch, tmp); + r = -EINVAL; + } + radeon_scratch_free(rdev, scratch); + radeon_ib_free(rdev, &ib); + return r; +} + +/* + * CP. + * On CIK, gfx and compute now have independant command processors. + * + * GFX + * Gfx consists of a single ring and can process both gfx jobs and + * compute jobs. The gfx CP consists of three microengines (ME): + * PFP - Pre-Fetch Parser + * ME - Micro Engine + * CE - Constant Engine + * The PFP and ME make up what is considered the Drawing Engine (DE). + * The CE is an asynchronous engine used for updating buffer desciptors + * used by the DE so that they can be loaded into cache in parallel + * while the DE is processing state update packets. + * + * Compute + * The compute CP consists of two microengines (ME): + * MEC1 - Compute MicroEngine 1 + * MEC2 - Compute MicroEngine 2 + * Each MEC supports 4 compute pipes and each pipe supports 8 queues. + * The queues are exposed to userspace and are programmed directly + * by the compute runtime. + */ +/** + * cik_cp_gfx_enable - enable/disable the gfx CP MEs + * + * @rdev: radeon_device pointer + * @enable: enable or disable the MEs + * + * Halts or unhalts the gfx MEs. + */ +static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32(CP_ME_CNTL, 0); + else { + if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); + rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; + } + udelay(50); +} + +/** + * cik_cp_gfx_load_microcode - load the gfx CP ME ucode + * + * @rdev: radeon_device pointer + * + * Loads the gfx PFP, ME, and CE ucode. + * Returns 0 for success, -EINVAL if the ucode is not available. + */ +static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) +{ + int i; + + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) + return -EINVAL; + + cik_cp_gfx_enable(rdev, false); + + if (rdev->new_fw) { + const struct gfx_firmware_header_v1_0 *pfp_hdr = + (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; + const struct gfx_firmware_header_v1_0 *ce_hdr = + (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; + const struct gfx_firmware_header_v1_0 *me_hdr = + (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; + const __le32 *fw_data; + u32 fw_size; + + radeon_ucode_print_gfx_hdr(&pfp_hdr->header); + radeon_ucode_print_gfx_hdr(&ce_hdr->header); + radeon_ucode_print_gfx_hdr(&me_hdr->header); + + /* PFP */ + fw_data = (const __le32 *) + (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; + WREG32(CP_PFP_UCODE_ADDR, 0); + for (i = 0; i < fw_size; i++) + WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); + WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version)); + + /* CE */ + fw_data = (const __le32 *) + (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; + WREG32(CP_CE_UCODE_ADDR, 0); + for (i = 0; i < fw_size; i++) + WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); + WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version)); + + /* ME */ + fw_data = (const __be32 *) + (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; + WREG32(CP_ME_RAM_WADDR, 0); + for (i = 0; i < fw_size; i++) + WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++)); + WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version)); + WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version)); + } else { + const __be32 *fw_data; + + /* PFP */ + fw_data = (const __be32 *)rdev->pfp_fw->data; + WREG32(CP_PFP_UCODE_ADDR, 0); + for (i = 0; i < CIK_PFP_UCODE_SIZE; i++) + WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); + WREG32(CP_PFP_UCODE_ADDR, 0); + + /* CE */ + fw_data = (const __be32 *)rdev->ce_fw->data; + WREG32(CP_CE_UCODE_ADDR, 0); + for (i = 0; i < CIK_CE_UCODE_SIZE; i++) + WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); + WREG32(CP_CE_UCODE_ADDR, 0); + + /* ME */ + fw_data = (const __be32 *)rdev->me_fw->data; + WREG32(CP_ME_RAM_WADDR, 0); + for (i = 0; i < CIK_ME_UCODE_SIZE; i++) + WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); + WREG32(CP_ME_RAM_WADDR, 0); + } + + return 0; +} + +/** + * cik_cp_gfx_start - start the gfx ring + * + * @rdev: radeon_device pointer + * + * Enables the ring and loads the clear state context and other + * packets required to init the ring. + * Returns 0 for success, error for failure. + */ +static int cik_cp_gfx_start(struct radeon_device *rdev) +{ + struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; + int r, i; + + /* init the CP */ + WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); + WREG32(CP_ENDIAN_SWAP, 0); + WREG32(CP_DEVICE_ID, 1); + + cik_cp_gfx_enable(rdev, true); + + r = radeon_ring_lock(rdev, ring, cik_default_size + 17); + if (r) { + DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); + return r; + } + + /* init the CE partitions. CE only used for gfx on CIK */ + radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); + radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); + radeon_ring_write(ring, 0x8000); + radeon_ring_write(ring, 0x8000); + + /* setup clear context state */ + radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); + radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); + + radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); + radeon_ring_write(ring, 0x80000000); + radeon_ring_write(ring, 0x80000000); + + for (i = 0; i < cik_default_size; i++) + radeon_ring_write(ring, cik_default_state[i]); + + radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); + radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); + + /* set clear context state */ + radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); + radeon_ring_write(ring, 0); + + radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); + radeon_ring_write(ring, 0x00000316); + radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ + radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ + + radeon_ring_unlock_commit(rdev, ring, false); + + return 0; +} + +/** + * cik_cp_gfx_fini - stop the gfx ring + * + * @rdev: radeon_device pointer + * + * Stop the gfx ring and tear down the driver ring + * info. + */ +static void cik_cp_gfx_fini(struct radeon_device *rdev) +{ + cik_cp_gfx_enable(rdev, false); + radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); +} + +/** + * cik_cp_gfx_resume - setup the gfx ring buffer registers + * + * @rdev: radeon_device pointer + * + * Program the location and size of the gfx ring buffer + * and test it to make sure it's working. + * Returns 0 for success, error for failure. + */ +static int cik_cp_gfx_resume(struct radeon_device *rdev) +{ + struct radeon_ring *ring; + u32 tmp; + u32 rb_bufsz; + u64 rb_addr; + int r; + + WREG32(CP_SEM_WAIT_TIMER, 0x0); + if (rdev->family != CHIP_HAWAII) + WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); + + /* Set the write pointer delay */ + WREG32(CP_RB_WPTR_DELAY, 0); + + /* set the RB to use vmid 0 */ + WREG32(CP_RB_VMID, 0); + + WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); + + /* ring 0 - compute and gfx */ + /* Set ring buffer size */ + ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; + rb_bufsz = order_base_2(ring->ring_size / 8); + tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; +#ifdef __BIG_ENDIAN + tmp |= BUF_SWAP_32BIT; +#endif + WREG32(CP_RB0_CNTL, tmp); + + /* Initialize the ring buffer's read and write pointers */ + WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); + ring->wptr = 0; + WREG32(CP_RB0_WPTR, ring->wptr); + + /* set the wb address wether it's enabled or not */ + WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); + WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); + + /* scratch register shadowing is no longer supported */ + WREG32(SCRATCH_UMSK, 0); + + if (!rdev->wb.enabled) + tmp |= RB_NO_UPDATE; + + mdelay(1); + WREG32(CP_RB0_CNTL, tmp); + + rb_addr = ring->gpu_addr >> 8; + WREG32(CP_RB0_BASE, rb_addr); + WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr)); + + /* start the ring */ + cik_cp_gfx_start(rdev); + rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; + r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); + if (r) { + rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; + return r; + } + + if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); + + return 0; +} + +u32 cik_gfx_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 rptr; + + if (rdev->wb.enabled) + rptr = rdev->wb.wb[ring->rptr_offs/4]; + else + rptr = RREG32(CP_RB0_RPTR); + + return rptr; +} + +u32 cik_gfx_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 wptr; + + wptr = RREG32(CP_RB0_WPTR); + + return wptr; +} + +void cik_gfx_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + WREG32(CP_RB0_WPTR, ring->wptr); + (void)RREG32(CP_RB0_WPTR); +} + +u32 cik_compute_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 rptr; + + if (rdev->wb.enabled) { + rptr = rdev->wb.wb[ring->rptr_offs/4]; + } else { + mutex_lock(&rdev->srbm_mutex); + cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); + rptr = RREG32(CP_HQD_PQ_RPTR); + cik_srbm_select(rdev, 0, 0, 0, 0); + mutex_unlock(&rdev->srbm_mutex); + } + + return rptr; +} + +u32 cik_compute_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 wptr; + + if (rdev->wb.enabled) { + /* XXX check if swapping is necessary on BE */ + wptr = rdev->wb.wb[ring->wptr_offs/4]; + } else { + mutex_lock(&rdev->srbm_mutex); + cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); + wptr = RREG32(CP_HQD_PQ_WPTR); + cik_srbm_select(rdev, 0, 0, 0, 0); + mutex_unlock(&rdev->srbm_mutex); + } + + return wptr; +} + +void cik_compute_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + /* XXX check if swapping is necessary on BE */ + rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; + WDOORBELL32(ring->doorbell_index, ring->wptr); +} + +static void cik_compute_stop(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 j, tmp; + + cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); + /* Disable wptr polling. */ + tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); + tmp &= ~WPTR_POLL_EN; + WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); + /* Disable HQD. */ + if (RREG32(CP_HQD_ACTIVE) & 1) { + WREG32(CP_HQD_DEQUEUE_REQUEST, 1); + for (j = 0; j < rdev->usec_timeout; j++) { + if (!(RREG32(CP_HQD_ACTIVE) & 1)) + break; + udelay(1); + } + WREG32(CP_HQD_DEQUEUE_REQUEST, 0); + WREG32(CP_HQD_PQ_RPTR, 0); + WREG32(CP_HQD_PQ_WPTR, 0); + } + cik_srbm_select(rdev, 0, 0, 0, 0); +} + +/** + * cik_cp_compute_enable - enable/disable the compute CP MEs + * + * @rdev: radeon_device pointer + * @enable: enable or disable the MEs + * + * Halts or unhalts the compute MEs. + */ +static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32(CP_MEC_CNTL, 0); + else { + /* + * To make hibernation reliable we need to clear compute ring + * configuration before halting the compute ring. + */ + mutex_lock(&rdev->srbm_mutex); + cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); + cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); + mutex_unlock(&rdev->srbm_mutex); + + WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT)); + rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; + rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; + } + udelay(50); +} + +/** + * cik_cp_compute_load_microcode - load the compute CP ME ucode + * + * @rdev: radeon_device pointer + * + * Loads the compute MEC1&2 ucode. + * Returns 0 for success, -EINVAL if the ucode is not available. + */ +static int cik_cp_compute_load_microcode(struct radeon_device *rdev) +{ + int i; + + if (!rdev->mec_fw) + return -EINVAL; + + cik_cp_compute_enable(rdev, false); + + if (rdev->new_fw) { + const struct gfx_firmware_header_v1_0 *mec_hdr = + (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; + const __le32 *fw_data; + u32 fw_size; + + radeon_ucode_print_gfx_hdr(&mec_hdr->header); + + /* MEC1 */ + fw_data = (const __le32 *) + (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; + WREG32(CP_MEC_ME1_UCODE_ADDR, 0); + for (i = 0; i < fw_size; i++) + WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++)); + WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version)); + + /* MEC2 */ + if (rdev->family == CHIP_KAVERI) { + const struct gfx_firmware_header_v1_0 *mec2_hdr = + (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; + + fw_data = (const __le32 *) + (rdev->mec2_fw->data + + le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4; + WREG32(CP_MEC_ME2_UCODE_ADDR, 0); + for (i = 0; i < fw_size; i++) + WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++)); + WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version)); + } + } else { + const __be32 *fw_data; + + /* MEC1 */ + fw_data = (const __be32 *)rdev->mec_fw->data; + WREG32(CP_MEC_ME1_UCODE_ADDR, 0); + for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) + WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++)); + WREG32(CP_MEC_ME1_UCODE_ADDR, 0); + + if (rdev->family == CHIP_KAVERI) { + /* MEC2 */ + fw_data = (const __be32 *)rdev->mec_fw->data; + WREG32(CP_MEC_ME2_UCODE_ADDR, 0); + for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) + WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++)); + WREG32(CP_MEC_ME2_UCODE_ADDR, 0); + } + } + + return 0; +} + +/** + * cik_cp_compute_start - start the compute queues + * + * @rdev: radeon_device pointer + * + * Enable the compute queues. + * Returns 0 for success, error for failure. + */ +static int cik_cp_compute_start(struct radeon_device *rdev) +{ + cik_cp_compute_enable(rdev, true); + + return 0; +} + +/** + * cik_cp_compute_fini - stop the compute queues + * + * @rdev: radeon_device pointer + * + * Stop the compute queues and tear down the driver queue + * info. + */ +static void cik_cp_compute_fini(struct radeon_device *rdev) +{ + int i, idx, r; + + cik_cp_compute_enable(rdev, false); + + for (i = 0; i < 2; i++) { + if (i == 0) + idx = CAYMAN_RING_TYPE_CP1_INDEX; + else + idx = CAYMAN_RING_TYPE_CP2_INDEX; + + if (rdev->ring[idx].mqd_obj) { + r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); + if (unlikely(r != 0)) + dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); + + radeon_bo_unpin(rdev->ring[idx].mqd_obj); + radeon_bo_unreserve(rdev->ring[idx].mqd_obj); + + radeon_bo_unref(&rdev->ring[idx].mqd_obj); + rdev->ring[idx].mqd_obj = NULL; + } + } +} + +static void cik_mec_fini(struct radeon_device *rdev) +{ + int r; + + if (rdev->mec.hpd_eop_obj) { + r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); + if (unlikely(r != 0)) + dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); + radeon_bo_unpin(rdev->mec.hpd_eop_obj); + radeon_bo_unreserve(rdev->mec.hpd_eop_obj); + + radeon_bo_unref(&rdev->mec.hpd_eop_obj); + rdev->mec.hpd_eop_obj = NULL; + } +} + +#define MEC_HPD_SIZE 2048 + +static int cik_mec_init(struct radeon_device *rdev) +{ + int r; + u32 *hpd; + + /* + * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total + * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total + * Nonetheless, we assign only 1 pipe because all other pipes will + * be handled by KFD + */ + rdev->mec.num_mec = 1; + rdev->mec.num_pipe = 1; + rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; + + if (rdev->mec.hpd_eop_obj == NULL) { + r = radeon_bo_create(rdev, + rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, + PAGE_SIZE, true, + RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL, + &rdev->mec.hpd_eop_obj); + if (r) { + dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); + return r; + } + } + + r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); + if (unlikely(r != 0)) { + cik_mec_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, + &rdev->mec.hpd_eop_gpu_addr); + if (r) { + dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); + cik_mec_fini(rdev); + return r; + } + r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); + if (r) { + dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); + cik_mec_fini(rdev); + return r; + } + + /* clear memory. Not sure if this is required or not */ + memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); + + radeon_bo_kunmap(rdev->mec.hpd_eop_obj); + radeon_bo_unreserve(rdev->mec.hpd_eop_obj); + + return 0; +} + +struct hqd_registers +{ + u32 cp_mqd_base_addr; + u32 cp_mqd_base_addr_hi; + u32 cp_hqd_active; + u32 cp_hqd_vmid; + u32 cp_hqd_persistent_state; + u32 cp_hqd_pipe_priority; + u32 cp_hqd_queue_priority; + u32 cp_hqd_quantum; + u32 cp_hqd_pq_base; + u32 cp_hqd_pq_base_hi; + u32 cp_hqd_pq_rptr; + u32 cp_hqd_pq_rptr_report_addr; + u32 cp_hqd_pq_rptr_report_addr_hi; + u32 cp_hqd_pq_wptr_poll_addr; + u32 cp_hqd_pq_wptr_poll_addr_hi; + u32 cp_hqd_pq_doorbell_control; + u32 cp_hqd_pq_wptr; + u32 cp_hqd_pq_control; + u32 cp_hqd_ib_base_addr; + u32 cp_hqd_ib_base_addr_hi; + u32 cp_hqd_ib_rptr; + u32 cp_hqd_ib_control; + u32 cp_hqd_iq_timer; + u32 cp_hqd_iq_rptr; + u32 cp_hqd_dequeue_request; + u32 cp_hqd_dma_offload; + u32 cp_hqd_sema_cmd; + u32 cp_hqd_msg_type; + u32 cp_hqd_atomic0_preop_lo; + u32 cp_hqd_atomic0_preop_hi; + u32 cp_hqd_atomic1_preop_lo; + u32 cp_hqd_atomic1_preop_hi; + u32 cp_hqd_hq_scheduler0; + u32 cp_hqd_hq_scheduler1; + u32 cp_mqd_control; +}; + +struct bonaire_mqd +{ + u32 header; + u32 dispatch_initiator; + u32 dimensions[3]; + u32 start_idx[3]; + u32 num_threads[3]; + u32 pipeline_stat_enable; + u32 perf_counter_enable; + u32 pgm[2]; + u32 tba[2]; + u32 tma[2]; + u32 pgm_rsrc[2]; + u32 vmid; + u32 resource_limits; + u32 static_thread_mgmt01[2]; + u32 tmp_ring_size; + u32 static_thread_mgmt23[2]; + u32 restart[3]; + u32 thread_trace_enable; + u32 reserved1; + u32 user_data[16]; + u32 vgtcs_invoke_count[2]; + struct hqd_registers queue_state; + u32 dequeue_cntr; + u32 interrupt_queue[64]; +}; + +/** + * cik_cp_compute_resume - setup the compute queue registers + * + * @rdev: radeon_device pointer + * + * Program the compute queues and test them to make sure they + * are working. + * Returns 0 for success, error for failure. + */ +static int cik_cp_compute_resume(struct radeon_device *rdev) +{ + int r, i, j, idx; + u32 tmp; + bool use_doorbell = true; + u64 hqd_gpu_addr; + u64 mqd_gpu_addr; + u64 eop_gpu_addr; + u64 wb_gpu_addr; + u32 *buf; + struct bonaire_mqd *mqd; + + r = cik_cp_compute_start(rdev); + if (r) + return r; + + /* fix up chicken bits */ + tmp = RREG32(CP_CPF_DEBUG); + tmp |= (1 << 23); + WREG32(CP_CPF_DEBUG, tmp); + + /* init the pipes */ + mutex_lock(&rdev->srbm_mutex); + + eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr; + + cik_srbm_select(rdev, 0, 0, 0, 0); + + /* write the EOP addr */ + WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); + WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); + + /* set the VMID assigned */ + WREG32(CP_HPD_EOP_VMID, 0); + + /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ + tmp = RREG32(CP_HPD_EOP_CONTROL); + tmp &= ~EOP_SIZE_MASK; + tmp |= order_base_2(MEC_HPD_SIZE / 8); + WREG32(CP_HPD_EOP_CONTROL, tmp); + + mutex_unlock(&rdev->srbm_mutex); + + /* init the queues. Just two for now. */ + for (i = 0; i < 2; i++) { + if (i == 0) + idx = CAYMAN_RING_TYPE_CP1_INDEX; + else + idx = CAYMAN_RING_TYPE_CP2_INDEX; + + if (rdev->ring[idx].mqd_obj == NULL) { + r = radeon_bo_create(rdev, + sizeof(struct bonaire_mqd), + PAGE_SIZE, true, + RADEON_GEM_DOMAIN_GTT, 0, NULL, + NULL, &rdev->ring[idx].mqd_obj); + if (r) { + dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); + return r; + } + } + + r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); + if (unlikely(r != 0)) { + cik_cp_compute_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, + &mqd_gpu_addr); + if (r) { + dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); + cik_cp_compute_fini(rdev); + return r; + } + r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); + if (r) { + dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); + cik_cp_compute_fini(rdev); + return r; + } + + /* init the mqd struct */ + memset(buf, 0, sizeof(struct bonaire_mqd)); + + mqd = (struct bonaire_mqd *)buf; + mqd->header = 0xC0310800; + mqd->static_thread_mgmt01[0] = 0xffffffff; + mqd->static_thread_mgmt01[1] = 0xffffffff; + mqd->static_thread_mgmt23[0] = 0xffffffff; + mqd->static_thread_mgmt23[1] = 0xffffffff; + + mutex_lock(&rdev->srbm_mutex); + cik_srbm_select(rdev, rdev->ring[idx].me, + rdev->ring[idx].pipe, + rdev->ring[idx].queue, 0); + + /* disable wptr polling */ + tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); + tmp &= ~WPTR_POLL_EN; + WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); + + /* enable doorbell? */ + mqd->queue_state.cp_hqd_pq_doorbell_control = + RREG32(CP_HQD_PQ_DOORBELL_CONTROL); + if (use_doorbell) + mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; + else + mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN; + WREG32(CP_HQD_PQ_DOORBELL_CONTROL, + mqd->queue_state.cp_hqd_pq_doorbell_control); + + /* disable the queue if it's active */ + mqd->queue_state.cp_hqd_dequeue_request = 0; + mqd->queue_state.cp_hqd_pq_rptr = 0; + mqd->queue_state.cp_hqd_pq_wptr= 0; + if (RREG32(CP_HQD_ACTIVE) & 1) { + WREG32(CP_HQD_DEQUEUE_REQUEST, 1); + for (j = 0; j < rdev->usec_timeout; j++) { + if (!(RREG32(CP_HQD_ACTIVE) & 1)) + break; + udelay(1); + } + WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request); + WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr); + WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); + } + + /* set the pointer to the MQD */ + mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc; + mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); + WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr); + WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi); + /* set MQD vmid to 0 */ + mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL); + mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK; + WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control); + + /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ + hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; + mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr; + mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); + WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base); + WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi); + + /* set up the HQD, this is similar to CP_RB0_CNTL */ + mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL); + mqd->queue_state.cp_hqd_pq_control &= + ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK); + + mqd->queue_state.cp_hqd_pq_control |= + order_base_2(rdev->ring[idx].ring_size / 8); + mqd->queue_state.cp_hqd_pq_control |= + (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8); +#ifdef __BIG_ENDIAN + mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT; +#endif + mqd->queue_state.cp_hqd_pq_control &= + ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE); + mqd->queue_state.cp_hqd_pq_control |= + PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */ + WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control); + + /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */ + if (i == 0) + wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; + else + wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; + mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc; + mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; + WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr); + WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI, + mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi); + + /* set the wb address wether it's enabled or not */ + if (i == 0) + wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; + else + wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; + mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc; + mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi = + upper_32_bits(wb_gpu_addr) & 0xffff; + WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR, + mqd->queue_state.cp_hqd_pq_rptr_report_addr); + WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI, + mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi); + + /* enable the doorbell if requested */ + if (use_doorbell) { + mqd->queue_state.cp_hqd_pq_doorbell_control = + RREG32(CP_HQD_PQ_DOORBELL_CONTROL); + mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK; + mqd->queue_state.cp_hqd_pq_doorbell_control |= + DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); + mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; + mqd->queue_state.cp_hqd_pq_doorbell_control &= + ~(DOORBELL_SOURCE | DOORBELL_HIT); + + } else { + mqd->queue_state.cp_hqd_pq_doorbell_control = 0; + } + WREG32(CP_HQD_PQ_DOORBELL_CONTROL, + mqd->queue_state.cp_hqd_pq_doorbell_control); + + /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */ + rdev->ring[idx].wptr = 0; + mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; + WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); + mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR); + + /* set the vmid for the queue */ + mqd->queue_state.cp_hqd_vmid = 0; + WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid); + + /* activate the queue */ + mqd->queue_state.cp_hqd_active = 1; + WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active); + + cik_srbm_select(rdev, 0, 0, 0, 0); + mutex_unlock(&rdev->srbm_mutex); + + radeon_bo_kunmap(rdev->ring[idx].mqd_obj); + radeon_bo_unreserve(rdev->ring[idx].mqd_obj); + + rdev->ring[idx].ready = true; + r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); + if (r) + rdev->ring[idx].ready = false; + } + + return 0; +} + +static void cik_cp_enable(struct radeon_device *rdev, bool enable) +{ + cik_cp_gfx_enable(rdev, enable); + cik_cp_compute_enable(rdev, enable); +} + +static int cik_cp_load_microcode(struct radeon_device *rdev) +{ + int r; + + r = cik_cp_gfx_load_microcode(rdev); + if (r) + return r; + r = cik_cp_compute_load_microcode(rdev); + if (r) + return r; + + return 0; +} + +static void cik_cp_fini(struct radeon_device *rdev) +{ + cik_cp_gfx_fini(rdev); + cik_cp_compute_fini(rdev); +} + +static int cik_cp_resume(struct radeon_device *rdev) +{ + int r; + + cik_enable_gui_idle_interrupt(rdev, false); + + r = cik_cp_load_microcode(rdev); + if (r) + return r; + + r = cik_cp_gfx_resume(rdev); + if (r) + return r; + r = cik_cp_compute_resume(rdev); + if (r) + return r; + + cik_enable_gui_idle_interrupt(rdev, true); + + return 0; +} + +static void cik_print_gpu_status_regs(struct radeon_device *rdev) +{ + dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", + RREG32(GRBM_STATUS)); + dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", + RREG32(GRBM_STATUS2)); + dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", + RREG32(GRBM_STATUS_SE0)); + dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", + RREG32(GRBM_STATUS_SE1)); + dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", + RREG32(GRBM_STATUS_SE2)); + dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", + RREG32(GRBM_STATUS_SE3)); + dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", + RREG32(SRBM_STATUS)); + dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", + RREG32(SRBM_STATUS2)); + dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", + RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET)); + dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", + RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET)); + dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); + dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", + RREG32(CP_STALLED_STAT1)); + dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", + RREG32(CP_STALLED_STAT2)); + dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", + RREG32(CP_STALLED_STAT3)); + dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", + RREG32(CP_CPF_BUSY_STAT)); + dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", + RREG32(CP_CPF_STALLED_STAT1)); + dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); + dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); + dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", + RREG32(CP_CPC_STALLED_STAT1)); + dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); +} + +/** + * cik_gpu_check_soft_reset - check which blocks are busy + * + * @rdev: radeon_device pointer + * + * Check which blocks are busy and return the relevant reset + * mask to be used by cik_gpu_soft_reset(). + * Returns a mask of the blocks to be reset. + */ +u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) +{ + u32 reset_mask = 0; + u32 tmp; + + /* GRBM_STATUS */ + tmp = RREG32(GRBM_STATUS); + if (tmp & (PA_BUSY | SC_BUSY | + BCI_BUSY | SX_BUSY | + TA_BUSY | VGT_BUSY | + DB_BUSY | CB_BUSY | + GDS_BUSY | SPI_BUSY | + IA_BUSY | IA_BUSY_NO_DMA)) + reset_mask |= RADEON_RESET_GFX; + + if (tmp & (CP_BUSY | CP_COHERENCY_BUSY)) + reset_mask |= RADEON_RESET_CP; + + /* GRBM_STATUS2 */ + tmp = RREG32(GRBM_STATUS2); + if (tmp & RLC_BUSY) + reset_mask |= RADEON_RESET_RLC; + + /* SDMA0_STATUS_REG */ + tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET); + if (!(tmp & SDMA_IDLE)) + reset_mask |= RADEON_RESET_DMA; + + /* SDMA1_STATUS_REG */ + tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET); + if (!(tmp & SDMA_IDLE)) + reset_mask |= RADEON_RESET_DMA1; + + /* SRBM_STATUS2 */ + tmp = RREG32(SRBM_STATUS2); + if (tmp & SDMA_BUSY) + reset_mask |= RADEON_RESET_DMA; + + if (tmp & SDMA1_BUSY) + reset_mask |= RADEON_RESET_DMA1; + + /* SRBM_STATUS */ + tmp = RREG32(SRBM_STATUS); + + if (tmp & IH_BUSY) + reset_mask |= RADEON_RESET_IH; + + if (tmp & SEM_BUSY) + reset_mask |= RADEON_RESET_SEM; + + if (tmp & GRBM_RQ_PENDING) + reset_mask |= RADEON_RESET_GRBM; + + if (tmp & VMC_BUSY) + reset_mask |= RADEON_RESET_VMC; + + if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | + MCC_BUSY | MCD_BUSY)) + reset_mask |= RADEON_RESET_MC; + + if (evergreen_is_display_hung(rdev)) + reset_mask |= RADEON_RESET_DISPLAY; + + /* Skip MC reset as it's mostly likely not hung, just busy */ + if (reset_mask & RADEON_RESET_MC) { + DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); + reset_mask &= ~RADEON_RESET_MC; + } + + return reset_mask; +} + +/** + * cik_gpu_soft_reset - soft reset GPU + * + * @rdev: radeon_device pointer + * @reset_mask: mask of which blocks to reset + * + * Soft reset the blocks specified in @reset_mask. + */ +static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) +{ + struct evergreen_mc_save save; + u32 grbm_soft_reset = 0, srbm_soft_reset = 0; + u32 tmp; + + if (reset_mask == 0) + return; + + dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); + + cik_print_gpu_status_regs(rdev); + dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", + RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); + dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", + RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); + + /* disable CG/PG */ + cik_fini_pg(rdev); + cik_fini_cg(rdev); + + /* stop the rlc */ + cik_rlc_stop(rdev); + + /* Disable GFX parsing/prefetching */ + WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); + + /* Disable MEC parsing/prefetching */ + WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); + + if (reset_mask & RADEON_RESET_DMA) { + /* sdma0 */ + tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); + tmp |= SDMA_HALT; + WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); + } + if (reset_mask & RADEON_RESET_DMA1) { + /* sdma1 */ + tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); + tmp |= SDMA_HALT; + WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); + } + + evergreen_mc_stop(rdev, &save); + if (evergreen_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); + } + + if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) + grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX; + + if (reset_mask & RADEON_RESET_CP) { + grbm_soft_reset |= SOFT_RESET_CP; + + srbm_soft_reset |= SOFT_RESET_GRBM; + } + + if (reset_mask & RADEON_RESET_DMA) + srbm_soft_reset |= SOFT_RESET_SDMA; + + if (reset_mask & RADEON_RESET_DMA1) + srbm_soft_reset |= SOFT_RESET_SDMA1; + + if (reset_mask & RADEON_RESET_DISPLAY) + srbm_soft_reset |= SOFT_RESET_DC; + + if (reset_mask & RADEON_RESET_RLC) + grbm_soft_reset |= SOFT_RESET_RLC; + + if (reset_mask & RADEON_RESET_SEM) + srbm_soft_reset |= SOFT_RESET_SEM; + + if (reset_mask & RADEON_RESET_IH) + srbm_soft_reset |= SOFT_RESET_IH; + + if (reset_mask & RADEON_RESET_GRBM) + srbm_soft_reset |= SOFT_RESET_GRBM; + + if (reset_mask & RADEON_RESET_VMC) + srbm_soft_reset |= SOFT_RESET_VMC; + + if (!(rdev->flags & RADEON_IS_IGP)) { + if (reset_mask & RADEON_RESET_MC) + srbm_soft_reset |= SOFT_RESET_MC; + } + + if (grbm_soft_reset) { + tmp = RREG32(GRBM_SOFT_RESET); + tmp |= grbm_soft_reset; + dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(GRBM_SOFT_RESET, tmp); + tmp = RREG32(GRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~grbm_soft_reset; + WREG32(GRBM_SOFT_RESET, tmp); + tmp = RREG32(GRBM_SOFT_RESET); + } + + if (srbm_soft_reset) { + tmp = RREG32(SRBM_SOFT_RESET); + tmp |= srbm_soft_reset; + dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~srbm_soft_reset; + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); + } + + /* Wait a little for things to settle down */ + udelay(50); + + evergreen_mc_resume(rdev, &save); + udelay(50); + + cik_print_gpu_status_regs(rdev); +} + +struct kv_reset_save_regs { + u32 gmcon_reng_execute; + u32 gmcon_misc; + u32 gmcon_misc3; +}; + +static void kv_save_regs_for_reset(struct radeon_device *rdev, + struct kv_reset_save_regs *save) +{ + save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE); + save->gmcon_misc = RREG32(GMCON_MISC); + save->gmcon_misc3 = RREG32(GMCON_MISC3); + + WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP); + WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE | + STCTRL_STUTTER_EN)); +} + +static void kv_restore_regs_for_reset(struct radeon_device *rdev, + struct kv_reset_save_regs *save) +{ + int i; + + WREG32(GMCON_PGFSM_WRITE, 0); + WREG32(GMCON_PGFSM_CONFIG, 0x200010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0); + WREG32(GMCON_PGFSM_CONFIG, 0x300010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0x210000); + WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0x21003); + WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0x2b00); + WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0); + WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0x420000); + WREG32(GMCON_PGFSM_CONFIG, 0x100010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0x120202); + WREG32(GMCON_PGFSM_CONFIG, 0x500010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0x3e3e36); + WREG32(GMCON_PGFSM_CONFIG, 0x600010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0x373f3e); + WREG32(GMCON_PGFSM_CONFIG, 0x700010ff); + + for (i = 0; i < 5; i++) + WREG32(GMCON_PGFSM_WRITE, 0); + + WREG32(GMCON_PGFSM_WRITE, 0x3e1332); + WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff); + + WREG32(GMCON_MISC3, save->gmcon_misc3); + WREG32(GMCON_MISC, save->gmcon_misc); + WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute); +} + +static void cik_gpu_pci_config_reset(struct radeon_device *rdev) +{ + struct evergreen_mc_save save; + struct kv_reset_save_regs kv_save = { 0 }; + u32 tmp, i; + + dev_info(rdev->dev, "GPU pci config reset\n"); + + /* disable dpm? */ + + /* disable cg/pg */ + cik_fini_pg(rdev); + cik_fini_cg(rdev); + + /* Disable GFX parsing/prefetching */ + WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); + + /* Disable MEC parsing/prefetching */ + WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); + + /* sdma0 */ + tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); + tmp |= SDMA_HALT; + WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); + /* sdma1 */ + tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); + tmp |= SDMA_HALT; + WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); + /* XXX other engines? */ + + /* halt the rlc, disable cp internal ints */ + cik_rlc_stop(rdev); + + udelay(50); + + /* disable mem access */ + evergreen_mc_stop(rdev, &save); + if (evergreen_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); + } + + if (rdev->flags & RADEON_IS_IGP) + kv_save_regs_for_reset(rdev, &kv_save); + + /* disable BM */ + pci_clear_master(rdev->pdev); + /* reset */ + radeon_pci_config_reset(rdev); + + udelay(100); + + /* wait for asic to come out of reset */ + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) + break; + udelay(1); + } + + /* does asic init need to be run first??? */ + if (rdev->flags & RADEON_IS_IGP) + kv_restore_regs_for_reset(rdev, &kv_save); +} + +/** + * cik_asic_reset - soft reset GPU + * + * @rdev: radeon_device pointer + * + * Look up which blocks are hung and attempt + * to reset them. + * Returns 0 for success. + */ +int cik_asic_reset(struct radeon_device *rdev) +{ + u32 reset_mask; + + reset_mask = cik_gpu_check_soft_reset(rdev); + + if (reset_mask) + r600_set_bios_scratch_engine_hung(rdev, true); + + /* try soft reset */ + cik_gpu_soft_reset(rdev, reset_mask); + + reset_mask = cik_gpu_check_soft_reset(rdev); + + /* try pci config reset */ + if (reset_mask && radeon_hard_reset) + cik_gpu_pci_config_reset(rdev); + + reset_mask = cik_gpu_check_soft_reset(rdev); + + if (!reset_mask) + r600_set_bios_scratch_engine_hung(rdev, false); + + return 0; +} + +/** + * cik_gfx_is_lockup - check if the 3D engine is locked up + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Check if the 3D engine is locked up (CIK). + * Returns true if the engine is locked, false if not. + */ +bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +{ + u32 reset_mask = cik_gpu_check_soft_reset(rdev); + + if (!(reset_mask & (RADEON_RESET_GFX | + RADEON_RESET_COMPUTE | + RADEON_RESET_CP))) { + radeon_ring_lockup_update(rdev, ring); + return false; + } + return radeon_ring_test_lockup(rdev, ring); +} + +/* MC */ +/** + * cik_mc_program - program the GPU memory controller + * + * @rdev: radeon_device pointer + * + * Set the location of vram, gart, and AGP in the GPU's + * physical address space (CIK). + */ +static void cik_mc_program(struct radeon_device *rdev) +{ + struct evergreen_mc_save save; + u32 tmp; + int i, j; + + /* Initialize HDP */ + for (i = 0, j = 0; i < 32; i++, j += 0x18) { + WREG32((0x2c14 + j), 0x00000000); + WREG32((0x2c18 + j), 0x00000000); + WREG32((0x2c1c + j), 0x00000000); + WREG32((0x2c20 + j), 0x00000000); + WREG32((0x2c24 + j), 0x00000000); + } + WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); + + evergreen_mc_stop(rdev, &save); + if (radeon_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); + } + /* Lockout access through VGA aperture*/ + WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); + /* Update configuration */ + WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, + rdev->mc.vram_start >> 12); + WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, + rdev->mc.vram_end >> 12); + WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, + rdev->vram_scratch.gpu_addr >> 12); + tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; + tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); + WREG32(MC_VM_FB_LOCATION, tmp); + /* XXX double check these! */ + WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); + WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); + WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); + WREG32(MC_VM_AGP_BASE, 0); + WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); + WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); + if (radeon_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); + } + evergreen_mc_resume(rdev, &save); + /* we need to own VRAM, so turn off the VGA renderer here + * to stop it overwriting our objects */ + rv515_vga_render_disable(rdev); +} + +/** + * cik_mc_init - initialize the memory controller driver params + * + * @rdev: radeon_device pointer + * + * Look up the amount of vram, vram width, and decide how to place + * vram and gart within the GPU's physical address space (CIK). + * Returns 0 for success. + */ +static int cik_mc_init(struct radeon_device *rdev) +{ + u32 tmp; + int chansize, numchan; + + /* Get VRAM informations */ + rdev->mc.vram_is_ddr = true; + tmp = RREG32(MC_ARB_RAMCFG); + if (tmp & CHANSIZE_MASK) { + chansize = 64; + } else { + chansize = 32; + } + tmp = RREG32(MC_SHARED_CHMAP); + switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { + case 0: + default: + numchan = 1; + break; + case 1: + numchan = 2; + break; + case 2: + numchan = 4; + break; + case 3: + numchan = 8; + break; + case 4: + numchan = 3; + break; + case 5: + numchan = 6; + break; + case 6: + numchan = 10; + break; + case 7: + numchan = 12; + break; + case 8: + numchan = 16; + break; + } + rdev->mc.vram_width = numchan * chansize; + /* Could aper size report 0 ? */ + rdev->mc.aper_base = rdev->fb_aper_offset; + rdev->mc.aper_size = rdev->fb_aper_size; + /* size in MB on si */ + rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; + rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; + rdev->mc.visible_vram_size = rdev->mc.aper_size; + si_vram_gtt_location(rdev, &rdev->mc); + radeon_update_bandwidth_info(rdev); + + return 0; +} + +/* + * GART + * VMID 0 is the physical GPU addresses as used by the kernel. + * VMIDs 1-15 are used for userspace clients and are handled + * by the radeon vm/hsa code. + */ +/** + * cik_pcie_gart_tlb_flush - gart tlb flush callback + * + * @rdev: radeon_device pointer + * + * Flush the TLB for the VMID 0 page table (CIK). + */ +void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) +{ + /* flush hdp cache */ + WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0); + + /* bits 0-15 are the VM contexts0-15 */ + WREG32(VM_INVALIDATE_REQUEST, 0x1); +} + +static void cik_pcie_init_compute_vmid(struct radeon_device *rdev) +{ + int i; + uint32_t sh_mem_bases, sh_mem_config; + + sh_mem_bases = 0x6000 | 0x6000 << 16; + sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED); + sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED); + + mutex_lock(&rdev->srbm_mutex); + for (i = 8; i < 16; i++) { + cik_srbm_select(rdev, 0, 0, 0, i); + /* CP and shaders */ + WREG32(SH_MEM_CONFIG, sh_mem_config); + WREG32(SH_MEM_APE1_BASE, 1); + WREG32(SH_MEM_APE1_LIMIT, 0); + WREG32(SH_MEM_BASES, sh_mem_bases); + } + cik_srbm_select(rdev, 0, 0, 0, 0); + mutex_unlock(&rdev->srbm_mutex); +} + +/** + * cik_pcie_gart_enable - gart enable + * + * @rdev: radeon_device pointer + * + * This sets up the TLBs, programs the page tables for VMID0, + * sets up the hw for VMIDs 1-15 which are allocated on + * demand, and sets up the global locations for the LDS, GDS, + * and GPUVM for FSA64 clients (CIK). + * Returns 0 for success, errors for failure. + */ +static int cik_pcie_gart_enable(struct radeon_device *rdev) +{ + int r, i; + + if (rdev->gart.robj == NULL) { + dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); + return -EINVAL; + } + r = radeon_gart_table_vram_pin(rdev); + if (r) + return r; + /* Setup TLB control */ + WREG32(MC_VM_MX_L1_TLB_CNTL, + (0xA << 7) | + ENABLE_L1_TLB | + ENABLE_L1_FRAGMENT_PROCESSING | + SYSTEM_ACCESS_MODE_NOT_IN_SYS | + ENABLE_ADVANCED_DRIVER_MODEL | + SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); + /* Setup L2 cache */ + WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | + ENABLE_L2_FRAGMENT_PROCESSING | + ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | + ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | + EFFECTIVE_L2_QUEUE_SIZE(7) | + CONTEXT1_IDENTITY_ACCESS_MODE(1)); + WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); + WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | + BANK_SELECT(4) | + L2_CACHE_BIGK_FRAGMENT_SIZE(4)); + /* setup context0 */ + WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); + WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); + WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); + WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, + (u32)(rdev->dummy_page.addr >> 12)); + WREG32(VM_CONTEXT0_CNTL2, 0); + WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | + RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); + + WREG32(0x15D4, 0); + WREG32(0x15D8, 0); + WREG32(0x15DC, 0); + + /* restore context1-15 */ + /* set vm size, must be a multiple of 4 */ + WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); + WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1); + for (i = 1; i < 16; i++) { + if (i < 8) + WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), + rdev->vm_manager.saved_table_addr[i]); + else + WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), + rdev->vm_manager.saved_table_addr[i]); + } + + /* enable context1-15 */ + WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, + (u32)(rdev->dummy_page.addr >> 12)); + WREG32(VM_CONTEXT1_CNTL2, 4); + WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | + PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | + RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | + RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | + DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | + DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | + PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | + PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | + VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | + VALID_PROTECTION_FAULT_ENABLE_DEFAULT | + READ_PROTECTION_FAULT_ENABLE_INTERRUPT | + READ_PROTECTION_FAULT_ENABLE_DEFAULT | + WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | + WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); + + if (rdev->family == CHIP_KAVERI) { + u32 tmp = RREG32(CHUB_CONTROL); + tmp &= ~BYPASS_VM; + WREG32(CHUB_CONTROL, tmp); + } + + /* XXX SH_MEM regs */ + /* where to put LDS, scratch, GPUVM in FSA64 space */ + mutex_lock(&rdev->srbm_mutex); + for (i = 0; i < 16; i++) { + cik_srbm_select(rdev, 0, 0, 0, i); + /* CP and shaders */ + WREG32(SH_MEM_CONFIG, 0); + WREG32(SH_MEM_APE1_BASE, 1); + WREG32(SH_MEM_APE1_LIMIT, 0); + WREG32(SH_MEM_BASES, 0); + /* SDMA GFX */ + WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0); + WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0); + WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0); + WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0); + /* XXX SDMA RLC - todo */ + } + cik_srbm_select(rdev, 0, 0, 0, 0); + mutex_unlock(&rdev->srbm_mutex); + + cik_pcie_init_compute_vmid(rdev); + + cik_pcie_gart_tlb_flush(rdev); + DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", + (unsigned)(rdev->mc.gtt_size >> 20), + (unsigned long long)rdev->gart.table_addr); + rdev->gart.ready = true; + return 0; +} + +/** + * cik_pcie_gart_disable - gart disable + * + * @rdev: radeon_device pointer + * + * This disables all VM page table (CIK). + */ +static void cik_pcie_gart_disable(struct radeon_device *rdev) +{ + unsigned i; + + for (i = 1; i < 16; ++i) { + uint32_t reg; + if (i < 8) + reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2); + else + reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2); + rdev->vm_manager.saved_table_addr[i] = RREG32(reg); + } + + /* Disable all tables */ + WREG32(VM_CONTEXT0_CNTL, 0); + WREG32(VM_CONTEXT1_CNTL, 0); + /* Setup TLB control */ + WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | + SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); + /* Setup L2 cache */ + WREG32(VM_L2_CNTL, + ENABLE_L2_FRAGMENT_PROCESSING | + ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | + ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | + EFFECTIVE_L2_QUEUE_SIZE(7) | + CONTEXT1_IDENTITY_ACCESS_MODE(1)); + WREG32(VM_L2_CNTL2, 0); + WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | + L2_CACHE_BIGK_FRAGMENT_SIZE(6)); + radeon_gart_table_vram_unpin(rdev); +} + +/** + * cik_pcie_gart_fini - vm fini callback + * + * @rdev: radeon_device pointer + * + * Tears down the driver GART/VM setup (CIK). + */ +static void cik_pcie_gart_fini(struct radeon_device *rdev) +{ + cik_pcie_gart_disable(rdev); + radeon_gart_table_vram_free(rdev); + radeon_gart_fini(rdev); +} + +/* vm parser */ +/** + * cik_ib_parse - vm ib_parse callback + * + * @rdev: radeon_device pointer + * @ib: indirect buffer pointer + * + * CIK uses hw IB checking so this is a nop (CIK). + */ +int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) +{ + return 0; +} + +/* + * vm + * VMID 0 is the physical GPU addresses as used by the kernel. + * VMIDs 1-15 are used for userspace clients and are handled + * by the radeon vm/hsa code. + */ +/** + * cik_vm_init - cik vm init callback + * + * @rdev: radeon_device pointer + * + * Inits cik specific vm parameters (number of VMs, base of vram for + * VMIDs 1-15) (CIK). + * Returns 0 for success. + */ +int cik_vm_init(struct radeon_device *rdev) +{ + /* + * number of VMs + * VMID 0 is reserved for System + * radeon graphics/compute will use VMIDs 1-7 + * amdkfd will use VMIDs 8-15 + */ + rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS; + /* base offset of vram pages */ + if (rdev->flags & RADEON_IS_IGP) { + u64 tmp = RREG32(MC_VM_FB_OFFSET); + tmp <<= 22; + rdev->vm_manager.vram_base_offset = tmp; + } else + rdev->vm_manager.vram_base_offset = 0; + + return 0; +} + +/** + * cik_vm_fini - cik vm fini callback + * + * @rdev: radeon_device pointer + * + * Tear down any asic specific VM setup (CIK). + */ +void cik_vm_fini(struct radeon_device *rdev) +{ +} + +/** + * cik_vm_decode_fault - print human readable fault info + * + * @rdev: radeon_device pointer + * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value + * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value + * + * Print human readable fault information (CIK). + */ +static void cik_vm_decode_fault(struct radeon_device *rdev, + u32 status, u32 addr, u32 mc_client) +{ + u32 mc_id; + u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; + u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; + char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, + (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; + + if (rdev->family == CHIP_HAWAII) + mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; + else + mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; + + printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", + protections, vmid, addr, + (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", + block, mc_client, mc_id); +} + +/** + * cik_vm_flush - cik vm flush using the CP + * + * @rdev: radeon_device pointer + * + * Update the page table base and flush the VM TLB + * using the CP (CIK). + */ +void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) +{ + int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX); + + radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | + WRITE_DATA_DST_SEL(0))); + if (vm_id < 8) { + radeon_ring_write(ring, + (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); + } else { + radeon_ring_write(ring, + (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); + } + radeon_ring_write(ring, 0); + radeon_ring_write(ring, pd_addr >> 12); + + /* update SH_MEM_* regs */ + radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | + WRITE_DATA_DST_SEL(0))); + radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, VMID(vm_id)); + + radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6)); + radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | + WRITE_DATA_DST_SEL(0))); + radeon_ring_write(ring, SH_MEM_BASES >> 2); + radeon_ring_write(ring, 0); + + radeon_ring_write(ring, 0); /* SH_MEM_BASES */ + radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */ + radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */ + radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */ + + radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | + WRITE_DATA_DST_SEL(0))); + radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, VMID(0)); + + /* HDP flush */ + cik_hdp_flush_cp_ring_emit(rdev, ring->idx); + + /* bits 0-15 are the VM contexts0-15 */ + radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | + WRITE_DATA_DST_SEL(0))); + radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, 1 << vm_id); + + /* wait for the invalidate to complete */ + radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); + radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ + WAIT_REG_MEM_FUNCTION(0) | /* always */ + WAIT_REG_MEM_ENGINE(0))); /* me */ + radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, 0); /* ref */ + radeon_ring_write(ring, 0); /* mask */ + radeon_ring_write(ring, 0x20); /* poll interval */ + + /* compute doesn't have PFP */ + if (usepfp) { + /* sync PFP to ME, otherwise we might get invalid PFP reads */ + radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); + radeon_ring_write(ring, 0x0); + } +} + +/* + * RLC + * The RLC is a multi-purpose microengine that handles a + * variety of functions, the most important of which is + * the interrupt controller. + */ +static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, + bool enable) +{ + u32 tmp = RREG32(CP_INT_CNTL_RING0); + + if (enable) + tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + else + tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + WREG32(CP_INT_CNTL_RING0, tmp); +} + +static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) +{ + u32 tmp; + + tmp = RREG32(RLC_LB_CNTL); + if (enable) + tmp |= LOAD_BALANCE_ENABLE; + else + tmp &= ~LOAD_BALANCE_ENABLE; + WREG32(RLC_LB_CNTL, tmp); +} + +static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) +{ + u32 i, j, k; + u32 mask; + + mutex_lock(&rdev->grbm_idx_mutex); + for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { + for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { + cik_select_se_sh(rdev, i, j); + for (k = 0; k < rdev->usec_timeout; k++) { + if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0) + break; + udelay(1); + } + } + } + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + mutex_unlock(&rdev->grbm_idx_mutex); + + mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; + for (k = 0; k < rdev->usec_timeout; k++) { + if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) + break; + udelay(1); + } +} + +static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) +{ + u32 tmp; + + tmp = RREG32(RLC_CNTL); + if (tmp != rlc) + WREG32(RLC_CNTL, rlc); +} + +static u32 cik_halt_rlc(struct radeon_device *rdev) +{ + u32 data, orig; + + orig = data = RREG32(RLC_CNTL); + + if (data & RLC_ENABLE) { + u32 i; + + data &= ~RLC_ENABLE; + WREG32(RLC_CNTL, data); + + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0) + break; + udelay(1); + } + + cik_wait_for_rlc_serdes(rdev); + } + + return orig; +} + +void cik_enter_rlc_safe_mode(struct radeon_device *rdev) +{ + u32 tmp, i, mask; + + tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE); + WREG32(RLC_GPR_REG2, tmp); + + mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS; + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(RLC_GPM_STAT) & mask) == mask) + break; + udelay(1); + } + + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(RLC_GPR_REG2) & REQ) == 0) + break; + udelay(1); + } +} + +void cik_exit_rlc_safe_mode(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE); + WREG32(RLC_GPR_REG2, tmp); +} + +/** + * cik_rlc_stop - stop the RLC ME + * + * @rdev: radeon_device pointer + * + * Halt the RLC ME (MicroEngine) (CIK). + */ +static void cik_rlc_stop(struct radeon_device *rdev) +{ + WREG32(RLC_CNTL, 0); + + cik_enable_gui_idle_interrupt(rdev, false); + + cik_wait_for_rlc_serdes(rdev); +} + +/** + * cik_rlc_start - start the RLC ME + * + * @rdev: radeon_device pointer + * + * Unhalt the RLC ME (MicroEngine) (CIK). + */ +static void cik_rlc_start(struct radeon_device *rdev) +{ + WREG32(RLC_CNTL, RLC_ENABLE); + + cik_enable_gui_idle_interrupt(rdev, true); + + udelay(50); +} + +/** + * cik_rlc_resume - setup the RLC hw + * + * @rdev: radeon_device pointer + * + * Initialize the RLC registers, load the ucode, + * and start the RLC (CIK). + * Returns 0 for success, -EINVAL if the ucode is not available. + */ +static int cik_rlc_resume(struct radeon_device *rdev) +{ + u32 i, size, tmp; + + if (!rdev->rlc_fw) + return -EINVAL; + + cik_rlc_stop(rdev); + + /* disable CG */ + tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc; + WREG32(RLC_CGCG_CGLS_CTRL, tmp); + + si_rlc_reset(rdev); + + cik_init_pg(rdev); + + cik_init_cg(rdev); + + WREG32(RLC_LB_CNTR_INIT, 0); + WREG32(RLC_LB_CNTR_MAX, 0x00008000); + + mutex_lock(&rdev->grbm_idx_mutex); + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); + WREG32(RLC_LB_PARAMS, 0x00600408); + WREG32(RLC_LB_CNTL, 0x80000004); + mutex_unlock(&rdev->grbm_idx_mutex); + + WREG32(RLC_MC_CNTL, 0); + WREG32(RLC_UCODE_CNTL, 0); + + if (rdev->new_fw) { + const struct rlc_firmware_header_v1_0 *hdr = + (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; + const __le32 *fw_data = (const __le32 *) + (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + + radeon_ucode_print_rlc_hdr(&hdr->header); + + size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; + WREG32(RLC_GPM_UCODE_ADDR, 0); + for (i = 0; i < size; i++) + WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); + WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version)); + } else { + const __be32 *fw_data; + + switch (rdev->family) { + case CHIP_BONAIRE: + case CHIP_HAWAII: + default: + size = BONAIRE_RLC_UCODE_SIZE; + break; + case CHIP_KAVERI: + size = KV_RLC_UCODE_SIZE; + break; + case CHIP_KABINI: + size = KB_RLC_UCODE_SIZE; + break; + case CHIP_MULLINS: + size = ML_RLC_UCODE_SIZE; + break; + } + + fw_data = (const __be32 *)rdev->rlc_fw->data; + WREG32(RLC_GPM_UCODE_ADDR, 0); + for (i = 0; i < size; i++) + WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++)); + WREG32(RLC_GPM_UCODE_ADDR, 0); + } + + /* XXX - find out what chips support lbpw */ + cik_enable_lbpw(rdev, false); + + if (rdev->family == CHIP_BONAIRE) + WREG32(RLC_DRIVER_DMA_STATUS, 0); + + cik_rlc_start(rdev); + + return 0; +} + +static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) +{ + u32 data, orig, tmp, tmp2; + + orig = data = RREG32(RLC_CGCG_CGLS_CTRL); + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { + cik_enable_gui_idle_interrupt(rdev, true); + + tmp = cik_halt_rlc(rdev); + + mutex_lock(&rdev->grbm_idx_mutex); + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); + WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); + tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; + WREG32(RLC_SERDES_WR_CTRL, tmp2); + mutex_unlock(&rdev->grbm_idx_mutex); + + cik_update_rlc(rdev, tmp); + + data |= CGCG_EN | CGLS_EN; + } else { + cik_enable_gui_idle_interrupt(rdev, false); + + RREG32(CB_CGTT_SCLK_CTRL); + RREG32(CB_CGTT_SCLK_CTRL); + RREG32(CB_CGTT_SCLK_CTRL); + RREG32(CB_CGTT_SCLK_CTRL); + + data &= ~(CGCG_EN | CGLS_EN); + } + + if (orig != data) + WREG32(RLC_CGCG_CGLS_CTRL, data); + +} + +static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) +{ + u32 data, orig, tmp = 0; + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { + if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { + if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { + orig = data = RREG32(CP_MEM_SLP_CNTL); + data |= CP_MEM_LS_EN; + if (orig != data) + WREG32(CP_MEM_SLP_CNTL, data); + } + } + + orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); + data |= 0x00000001; + data &= 0xfffffffd; + if (orig != data) + WREG32(RLC_CGTT_MGCG_OVERRIDE, data); + + tmp = cik_halt_rlc(rdev); + + mutex_lock(&rdev->grbm_idx_mutex); + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); + WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); + data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; + WREG32(RLC_SERDES_WR_CTRL, data); + mutex_unlock(&rdev->grbm_idx_mutex); + + cik_update_rlc(rdev, tmp); + + if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { + orig = data = RREG32(CGTS_SM_CTRL_REG); + data &= ~SM_MODE_MASK; + data |= SM_MODE(0x2); + data |= SM_MODE_ENABLE; + data &= ~CGTS_OVERRIDE; + if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && + (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) + data &= ~CGTS_LS_OVERRIDE; + data &= ~ON_MONITOR_ADD_MASK; + data |= ON_MONITOR_ADD_EN; + data |= ON_MONITOR_ADD(0x96); + if (orig != data) + WREG32(CGTS_SM_CTRL_REG, data); + } + } else { + orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); + data |= 0x00000003; + if (orig != data) + WREG32(RLC_CGTT_MGCG_OVERRIDE, data); + + data = RREG32(RLC_MEM_SLP_CNTL); + if (data & RLC_MEM_LS_EN) { + data &= ~RLC_MEM_LS_EN; + WREG32(RLC_MEM_SLP_CNTL, data); + } + + data = RREG32(CP_MEM_SLP_CNTL); + if (data & CP_MEM_LS_EN) { + data &= ~CP_MEM_LS_EN; + WREG32(CP_MEM_SLP_CNTL, data); + } + + orig = data = RREG32(CGTS_SM_CTRL_REG); + data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE; + if (orig != data) + WREG32(CGTS_SM_CTRL_REG, data); + + tmp = cik_halt_rlc(rdev); + + mutex_lock(&rdev->grbm_idx_mutex); + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); + WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); + data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; + WREG32(RLC_SERDES_WR_CTRL, data); + mutex_unlock(&rdev->grbm_idx_mutex); + + cik_update_rlc(rdev, tmp); + } +} + +static const u32 mc_cg_registers[] = +{ + MC_HUB_MISC_HUB_CG, + MC_HUB_MISC_SIP_CG, + MC_HUB_MISC_VM_CG, + MC_XPB_CLK_GAT, + ATC_MISC_CG, + MC_CITF_MISC_WR_CG, + MC_CITF_MISC_RD_CG, + MC_CITF_MISC_VM_CG, + VM_L2_CG, +}; + +static void cik_enable_mc_ls(struct radeon_device *rdev, + bool enable) +{ + int i; + u32 orig, data; + + for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { + orig = data = RREG32(mc_cg_registers[i]); + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) + data |= MC_LS_ENABLE; + else + data &= ~MC_LS_ENABLE; + if (data != orig) + WREG32(mc_cg_registers[i], data); + } +} + +static void cik_enable_mc_mgcg(struct radeon_device *rdev, + bool enable) +{ + int i; + u32 orig, data; + + for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { + orig = data = RREG32(mc_cg_registers[i]); + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) + data |= MC_CG_ENABLE; + else + data &= ~MC_CG_ENABLE; + if (data != orig) + WREG32(mc_cg_registers[i], data); + } +} + +static void cik_enable_sdma_mgcg(struct radeon_device *rdev, + bool enable) +{ + u32 orig, data; + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { + WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100); + WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100); + } else { + orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET); + data |= 0xff000000; + if (data != orig) + WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data); + + orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET); + data |= 0xff000000; + if (data != orig) + WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data); + } +} + +static void cik_enable_sdma_mgls(struct radeon_device *rdev, + bool enable) +{ + u32 orig, data; + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { + orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); + data |= 0x100; + if (orig != data) + WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); + + orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); + data |= 0x100; + if (orig != data) + WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); + } else { + orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); + data &= ~0x100; + if (orig != data) + WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); + + orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); + data &= ~0x100; + if (orig != data) + WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); + } +} + +static void cik_enable_uvd_mgcg(struct radeon_device *rdev, + bool enable) +{ + u32 orig, data; + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { + data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); + data = 0xfff; + WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); + + orig = data = RREG32(UVD_CGC_CTRL); + data |= DCM; + if (orig != data) + WREG32(UVD_CGC_CTRL, data); + } else { + data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); + data &= ~0xfff; + WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); + + orig = data = RREG32(UVD_CGC_CTRL); + data &= ~DCM; + if (orig != data) + WREG32(UVD_CGC_CTRL, data); + } +} + +static void cik_enable_bif_mgls(struct radeon_device *rdev, + bool enable) +{ + u32 orig, data; + + orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) + data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | + REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; + else + data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | + REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); + + if (orig != data) + WREG32_PCIE_PORT(PCIE_CNTL2, data); +} + +static void cik_enable_hdp_mgcg(struct radeon_device *rdev, + bool enable) +{ + u32 orig, data; + + orig = data = RREG32(HDP_HOST_PATH_CNTL); + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) + data &= ~CLOCK_GATING_DIS; + else + data |= CLOCK_GATING_DIS; + + if (orig != data) + WREG32(HDP_HOST_PATH_CNTL, data); +} + +static void cik_enable_hdp_ls(struct radeon_device *rdev, + bool enable) +{ + u32 orig, data; + + orig = data = RREG32(HDP_MEM_POWER_LS); + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) + data |= HDP_LS_ENABLE; + else + data &= ~HDP_LS_ENABLE; + + if (orig != data) + WREG32(HDP_MEM_POWER_LS, data); +} + +void cik_update_cg(struct radeon_device *rdev, + u32 block, bool enable) +{ + + if (block & RADEON_CG_BLOCK_GFX) { + cik_enable_gui_idle_interrupt(rdev, false); + /* order matters! */ + if (enable) { + cik_enable_mgcg(rdev, true); + cik_enable_cgcg(rdev, true); + } else { + cik_enable_cgcg(rdev, false); + cik_enable_mgcg(rdev, false); + } + cik_enable_gui_idle_interrupt(rdev, true); + } + + if (block & RADEON_CG_BLOCK_MC) { + if (!(rdev->flags & RADEON_IS_IGP)) { + cik_enable_mc_mgcg(rdev, enable); + cik_enable_mc_ls(rdev, enable); + } + } + + if (block & RADEON_CG_BLOCK_SDMA) { + cik_enable_sdma_mgcg(rdev, enable); + cik_enable_sdma_mgls(rdev, enable); + } + + if (block & RADEON_CG_BLOCK_BIF) { + cik_enable_bif_mgls(rdev, enable); + } + + if (block & RADEON_CG_BLOCK_UVD) { + if (rdev->has_uvd) + cik_enable_uvd_mgcg(rdev, enable); + } + + if (block & RADEON_CG_BLOCK_HDP) { + cik_enable_hdp_mgcg(rdev, enable); + cik_enable_hdp_ls(rdev, enable); + } + + if (block & RADEON_CG_BLOCK_VCE) { + vce_v2_0_enable_mgcg(rdev, enable); + } +} + +static void cik_init_cg(struct radeon_device *rdev) +{ + + cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); + + if (rdev->has_uvd) + si_init_uvd_internal_cg(rdev); + + cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | + RADEON_CG_BLOCK_SDMA | + RADEON_CG_BLOCK_BIF | + RADEON_CG_BLOCK_UVD | + RADEON_CG_BLOCK_HDP), true); +} + +static void cik_fini_cg(struct radeon_device *rdev) +{ + cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | + RADEON_CG_BLOCK_SDMA | + RADEON_CG_BLOCK_BIF | + RADEON_CG_BLOCK_UVD | + RADEON_CG_BLOCK_HDP), false); + + cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); +} + +static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, + bool enable) +{ + u32 data, orig; + + orig = data = RREG32(RLC_PG_CNTL); + if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) + data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE; + else + data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE; + if (orig != data) + WREG32(RLC_PG_CNTL, data); +} + +static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, + bool enable) +{ + u32 data, orig; + + orig = data = RREG32(RLC_PG_CNTL); + if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) + data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE; + else + data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE; + if (orig != data) + WREG32(RLC_PG_CNTL, data); +} + +static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) +{ + u32 data, orig; + + orig = data = RREG32(RLC_PG_CNTL); + if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) + data &= ~DISABLE_CP_PG; + else + data |= DISABLE_CP_PG; + if (orig != data) + WREG32(RLC_PG_CNTL, data); +} + +static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) +{ + u32 data, orig; + + orig = data = RREG32(RLC_PG_CNTL); + if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) + data &= ~DISABLE_GDS_PG; + else + data |= DISABLE_GDS_PG; + if (orig != data) + WREG32(RLC_PG_CNTL, data); +} + +#define CP_ME_TABLE_SIZE 96 +#define CP_ME_TABLE_OFFSET 2048 +#define CP_MEC_TABLE_OFFSET 4096 + +void cik_init_cp_pg_table(struct radeon_device *rdev) +{ + volatile u32 *dst_ptr; + int me, i, max_me = 4; + u32 bo_offset = 0; + u32 table_offset, table_size; + + if (rdev->family == CHIP_KAVERI) + max_me = 5; + + if (rdev->rlc.cp_table_ptr == NULL) + return; + + /* write the cp table buffer */ + dst_ptr = rdev->rlc.cp_table_ptr; + for (me = 0; me < max_me; me++) { + if (rdev->new_fw) { + const __le32 *fw_data; + const struct gfx_firmware_header_v1_0 *hdr; + + if (me == 0) { + hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; + fw_data = (const __le32 *) + (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 1) { + hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; + fw_data = (const __le32 *) + (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 2) { + hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; + fw_data = (const __le32 *) + (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 3) { + hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; + fw_data = (const __le32 *) + (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else { + hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; + fw_data = (const __le32 *) + (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } + + for (i = 0; i < table_size; i ++) { + dst_ptr[bo_offset + i] = + cpu_to_le32(le32_to_cpu(fw_data[table_offset + i])); + } + bo_offset += table_size; + } else { + const __be32 *fw_data; + table_size = CP_ME_TABLE_SIZE; + + if (me == 0) { + fw_data = (const __be32 *)rdev->ce_fw->data; + table_offset = CP_ME_TABLE_OFFSET; + } else if (me == 1) { + fw_data = (const __be32 *)rdev->pfp_fw->data; + table_offset = CP_ME_TABLE_OFFSET; + } else if (me == 2) { + fw_data = (const __be32 *)rdev->me_fw->data; + table_offset = CP_ME_TABLE_OFFSET; + } else { + fw_data = (const __be32 *)rdev->mec_fw->data; + table_offset = CP_MEC_TABLE_OFFSET; + } + + for (i = 0; i < table_size; i ++) { + dst_ptr[bo_offset + i] = + cpu_to_le32(be32_to_cpu(fw_data[table_offset + i])); + } + bo_offset += table_size; + } + } +} + +static void cik_enable_gfx_cgpg(struct radeon_device *rdev, + bool enable) +{ + u32 data, orig; + + if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { + orig = data = RREG32(RLC_PG_CNTL); + data |= GFX_PG_ENABLE; + if (orig != data) + WREG32(RLC_PG_CNTL, data); + + orig = data = RREG32(RLC_AUTO_PG_CTRL); + data |= AUTO_PG_EN; + if (orig != data) + WREG32(RLC_AUTO_PG_CTRL, data); + } else { + orig = data = RREG32(RLC_PG_CNTL); + data &= ~GFX_PG_ENABLE; + if (orig != data) + WREG32(RLC_PG_CNTL, data); + + orig = data = RREG32(RLC_AUTO_PG_CTRL); + data &= ~AUTO_PG_EN; + if (orig != data) + WREG32(RLC_AUTO_PG_CTRL, data); + + data = RREG32(DB_RENDER_CONTROL); + } +} + +static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) +{ + u32 mask = 0, tmp, tmp1; + int i; + + mutex_lock(&rdev->grbm_idx_mutex); + cik_select_se_sh(rdev, se, sh); + tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); + tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + mutex_unlock(&rdev->grbm_idx_mutex); + + tmp &= 0xffff0000; + + tmp |= tmp1; + tmp >>= 16; + + for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { + mask <<= 1; + mask |= 1; + } + + return (~tmp) & mask; +} + +static void cik_init_ao_cu_mask(struct radeon_device *rdev) +{ + u32 i, j, k, active_cu_number = 0; + u32 mask, counter, cu_bitmap; + u32 tmp = 0; + + for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { + for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { + mask = 1; + cu_bitmap = 0; + counter = 0; + for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { + if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { + if (counter < 2) + cu_bitmap |= mask; + counter ++; + } + mask <<= 1; + } + + active_cu_number += counter; + tmp |= (cu_bitmap << (i * 16 + j * 8)); + } + } + + WREG32(RLC_PG_AO_CU_MASK, tmp); + + tmp = RREG32(RLC_MAX_PG_CU); + tmp &= ~MAX_PU_CU_MASK; + tmp |= MAX_PU_CU(active_cu_number); + WREG32(RLC_MAX_PG_CU, tmp); +} + +static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, + bool enable) +{ + u32 data, orig; + + orig = data = RREG32(RLC_PG_CNTL); + if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) + data |= STATIC_PER_CU_PG_ENABLE; + else + data &= ~STATIC_PER_CU_PG_ENABLE; + if (orig != data) + WREG32(RLC_PG_CNTL, data); +} + +static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, + bool enable) +{ + u32 data, orig; + + orig = data = RREG32(RLC_PG_CNTL); + if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) + data |= DYN_PER_CU_PG_ENABLE; + else + data &= ~DYN_PER_CU_PG_ENABLE; + if (orig != data) + WREG32(RLC_PG_CNTL, data); +} + +#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90 +#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D + +static void cik_init_gfx_cgpg(struct radeon_device *rdev) +{ + u32 data, orig; + u32 i; + + if (rdev->rlc.cs_data) { + WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); + WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); + WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); + WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); + } else { + WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); + for (i = 0; i < 3; i++) + WREG32(RLC_GPM_SCRATCH_DATA, 0); + } + if (rdev->rlc.reg_list) { + WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET); + for (i = 0; i < rdev->rlc.reg_list_size; i++) + WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); + } + + orig = data = RREG32(RLC_PG_CNTL); + data |= GFX_PG_SRC; + if (orig != data) + WREG32(RLC_PG_CNTL, data); + + WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); + WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); + + data = RREG32(CP_RB_WPTR_POLL_CNTL); + data &= ~IDLE_POLL_COUNT_MASK; + data |= IDLE_POLL_COUNT(0x60); + WREG32(CP_RB_WPTR_POLL_CNTL, data); + + data = 0x10101010; + WREG32(RLC_PG_DELAY, data); + + data = RREG32(RLC_PG_DELAY_2); + data &= ~0xff; + data |= 0x3; + WREG32(RLC_PG_DELAY_2, data); + + data = RREG32(RLC_AUTO_PG_CTRL); + data &= ~GRBM_REG_SGIT_MASK; + data |= GRBM_REG_SGIT(0x700); + WREG32(RLC_AUTO_PG_CTRL, data); + +} + +static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) +{ + cik_enable_gfx_cgpg(rdev, enable); + cik_enable_gfx_static_mgpg(rdev, enable); + cik_enable_gfx_dynamic_mgpg(rdev, enable); +} + +u32 cik_get_csb_size(struct radeon_device *rdev) +{ + u32 count = 0; + const struct cs_section_def *sect = NULL; + const struct cs_extent_def *ext = NULL; + + if (rdev->rlc.cs_data == NULL) + return 0; + + /* begin clear state */ + count += 2; + /* context control state */ + count += 3; + + for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { + for (ext = sect->section; ext->extent != NULL; ++ext) { + if (sect->id == SECT_CONTEXT) + count += 2 + ext->reg_count; + else + return 0; + } + } + /* pa_sc_raster_config/pa_sc_raster_config1 */ + count += 4; + /* end clear state */ + count += 2; + /* clear state */ + count += 2; + + return count; +} + +void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) +{ + u32 count = 0, i; + const struct cs_section_def *sect = NULL; + const struct cs_extent_def *ext = NULL; + + if (rdev->rlc.cs_data == NULL) + return; + if (buffer == NULL) + return; + + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); + buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); + + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); + buffer[count++] = cpu_to_le32(0x80000000); + buffer[count++] = cpu_to_le32(0x80000000); + + for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { + for (ext = sect->section; ext->extent != NULL; ++ext) { + if (sect->id == SECT_CONTEXT) { + buffer[count++] = + cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); + buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); + for (i = 0; i < ext->reg_count; i++) + buffer[count++] = cpu_to_le32(ext->extent[i]); + } else { + return; + } + } + } + + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); + buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); + switch (rdev->family) { + case CHIP_BONAIRE: + buffer[count++] = cpu_to_le32(0x16000012); + buffer[count++] = cpu_to_le32(0x00000000); + break; + case CHIP_KAVERI: + buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ + buffer[count++] = cpu_to_le32(0x00000000); + break; + case CHIP_KABINI: + case CHIP_MULLINS: + buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ + buffer[count++] = cpu_to_le32(0x00000000); + break; + case CHIP_HAWAII: + buffer[count++] = cpu_to_le32(0x3a00161a); + buffer[count++] = cpu_to_le32(0x0000002e); + break; + default: + buffer[count++] = cpu_to_le32(0x00000000); + buffer[count++] = cpu_to_le32(0x00000000); + break; + } + + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); + buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); + + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); + buffer[count++] = cpu_to_le32(0); +} + +static void cik_init_pg(struct radeon_device *rdev) +{ + if (rdev->pg_flags) { + cik_enable_sck_slowdown_on_pu(rdev, true); + cik_enable_sck_slowdown_on_pd(rdev, true); + if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { + cik_init_gfx_cgpg(rdev); + cik_enable_cp_pg(rdev, true); + cik_enable_gds_pg(rdev, true); + } + cik_init_ao_cu_mask(rdev); + cik_update_gfx_pg(rdev, true); + } +} + +static void cik_fini_pg(struct radeon_device *rdev) +{ + if (rdev->pg_flags) { + cik_update_gfx_pg(rdev, false); + if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { + cik_enable_cp_pg(rdev, false); + cik_enable_gds_pg(rdev, false); + } + } +} + +/* + * Interrupts + * Starting with r6xx, interrupts are handled via a ring buffer. + * Ring buffers are areas of GPU accessible memory that the GPU + * writes interrupt vectors into and the host reads vectors out of. + * There is a rptr (read pointer) that determines where the + * host is currently reading, and a wptr (write pointer) + * which determines where the GPU has written. When the + * pointers are equal, the ring is idle. When the GPU + * writes vectors to the ring buffer, it increments the + * wptr. When there is an interrupt, the host then starts + * fetching commands and processing them until the pointers are + * equal again at which point it updates the rptr. + */ + +/** + * cik_enable_interrupts - Enable the interrupt ring buffer + * + * @rdev: radeon_device pointer + * + * Enable the interrupt ring buffer (CIK). + */ +static void cik_enable_interrupts(struct radeon_device *rdev) +{ + u32 ih_cntl = RREG32(IH_CNTL); + u32 ih_rb_cntl = RREG32(IH_RB_CNTL); + + ih_cntl |= ENABLE_INTR; + ih_rb_cntl |= IH_RB_ENABLE; + WREG32(IH_CNTL, ih_cntl); + WREG32(IH_RB_CNTL, ih_rb_cntl); + rdev->ih.enabled = true; +} + +/** + * cik_disable_interrupts - Disable the interrupt ring buffer + * + * @rdev: radeon_device pointer + * + * Disable the interrupt ring buffer (CIK). + */ +static void cik_disable_interrupts(struct radeon_device *rdev) +{ + u32 ih_rb_cntl = RREG32(IH_RB_CNTL); + u32 ih_cntl = RREG32(IH_CNTL); + + ih_rb_cntl &= ~IH_RB_ENABLE; + ih_cntl &= ~ENABLE_INTR; + WREG32(IH_RB_CNTL, ih_rb_cntl); + WREG32(IH_CNTL, ih_cntl); + /* set rptr, wptr to 0 */ + WREG32(IH_RB_RPTR, 0); + WREG32(IH_RB_WPTR, 0); + rdev->ih.enabled = false; + rdev->ih.rptr = 0; +} + +/** + * cik_disable_interrupt_state - Disable all interrupt sources + * + * @rdev: radeon_device pointer + * + * Clear all interrupt enable bits used by the driver (CIK). + */ +static void cik_disable_interrupt_state(struct radeon_device *rdev) +{ + u32 tmp; + + /* gfx ring */ + tmp = RREG32(CP_INT_CNTL_RING0) & + (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + WREG32(CP_INT_CNTL_RING0, tmp); + /* sdma */ + tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; + WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp); + tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; + WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp); + /* compute queues */ + WREG32(CP_ME1_PIPE0_INT_CNTL, 0); + WREG32(CP_ME1_PIPE1_INT_CNTL, 0); + WREG32(CP_ME1_PIPE2_INT_CNTL, 0); + WREG32(CP_ME1_PIPE3_INT_CNTL, 0); + WREG32(CP_ME2_PIPE0_INT_CNTL, 0); + WREG32(CP_ME2_PIPE1_INT_CNTL, 0); + WREG32(CP_ME2_PIPE2_INT_CNTL, 0); + WREG32(CP_ME2_PIPE3_INT_CNTL, 0); + /* grbm */ + WREG32(GRBM_INT_CNTL, 0); + /* SRBM */ + WREG32(SRBM_INT_CNTL, 0); + /* vline/vblank, etc. */ + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); + if (rdev->num_crtc >= 4) { + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); + } + if (rdev->num_crtc >= 6) { + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); + } + /* pflip */ + if (rdev->num_crtc >= 2) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); + } + if (rdev->num_crtc >= 4) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); + } + if (rdev->num_crtc >= 6) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); + } + + /* dac hotplug */ + WREG32(DAC_AUTODETECT_INT_CONTROL, 0); + + /* digital hotplug */ + tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD1_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD2_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD3_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD4_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD5_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD6_INT_CONTROL, tmp); + +} + +/** + * cik_irq_init - init and enable the interrupt ring + * + * @rdev: radeon_device pointer + * + * Allocate a ring buffer for the interrupt controller, + * enable the RLC, disable interrupts, enable the IH + * ring buffer and enable it (CIK). + * Called at device load and reume. + * Returns 0 for success, errors for failure. + */ +static int cik_irq_init(struct radeon_device *rdev) +{ + int ret = 0; + int rb_bufsz; + u32 interrupt_cntl, ih_cntl, ih_rb_cntl; + + /* allocate ring */ + ret = r600_ih_ring_alloc(rdev); + if (ret) + return ret; + + /* disable irqs */ + cik_disable_interrupts(rdev); + + /* init rlc */ + ret = cik_rlc_resume(rdev); + if (ret) { + r600_ih_ring_fini(rdev); + return ret; + } + + /* setup interrupt control */ + /* XXX this should actually be a bus address, not an MC address. same on older asics */ + WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); + interrupt_cntl = RREG32(INTERRUPT_CNTL); + /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi + * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN + */ + interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; + /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ + interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; + WREG32(INTERRUPT_CNTL, interrupt_cntl); + + WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); + rb_bufsz = order_base_2(rdev->ih.ring_size / 4); + + ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | + IH_WPTR_OVERFLOW_CLEAR | + (rb_bufsz << 1)); + + if (rdev->wb.enabled) + ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; + + /* set the writeback address whether it's enabled or not */ + WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); + WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); + + WREG32(IH_RB_CNTL, ih_rb_cntl); + + /* set rptr, wptr to 0 */ + WREG32(IH_RB_RPTR, 0); + WREG32(IH_RB_WPTR, 0); + + /* Default settings for IH_CNTL (disabled at first) */ + ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); + /* RPTR_REARM only works if msi's are enabled */ + if (rdev->msi_enabled) + ih_cntl |= RPTR_REARM; + WREG32(IH_CNTL, ih_cntl); + + /* force the active interrupt state to all disabled */ + cik_disable_interrupt_state(rdev); + + pci_set_master(rdev->pdev); + + /* enable irqs */ + cik_enable_interrupts(rdev); + + return ret; +} + +/** + * cik_irq_set - enable/disable interrupt sources + * + * @rdev: radeon_device pointer + * + * Enable interrupt sources on the GPU (vblanks, hpd, + * etc.) (CIK). + * Returns 0 for success, errors for failure. + */ +int cik_irq_set(struct radeon_device *rdev) +{ + u32 cp_int_cntl; + u32 cp_m1p0; + u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; + u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; + u32 grbm_int_cntl = 0; + u32 dma_cntl, dma_cntl1; + + if (!rdev->irq.installed) { + WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); + return -EINVAL; + } + /* don't enable anything if the ih is disabled */ + if (!rdev->ih.enabled) { + cik_disable_interrupts(rdev); + /* force the active interrupt state to all disabled */ + cik_disable_interrupt_state(rdev); + return 0; + } + + cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & + (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE; + + hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + + dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; + dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; + + cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; + + /* enable CP interrupts on all rings */ + if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { + DRM_DEBUG("cik_irq_set: sw int gfx\n"); + cp_int_cntl |= TIME_STAMP_INT_ENABLE; + } + if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { + struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; + DRM_DEBUG("si_irq_set: sw int cp1\n"); + if (ring->me == 1) { + switch (ring->pipe) { + case 0: + cp_m1p0 |= TIME_STAMP_INT_ENABLE; + break; + default: + DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); + break; + } + } else { + DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me); + } + } + if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { + struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; + DRM_DEBUG("si_irq_set: sw int cp2\n"); + if (ring->me == 1) { + switch (ring->pipe) { + case 0: + cp_m1p0 |= TIME_STAMP_INT_ENABLE; + break; + default: + DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); + break; + } + } else { + DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me); + } + } + + if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { + DRM_DEBUG("cik_irq_set: sw int dma\n"); + dma_cntl |= TRAP_ENABLE; + } + + if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { + DRM_DEBUG("cik_irq_set: sw int dma1\n"); + dma_cntl1 |= TRAP_ENABLE; + } + + if (rdev->irq.crtc_vblank_int[0] || + atomic_read(&rdev->irq.pflip[0])) { + DRM_DEBUG("cik_irq_set: vblank 0\n"); + crtc1 |= VBLANK_INTERRUPT_MASK; + } + if (rdev->irq.crtc_vblank_int[1] || + atomic_read(&rdev->irq.pflip[1])) { + DRM_DEBUG("cik_irq_set: vblank 1\n"); + crtc2 |= VBLANK_INTERRUPT_MASK; + } + if (rdev->irq.crtc_vblank_int[2] || + atomic_read(&rdev->irq.pflip[2])) { + DRM_DEBUG("cik_irq_set: vblank 2\n"); + crtc3 |= VBLANK_INTERRUPT_MASK; + } + if (rdev->irq.crtc_vblank_int[3] || + atomic_read(&rdev->irq.pflip[3])) { + DRM_DEBUG("cik_irq_set: vblank 3\n"); + crtc4 |= VBLANK_INTERRUPT_MASK; + } + if (rdev->irq.crtc_vblank_int[4] || + atomic_read(&rdev->irq.pflip[4])) { + DRM_DEBUG("cik_irq_set: vblank 4\n"); + crtc5 |= VBLANK_INTERRUPT_MASK; + } + if (rdev->irq.crtc_vblank_int[5] || + atomic_read(&rdev->irq.pflip[5])) { + DRM_DEBUG("cik_irq_set: vblank 5\n"); + crtc6 |= VBLANK_INTERRUPT_MASK; + } + if (rdev->irq.hpd[0]) { + DRM_DEBUG("cik_irq_set: hpd 1\n"); + hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; + } + if (rdev->irq.hpd[1]) { + DRM_DEBUG("cik_irq_set: hpd 2\n"); + hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; + } + if (rdev->irq.hpd[2]) { + DRM_DEBUG("cik_irq_set: hpd 3\n"); + hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; + } + if (rdev->irq.hpd[3]) { + DRM_DEBUG("cik_irq_set: hpd 4\n"); + hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; + } + if (rdev->irq.hpd[4]) { + DRM_DEBUG("cik_irq_set: hpd 5\n"); + hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; + } + if (rdev->irq.hpd[5]) { + DRM_DEBUG("cik_irq_set: hpd 6\n"); + hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; + } + + WREG32(CP_INT_CNTL_RING0, cp_int_cntl); + + WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl); + WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); + + WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); + + WREG32(GRBM_INT_CNTL, grbm_int_cntl); + + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); + if (rdev->num_crtc >= 4) { + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); + } + if (rdev->num_crtc >= 6) { + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); + WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); + } + + if (rdev->num_crtc >= 2) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + } + if (rdev->num_crtc >= 4) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + } + if (rdev->num_crtc >= 6) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + } + + WREG32(DC_HPD1_INT_CONTROL, hpd1); + WREG32(DC_HPD2_INT_CONTROL, hpd2); + WREG32(DC_HPD3_INT_CONTROL, hpd3); + WREG32(DC_HPD4_INT_CONTROL, hpd4); + WREG32(DC_HPD5_INT_CONTROL, hpd5); + WREG32(DC_HPD6_INT_CONTROL, hpd6); + + /* posting read */ + RREG32(SRBM_STATUS); + + return 0; +} + +/** + * cik_irq_ack - ack interrupt sources + * + * @rdev: radeon_device pointer + * + * Ack interrupt sources on the GPU (vblanks, hpd, + * etc.) (CIK). Certain interrupts sources are sw + * generated and do not require an explicit ack. + */ +static inline void cik_irq_ack(struct radeon_device *rdev) +{ + u32 tmp; + + rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); + rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); + rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); + rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); + rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); + rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); + rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); + + rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS + + EVERGREEN_CRTC0_REGISTER_OFFSET); + rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS + + EVERGREEN_CRTC1_REGISTER_OFFSET); + if (rdev->num_crtc >= 4) { + rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS + + EVERGREEN_CRTC2_REGISTER_OFFSET); + rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS + + EVERGREEN_CRTC3_REGISTER_OFFSET); + } + if (rdev->num_crtc >= 6) { + rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS + + EVERGREEN_CRTC4_REGISTER_OFFSET); + rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS + + EVERGREEN_CRTC5_REGISTER_OFFSET); + } + + if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED) + WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, + GRPH_PFLIP_INT_CLEAR); + if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED) + WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, + GRPH_PFLIP_INT_CLEAR); + if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) + WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); + if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) + WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); + if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) + WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); + if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) + WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); + + if (rdev->num_crtc >= 4) { + if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED) + WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, + GRPH_PFLIP_INT_CLEAR); + if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED) + WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, + GRPH_PFLIP_INT_CLEAR); + if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) + WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); + if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) + WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); + if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) + WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); + if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) + WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); + } + + if (rdev->num_crtc >= 6) { + if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED) + WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, + GRPH_PFLIP_INT_CLEAR); + if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED) + WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, + GRPH_PFLIP_INT_CLEAR); + if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) + WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); + if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) + WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); + if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) + WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); + if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) + WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); + } + + if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { + tmp = RREG32(DC_HPD1_INT_CONTROL); + tmp |= DC_HPDx_INT_ACK; + WREG32(DC_HPD1_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { + tmp = RREG32(DC_HPD2_INT_CONTROL); + tmp |= DC_HPDx_INT_ACK; + WREG32(DC_HPD2_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { + tmp = RREG32(DC_HPD3_INT_CONTROL); + tmp |= DC_HPDx_INT_ACK; + WREG32(DC_HPD3_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { + tmp = RREG32(DC_HPD4_INT_CONTROL); + tmp |= DC_HPDx_INT_ACK; + WREG32(DC_HPD4_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { + tmp = RREG32(DC_HPD5_INT_CONTROL); + tmp |= DC_HPDx_INT_ACK; + WREG32(DC_HPD5_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { + tmp = RREG32(DC_HPD6_INT_CONTROL); + tmp |= DC_HPDx_INT_ACK; + WREG32(DC_HPD6_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) { + tmp = RREG32(DC_HPD1_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD1_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) { + tmp = RREG32(DC_HPD2_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD2_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) { + tmp = RREG32(DC_HPD3_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD3_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) { + tmp = RREG32(DC_HPD4_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD4_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) { + tmp = RREG32(DC_HPD5_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD5_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) { + tmp = RREG32(DC_HPD6_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD6_INT_CONTROL, tmp); + } +} + +/** + * cik_irq_disable - disable interrupts + * + * @rdev: radeon_device pointer + * + * Disable interrupts on the hw (CIK). + */ +static void cik_irq_disable(struct radeon_device *rdev) +{ + cik_disable_interrupts(rdev); + /* Wait and acknowledge irq */ + mdelay(1); + cik_irq_ack(rdev); + cik_disable_interrupt_state(rdev); +} + +/** + * cik_irq_disable - disable interrupts for suspend + * + * @rdev: radeon_device pointer + * + * Disable interrupts and stop the RLC (CIK). + * Used for suspend. + */ +static void cik_irq_suspend(struct radeon_device *rdev) +{ + cik_irq_disable(rdev); + cik_rlc_stop(rdev); +} + +/** + * cik_irq_fini - tear down interrupt support + * + * @rdev: radeon_device pointer + * + * Disable interrupts on the hw and free the IH ring + * buffer (CIK). + * Used for driver unload. + */ +static void cik_irq_fini(struct radeon_device *rdev) +{ + cik_irq_suspend(rdev); + r600_ih_ring_fini(rdev); +} + +/** + * cik_get_ih_wptr - get the IH ring buffer wptr + * + * @rdev: radeon_device pointer + * + * Get the IH ring buffer wptr from either the register + * or the writeback memory buffer (CIK). Also check for + * ring buffer overflow and deal with it. + * Used by cik_irq_process(). + * Returns the value of the wptr. + */ +static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) +{ + u32 wptr, tmp; + + if (rdev->wb.enabled) + wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); + else + wptr = RREG32(IH_RB_WPTR); + + if (wptr & RB_OVERFLOW) { + wptr &= ~RB_OVERFLOW; + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 16). Hopefully + * this should allow us to catchup. + */ + dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); + rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; + tmp = RREG32(IH_RB_CNTL); + tmp |= IH_WPTR_OVERFLOW_CLEAR; + WREG32(IH_RB_CNTL, tmp); + } + return (wptr & rdev->ih.ptr_mask); +} + +/* CIK IV Ring + * Each IV ring entry is 128 bits: + * [7:0] - interrupt source id + * [31:8] - reserved + * [59:32] - interrupt source data + * [63:60] - reserved + * [71:64] - RINGID + * CP: + * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] + * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher + * - for gfx, hw shader state (0=PS...5=LS, 6=CS) + * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes + * PIPE_ID - ME0 0=3D + * - ME1&2 compute dispatcher (4 pipes each) + * SDMA: + * INSTANCE_ID [1:0], QUEUE_ID[1:0] + * INSTANCE_ID - 0 = sdma0, 1 = sdma1 + * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 + * [79:72] - VMID + * [95:80] - PASID + * [127:96] - reserved + */ +/** + * cik_irq_process - interrupt handler + * + * @rdev: radeon_device pointer + * + * Interrupt hander (CIK). Walk the IH ring, + * ack interrupts and schedule work to handle + * interrupt events. + * Returns irq process return code. + */ +int cik_irq_process(struct radeon_device *rdev) +{ + struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; + struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; + u32 wptr; + u32 rptr; + u32 src_id, src_data, ring_id; + u8 me_id, pipe_id, queue_id; + u32 ring_index; + bool queue_hotplug = false; + bool queue_dp = false; + bool queue_reset = false; + u32 addr, status, mc_client; + bool queue_thermal = false; + + if (!rdev->ih.enabled || rdev->shutdown) + return IRQ_NONE; + + wptr = cik_get_ih_wptr(rdev); + +restart_ih: + /* is somebody else already processing irqs? */ + if (atomic_xchg(&rdev->ih.lock, 1)) + return IRQ_NONE; + + rptr = rdev->ih.rptr; + DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr); + + /* Order reading of wptr vs. reading of IH ring data */ + rmb(); + + /* display interrupts */ + cik_irq_ack(rdev); + + while (rptr != wptr) { + /* wptr/rptr are in bytes! */ + ring_index = rptr / 4; + +#ifdef notyet + radeon_kfd_interrupt(rdev, + (const void *) &rdev->ih.ring[ring_index]); +#endif + + src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; + src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; + ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; + + switch (src_id) { + case 1: /* D1 vblank/vline */ + switch (src_data) { + case 0: /* D1 vblank */ + if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[0]) { + drm_handle_vblank(rdev->ddev, 0); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); + } + if (atomic_read(&rdev->irq.pflip[0])) + radeon_crtc_handle_vblank(rdev, 0); + rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D1 vblank\n"); + + break; + case 1: /* D1 vline */ + if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; + DRM_DEBUG("IH: D1 vline\n"); + + break; + default: + DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); + break; + } + break; + case 2: /* D2 vblank/vline */ + switch (src_data) { + case 0: /* D2 vblank */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[1]) { + drm_handle_vblank(rdev->ddev, 1); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); + } + if (atomic_read(&rdev->irq.pflip[1])) + radeon_crtc_handle_vblank(rdev, 1); + rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D2 vblank\n"); + + break; + case 1: /* D2 vline */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; + DRM_DEBUG("IH: D2 vline\n"); + + break; + default: + DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); + break; + } + break; + case 3: /* D3 vblank/vline */ + switch (src_data) { + case 0: /* D3 vblank */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[2]) { + drm_handle_vblank(rdev->ddev, 2); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); + } + if (atomic_read(&rdev->irq.pflip[2])) + radeon_crtc_handle_vblank(rdev, 2); + rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D3 vblank\n"); + + break; + case 1: /* D3 vline */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; + DRM_DEBUG("IH: D3 vline\n"); + + break; + default: + DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); + break; + } + break; + case 4: /* D4 vblank/vline */ + switch (src_data) { + case 0: /* D4 vblank */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[3]) { + drm_handle_vblank(rdev->ddev, 3); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); + } + if (atomic_read(&rdev->irq.pflip[3])) + radeon_crtc_handle_vblank(rdev, 3); + rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D4 vblank\n"); + + break; + case 1: /* D4 vline */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; + DRM_DEBUG("IH: D4 vline\n"); + + break; + default: + DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); + break; + } + break; + case 5: /* D5 vblank/vline */ + switch (src_data) { + case 0: /* D5 vblank */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[4]) { + drm_handle_vblank(rdev->ddev, 4); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); + } + if (atomic_read(&rdev->irq.pflip[4])) + radeon_crtc_handle_vblank(rdev, 4); + rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D5 vblank\n"); + + break; + case 1: /* D5 vline */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; + DRM_DEBUG("IH: D5 vline\n"); + + break; + default: + DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); + break; + } + break; + case 6: /* D6 vblank/vline */ + switch (src_data) { + case 0: /* D6 vblank */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[5]) { + drm_handle_vblank(rdev->ddev, 5); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); + } + if (atomic_read(&rdev->irq.pflip[5])) + radeon_crtc_handle_vblank(rdev, 5); + rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D6 vblank\n"); + + break; + case 1: /* D6 vline */ + if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; + DRM_DEBUG("IH: D6 vline\n"); + + break; + default: + DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); + break; + } + break; + case 8: /* D1 page flip */ + case 10: /* D2 page flip */ + case 12: /* D3 page flip */ + case 14: /* D4 page flip */ + case 16: /* D5 page flip */ + case 18: /* D6 page flip */ + DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); + if (radeon_use_pflipirq > 0) + radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); + break; + case 42: /* HPD hotplug */ + switch (src_data) { + case 0: + if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD1\n"); + + break; + case 1: + if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD2\n"); + + break; + case 2: + if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD3\n"); + + break; + case 3: + if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD4\n"); + + break; + case 4: + if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD5\n"); + + break; + case 5: + if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD6\n"); + + break; + case 6: + if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 1\n"); + + break; + case 7: + if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 2\n"); + + break; + case 8: + if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 3\n"); + + break; + case 9: + if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 4\n"); + + break; + case 10: + if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 5\n"); + + break; + case 11: + if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 6\n"); + + break; + default: + DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); + break; + } + break; + case 96: + DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); + WREG32(SRBM_INT_ACK, 0x1); + break; + case 124: /* UVD */ + DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); + radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); + break; + case 146: + case 147: + addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); + status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); + mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); + /* reset addr and status */ + WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); + if (addr == 0x0 && status == 0x0) + break; + dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); + dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", + addr); + dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", + status); + cik_vm_decode_fault(rdev, status, addr, mc_client); + break; + case 167: /* VCE */ + DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data); + switch (src_data) { + case 0: + radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); + break; + case 1: + radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); + break; + default: + DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); + break; + } + break; + case 176: /* GFX RB CP_INT */ + case 177: /* GFX IB CP_INT */ + radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); + break; + case 181: /* CP EOP event */ + DRM_DEBUG("IH: CP EOP\n"); + /* XXX check the bitfield order! */ + me_id = (ring_id & 0x60) >> 5; + pipe_id = (ring_id & 0x18) >> 3; + queue_id = (ring_id & 0x7) >> 0; + switch (me_id) { + case 0: + radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); + break; + case 1: + case 2: + if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id)) + radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); + if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id)) + radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); + break; + } + break; + case 184: /* CP Privileged reg access */ + DRM_ERROR("Illegal register access in command stream\n"); + /* XXX check the bitfield order! */ + me_id = (ring_id & 0x60) >> 5; + pipe_id = (ring_id & 0x18) >> 3; + queue_id = (ring_id & 0x7) >> 0; + switch (me_id) { + case 0: + /* This results in a full GPU reset, but all we need to do is soft + * reset the CP for gfx + */ + queue_reset = true; + break; + case 1: + /* XXX compute */ + queue_reset = true; + break; + case 2: + /* XXX compute */ + queue_reset = true; + break; + } + break; + case 185: /* CP Privileged inst */ + DRM_ERROR("Illegal instruction in command stream\n"); + /* XXX check the bitfield order! */ + me_id = (ring_id & 0x60) >> 5; + pipe_id = (ring_id & 0x18) >> 3; + queue_id = (ring_id & 0x7) >> 0; + switch (me_id) { + case 0: + /* This results in a full GPU reset, but all we need to do is soft + * reset the CP for gfx + */ + queue_reset = true; + break; + case 1: + /* XXX compute */ + queue_reset = true; + break; + case 2: + /* XXX compute */ + queue_reset = true; + break; + } + break; + case 224: /* SDMA trap event */ + /* XXX check the bitfield order! */ + me_id = (ring_id & 0x3) >> 0; + queue_id = (ring_id & 0xc) >> 2; + DRM_DEBUG("IH: SDMA trap\n"); + switch (me_id) { + case 0: + switch (queue_id) { + case 0: + radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); + break; + case 1: + /* XXX compute */ + break; + case 2: + /* XXX compute */ + break; + } + break; + case 1: + switch (queue_id) { + case 0: + radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); + break; + case 1: + /* XXX compute */ + break; + case 2: + /* XXX compute */ + break; + } + break; + } + break; + case 230: /* thermal low to high */ + DRM_DEBUG("IH: thermal low to high\n"); + rdev->pm.dpm.thermal.high_to_low = false; + queue_thermal = true; + break; + case 231: /* thermal high to low */ + DRM_DEBUG("IH: thermal high to low\n"); + rdev->pm.dpm.thermal.high_to_low = true; + queue_thermal = true; + break; + case 233: /* GUI IDLE */ + DRM_DEBUG("IH: GUI idle\n"); + break; + case 241: /* SDMA Privileged inst */ + case 247: /* SDMA Privileged inst */ + DRM_ERROR("Illegal instruction in SDMA command stream\n"); + /* XXX check the bitfield order! */ + me_id = (ring_id & 0x3) >> 0; + queue_id = (ring_id & 0xc) >> 2; + switch (me_id) { + case 0: + switch (queue_id) { + case 0: + queue_reset = true; + break; + case 1: + /* XXX compute */ + queue_reset = true; + break; + case 2: + /* XXX compute */ + queue_reset = true; + break; + } + break; + case 1: + switch (queue_id) { + case 0: + queue_reset = true; + break; + case 1: + /* XXX compute */ + queue_reset = true; + break; + case 2: + /* XXX compute */ + queue_reset = true; + break; + } + break; + } + break; + default: + DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); + break; + } + + /* wptr/rptr are in bytes! */ + rptr += 16; + rptr &= rdev->ih.ptr_mask; + WREG32(IH_RB_RPTR, rptr); + } + if (queue_dp) + schedule_work(&rdev->dp_work); + if (queue_hotplug) + schedule_delayed_work(&rdev->hotplug_work, 0); + if (queue_reset) { + rdev->needs_reset = true; + wake_up_all(&rdev->fence_queue); + } + if (queue_thermal) + schedule_work(&rdev->pm.dpm.thermal.work); + rdev->ih.rptr = rptr; + atomic_set(&rdev->ih.lock, 0); + + /* make sure wptr hasn't changed while processing */ + wptr = cik_get_ih_wptr(rdev); + if (wptr != rptr) + goto restart_ih; + + return IRQ_HANDLED; +} + +/* + * startup/shutdown callbacks + */ +/** + * cik_startup - program the asic to a functional state + * + * @rdev: radeon_device pointer + * + * Programs the asic to a functional state (CIK). + * Called by cik_init() and cik_resume(). + * Returns 0 for success, error for failure. + */ +static int cik_startup(struct radeon_device *rdev) +{ + struct radeon_ring *ring; + u32 nop; + int r; + + /* enable pcie gen2/3 link */ + cik_pcie_gen3_enable(rdev); + /* enable aspm */ + cik_program_aspm(rdev); + + /* scratch needs to be initialized before MC */ + r = r600_vram_scratch_init(rdev); + if (r) + return r; + + cik_mc_program(rdev); + + if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { + r = ci_mc_load_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load MC firmware!\n"); + return r; + } + } + + r = cik_pcie_gart_enable(rdev); + if (r) + return r; + cik_gpu_init(rdev); + + /* allocate rlc buffers */ + if (rdev->flags & RADEON_IS_IGP) { + if (rdev->family == CHIP_KAVERI) { + rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; + rdev->rlc.reg_list_size = + (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list); + } else { + rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; + rdev->rlc.reg_list_size = + (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list); + } + } + rdev->rlc.cs_data = ci_cs_data; + rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4; + r = sumo_rlc_init(rdev); + if (r) { + DRM_ERROR("Failed to init rlc BOs!\n"); + return r; + } + + /* allocate wb buffer */ + r = radeon_wb_init(rdev); + if (r) + return r; + + /* allocate mec buffers */ + r = cik_mec_init(rdev); + if (r) { + DRM_ERROR("Failed to init MEC BOs!\n"); + return r; + } + + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + + r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + + r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + + r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); + return r; + } + + r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); + return r; + } + + r = radeon_uvd_resume(rdev); + if (!r) { + r = uvd_v4_2_resume(rdev); + if (!r) { + r = radeon_fence_driver_start_ring(rdev, + R600_RING_TYPE_UVD_INDEX); + if (r) + dev_err(rdev->dev, "UVD fences init error (%d).\n", r); + } + } + if (r) + rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; + + r = radeon_vce_resume(rdev); + if (!r) { + r = vce_v2_0_resume(rdev); + if (!r) + r = radeon_fence_driver_start_ring(rdev, + TN_RING_TYPE_VCE1_INDEX); + if (!r) + r = radeon_fence_driver_start_ring(rdev, + TN_RING_TYPE_VCE2_INDEX); + } + if (r) { + dev_err(rdev->dev, "VCE init error (%d).\n", r); + rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; + rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; + } + + /* Enable IRQ */ + if (!rdev->irq.installed) { + r = radeon_irq_kms_init(rdev); + if (r) + return r; + } + + r = cik_irq_init(rdev); + if (r) { + DRM_ERROR("radeon: IH init failed (%d).\n", r); + radeon_irq_kms_fini(rdev); + return r; + } + cik_irq_set(rdev); + + if (rdev->family == CHIP_HAWAII) { + if (rdev->new_fw) + nop = PACKET3(PACKET3_NOP, 0x3FFF); + else + nop = RADEON_CP_PACKET2; + } else { + nop = PACKET3(PACKET3_NOP, 0x3FFF); + } + + ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; + r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, + nop); + if (r) + return r; + + /* set up the compute queues */ + /* type-2 packets are deprecated on MEC, use type-3 instead */ + ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; + r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, + nop); + if (r) + return r; + ring->me = 1; /* first MEC */ + ring->pipe = 0; /* first pipe */ + ring->queue = 0; /* first queue */ + ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET; + + /* type-2 packets are deprecated on MEC, use type-3 instead */ + ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; + r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, + nop); + if (r) + return r; + /* dGPU only have 1 MEC */ + ring->me = 1; /* first MEC */ + ring->pipe = 0; /* first pipe */ + ring->queue = 1; /* second queue */ + ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET; + + ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; + r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, + SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); + if (r) + return r; + + ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; + r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, + SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); + if (r) + return r; + + r = cik_cp_resume(rdev); + if (r) + return r; + + r = cik_sdma_resume(rdev); + if (r) + return r; + + ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + if (ring->ring_size) { + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + RADEON_CP_PACKET2); + if (!r) + r = uvd_v1_0_init(rdev); + if (r) + DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); + } + + r = -ENOENT; + + ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; + if (ring->ring_size) + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + VCE_CMD_NO_OP); + + ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; + if (ring->ring_size) + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + VCE_CMD_NO_OP); + + if (!r) + r = vce_v1_0_init(rdev); + else if (r != -ENOENT) + DRM_ERROR("radeon: failed initializing VCE (%d).\n", r); + + r = radeon_ib_pool_init(rdev); + if (r) { + dev_err(rdev->dev, "IB initialization failed (%d).\n", r); + return r; + } + + r = radeon_vm_manager_init(rdev); + if (r) { + dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); + return r; + } + + r = radeon_audio_init(rdev); + if (r) + return r; + +#ifdef notyet + r = radeon_kfd_resume(rdev); + if (r) + return r; +#endif + + return 0; +} + +/** + * cik_resume - resume the asic to a functional state + * + * @rdev: radeon_device pointer + * + * Programs the asic to a functional state (CIK). + * Called at resume. + * Returns 0 for success, error for failure. + */ +int cik_resume(struct radeon_device *rdev) +{ + int r; + + /* post card */ + atom_asic_init(rdev->mode_info.atom_context); + + /* init golden registers */ + cik_init_golden_registers(rdev); + + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_resume(rdev); + + rdev->accel_working = true; + r = cik_startup(rdev); + if (r) { + DRM_ERROR("cik startup failed on resume\n"); + rdev->accel_working = false; + return r; + } + + return r; + +} + +/** + * cik_suspend - suspend the asic + * + * @rdev: radeon_device pointer + * + * Bring the chip into a state suitable for suspend (CIK). + * Called at suspend. + * Returns 0 for success. + */ +int cik_suspend(struct radeon_device *rdev) +{ +#ifdef notyet + radeon_kfd_suspend(rdev); +#endif + radeon_pm_suspend(rdev); + radeon_audio_fini(rdev); + radeon_vm_manager_fini(rdev); + cik_cp_enable(rdev, false); + cik_sdma_enable(rdev, false); + uvd_v1_0_fini(rdev); + radeon_uvd_suspend(rdev); + radeon_vce_suspend(rdev); + cik_fini_pg(rdev); + cik_fini_cg(rdev); + cik_irq_suspend(rdev); + radeon_wb_disable(rdev); + cik_pcie_gart_disable(rdev); + return 0; +} + +/* Plan is to move initialization in that function and use + * helper function so that radeon_device_init pretty much + * do nothing more than calling asic specific function. This + * should also allow to remove a bunch of callback function + * like vram_info. + */ +/** + * cik_init - asic specific driver and hw init + * + * @rdev: radeon_device pointer + * + * Setup asic specific driver variables and program the hw + * to a functional state (CIK). + * Called at driver startup. + * Returns 0 for success, errors for failure. + */ +int cik_init(struct radeon_device *rdev) +{ + struct radeon_ring *ring; + int r; + + /* Read BIOS */ + if (!radeon_get_bios(rdev)) { + if (ASIC_IS_AVIVO(rdev)) + return -EINVAL; + } + /* Must be an ATOMBIOS */ + if (!rdev->is_atom_bios) { + dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); + return -EINVAL; + } + r = radeon_atombios_init(rdev); + if (r) + return r; + + /* Post card if necessary */ + if (!radeon_card_posted(rdev)) { + if (!rdev->bios) { + dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); + return -EINVAL; + } + DRM_INFO("GPU not posted. posting now...\n"); + atom_asic_init(rdev->mode_info.atom_context); + } + /* init golden registers */ + cik_init_golden_registers(rdev); + /* Initialize scratch registers */ + cik_scratch_init(rdev); + /* Initialize surface registers */ + radeon_surface_init(rdev); + /* Initialize clocks */ + radeon_get_clock_info(rdev->ddev); + + /* Fence driver */ + r = radeon_fence_driver_init(rdev); + if (r) + return r; + + /* initialize memory controller */ + r = cik_mc_init(rdev); + if (r) + return r; + /* Memory manager */ + r = radeon_bo_init(rdev); + if (r) + return r; + + if (rdev->flags & RADEON_IS_IGP) { + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || + !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { + r = cik_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + } else { + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || + !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || + !rdev->mc_fw) { + r = cik_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + } + + /* Initialize power management */ + radeon_pm_init(rdev); + + ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 1024 * 1024); + + ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 1024 * 1024); + r = radeon_doorbell_get(rdev, &ring->doorbell_index); + if (r) + return r; + + ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 1024 * 1024); + r = radeon_doorbell_get(rdev, &ring->doorbell_index); + if (r) + return r; + + ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 256 * 1024); + + ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 256 * 1024); + + r = radeon_uvd_init(rdev); + if (!r) { + ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 4096); + } + + r = radeon_vce_init(rdev); + if (!r) { + ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 4096); + + ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 4096); + } + + rdev->ih.ring_obj = NULL; + r600_ih_ring_init(rdev, 64 * 1024); + + r = r600_pcie_gart_init(rdev); + if (r) + return r; + + rdev->accel_working = true; + r = cik_startup(rdev); + if (r) { + dev_err(rdev->dev, "disabling GPU acceleration\n"); + cik_cp_fini(rdev); + cik_sdma_fini(rdev); + cik_irq_fini(rdev); + sumo_rlc_fini(rdev); + cik_mec_fini(rdev); + radeon_wb_fini(rdev); + radeon_ib_pool_fini(rdev); + radeon_vm_manager_fini(rdev); + radeon_irq_kms_fini(rdev); + cik_pcie_gart_fini(rdev); + rdev->accel_working = false; + } + + /* Don't start up if the MC ucode is missing. + * The default clocks and voltages before the MC ucode + * is loaded are not suffient for advanced operations. + */ + if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { + DRM_ERROR("radeon: MC ucode required for NI+.\n"); + return -EINVAL; + } + + return 0; +} + +/** + * cik_fini - asic specific driver and hw fini + * + * @rdev: radeon_device pointer + * + * Tear down the asic specific driver variables and program the hw + * to an idle state (CIK). + * Called at driver unload. + */ +void cik_fini(struct radeon_device *rdev) +{ + radeon_pm_fini(rdev); + cik_cp_fini(rdev); + cik_sdma_fini(rdev); + cik_fini_pg(rdev); + cik_fini_cg(rdev); + cik_irq_fini(rdev); + sumo_rlc_fini(rdev); + cik_mec_fini(rdev); + radeon_wb_fini(rdev); + radeon_vm_manager_fini(rdev); + radeon_ib_pool_fini(rdev); + radeon_irq_kms_fini(rdev); + uvd_v1_0_fini(rdev); + radeon_uvd_fini(rdev); + radeon_vce_fini(rdev); + cik_pcie_gart_fini(rdev); + r600_vram_scratch_fini(rdev); + radeon_gem_fini(rdev); + radeon_fence_driver_fini(rdev); + radeon_bo_fini(rdev); + radeon_atombios_fini(rdev); + kfree(rdev->bios); + rdev->bios = NULL; +} + +void dce8_program_fmt(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + int bpc = 0; + u32 tmp = 0; + enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; + + if (connector) { + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + bpc = radeon_get_monitor_bpc(connector); + dither = radeon_connector->dither; + } + + /* LVDS/eDP FMT is set up by atom */ + if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) + return; + + /* not needed for analog */ + if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || + (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) + return; + + if (bpc == 0) + return; + + switch (bpc) { + case 6: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | + FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0)); + else + tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0)); + break; + case 8: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | + FMT_RGB_RANDOM_ENABLE | + FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1)); + else + tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1)); + break; + case 10: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | + FMT_RGB_RANDOM_ENABLE | + FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2)); + else + tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2)); + break; + default: + /* not needed */ + break; + } + + WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); +} + +/* display watermark setup */ +/** + * dce8_line_buffer_adjust - Set up the line buffer + * + * @rdev: radeon_device pointer + * @radeon_crtc: the selected display controller + * @mode: the current display mode on the selected display + * controller + * + * Setup up the line buffer allocation for + * the selected display controller (CIK). + * Returns the line buffer size in pixels. + */ +static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, + struct radeon_crtc *radeon_crtc, + struct drm_display_mode *mode) +{ + u32 tmp, buffer_alloc, i; + u32 pipe_offset = radeon_crtc->crtc_id * 0x20; + /* + * Line Buffer Setup + * There are 6 line buffers, one for each display controllers. + * There are 3 partitions per LB. Select the number of partitions + * to enable based on the display width. For display widths larger + * than 4096, you need use to use 2 display controllers and combine + * them using the stereo blender. + */ + if (radeon_crtc->base.enabled && mode) { + if (mode->crtc_hdisplay < 1920) { + tmp = 1; + buffer_alloc = 2; + } else if (mode->crtc_hdisplay < 2560) { + tmp = 2; + buffer_alloc = 2; + } else if (mode->crtc_hdisplay < 4096) { + tmp = 0; + buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; + } else { + DRM_DEBUG_KMS("Mode too big for LB!\n"); + tmp = 0; + buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; + } + } else { + tmp = 1; + buffer_alloc = 0; + } + + WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset, + LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0)); + + WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, + DMIF_BUFFERS_ALLOCATED(buffer_alloc)); + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & + DMIF_BUFFERS_ALLOCATED_COMPLETED) + break; + udelay(1); + } + + if (radeon_crtc->base.enabled && mode) { + switch (tmp) { + case 0: + default: + return 4096 * 2; + case 1: + return 1920 * 2; + case 2: + return 2560 * 2; + } + } + + /* controller not enabled, so no lb used */ + return 0; +} + +/** + * cik_get_number_of_dram_channels - get the number of dram channels + * + * @rdev: radeon_device pointer + * + * Look up the number of video ram channels (CIK). + * Used for display watermark bandwidth calculations + * Returns the number of dram channels + */ +static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) +{ + u32 tmp = RREG32(MC_SHARED_CHMAP); + + switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { + case 0: + default: + return 1; + case 1: + return 2; + case 2: + return 4; + case 3: + return 8; + case 4: + return 3; + case 5: + return 6; + case 6: + return 10; + case 7: + return 12; + case 8: + return 16; + } +} + +struct dce8_wm_params { + u32 dram_channels; /* number of dram channels */ + u32 yclk; /* bandwidth per dram data pin in kHz */ + u32 sclk; /* engine clock in kHz */ + u32 disp_clk; /* display clock in kHz */ + u32 src_width; /* viewport width */ + u32 active_time; /* active display time in ns */ + u32 blank_time; /* blank time in ns */ + bool interlaced; /* mode is interlaced */ + fixed20_12 vsc; /* vertical scale ratio */ + u32 num_heads; /* number of active crtcs */ + u32 bytes_per_pixel; /* bytes per pixel display + overlay */ + u32 lb_size; /* line buffer allocated to pipe */ + u32 vtaps; /* vertical scaler taps */ +}; + +/** + * dce8_dram_bandwidth - get the dram bandwidth + * + * @wm: watermark calculation data + * + * Calculate the raw dram bandwidth (CIK). + * Used for display watermark bandwidth calculations + * Returns the dram bandwidth in MBytes/s + */ +static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm) +{ + /* Calculate raw DRAM Bandwidth */ + fixed20_12 dram_efficiency; /* 0.7 */ + fixed20_12 yclk, dram_channels, bandwidth; + fixed20_12 a; + + a.full = dfixed_const(1000); + yclk.full = dfixed_const(wm->yclk); + yclk.full = dfixed_div(yclk, a); + dram_channels.full = dfixed_const(wm->dram_channels * 4); + a.full = dfixed_const(10); + dram_efficiency.full = dfixed_const(7); + dram_efficiency.full = dfixed_div(dram_efficiency, a); + bandwidth.full = dfixed_mul(dram_channels, yclk); + bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); + + return dfixed_trunc(bandwidth); +} + +/** + * dce8_dram_bandwidth_for_display - get the dram bandwidth for display + * + * @wm: watermark calculation data + * + * Calculate the dram bandwidth used for display (CIK). + * Used for display watermark bandwidth calculations + * Returns the dram bandwidth for display in MBytes/s + */ +static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm) +{ + /* Calculate DRAM Bandwidth and the part allocated to display. */ + fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ + fixed20_12 yclk, dram_channels, bandwidth; + fixed20_12 a; + + a.full = dfixed_const(1000); + yclk.full = dfixed_const(wm->yclk); + yclk.full = dfixed_div(yclk, a); + dram_channels.full = dfixed_const(wm->dram_channels * 4); + a.full = dfixed_const(10); + disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ + disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); + bandwidth.full = dfixed_mul(dram_channels, yclk); + bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); + + return dfixed_trunc(bandwidth); +} + +/** + * dce8_data_return_bandwidth - get the data return bandwidth + * + * @wm: watermark calculation data + * + * Calculate the data return bandwidth used for display (CIK). + * Used for display watermark bandwidth calculations + * Returns the data return bandwidth in MBytes/s + */ +static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm) +{ + /* Calculate the display Data return Bandwidth */ + fixed20_12 return_efficiency; /* 0.8 */ + fixed20_12 sclk, bandwidth; + fixed20_12 a; + + a.full = dfixed_const(1000); + sclk.full = dfixed_const(wm->sclk); + sclk.full = dfixed_div(sclk, a); + a.full = dfixed_const(10); + return_efficiency.full = dfixed_const(8); + return_efficiency.full = dfixed_div(return_efficiency, a); + a.full = dfixed_const(32); + bandwidth.full = dfixed_mul(a, sclk); + bandwidth.full = dfixed_mul(bandwidth, return_efficiency); + + return dfixed_trunc(bandwidth); +} + +/** + * dce8_dmif_request_bandwidth - get the dmif bandwidth + * + * @wm: watermark calculation data + * + * Calculate the dmif bandwidth used for display (CIK). + * Used for display watermark bandwidth calculations + * Returns the dmif bandwidth in MBytes/s + */ +static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm) +{ + /* Calculate the DMIF Request Bandwidth */ + fixed20_12 disp_clk_request_efficiency; /* 0.8 */ + fixed20_12 disp_clk, bandwidth; + fixed20_12 a, b; + + a.full = dfixed_const(1000); + disp_clk.full = dfixed_const(wm->disp_clk); + disp_clk.full = dfixed_div(disp_clk, a); + a.full = dfixed_const(32); + b.full = dfixed_mul(a, disp_clk); + + a.full = dfixed_const(10); + disp_clk_request_efficiency.full = dfixed_const(8); + disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); + + bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency); + + return dfixed_trunc(bandwidth); +} + +/** + * dce8_available_bandwidth - get the min available bandwidth + * + * @wm: watermark calculation data + * + * Calculate the min available bandwidth used for display (CIK). + * Used for display watermark bandwidth calculations + * Returns the min available bandwidth in MBytes/s + */ +static u32 dce8_available_bandwidth(struct dce8_wm_params *wm) +{ + /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ + u32 dram_bandwidth = dce8_dram_bandwidth(wm); + u32 data_return_bandwidth = dce8_data_return_bandwidth(wm); + u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm); + + return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); +} + +/** + * dce8_average_bandwidth - get the average available bandwidth + * + * @wm: watermark calculation data + * + * Calculate the average available bandwidth used for display (CIK). + * Used for display watermark bandwidth calculations + * Returns the average available bandwidth in MBytes/s + */ +static u32 dce8_average_bandwidth(struct dce8_wm_params *wm) +{ + /* Calculate the display mode Average Bandwidth + * DisplayMode should contain the source and destination dimensions, + * timing, etc. + */ + fixed20_12 bpp; + fixed20_12 line_time; + fixed20_12 src_width; + fixed20_12 bandwidth; + fixed20_12 a; + + a.full = dfixed_const(1000); + line_time.full = dfixed_const(wm->active_time + wm->blank_time); + line_time.full = dfixed_div(line_time, a); + bpp.full = dfixed_const(wm->bytes_per_pixel); + src_width.full = dfixed_const(wm->src_width); + bandwidth.full = dfixed_mul(src_width, bpp); + bandwidth.full = dfixed_mul(bandwidth, wm->vsc); + bandwidth.full = dfixed_div(bandwidth, line_time); + + return dfixed_trunc(bandwidth); +} + +/** + * dce8_latency_watermark - get the latency watermark + * + * @wm: watermark calculation data + * + * Calculate the latency watermark (CIK). + * Used for display watermark bandwidth calculations + * Returns the latency watermark in ns + */ +static u32 dce8_latency_watermark(struct dce8_wm_params *wm) +{ + /* First calculate the latency in ns */ + u32 mc_latency = 2000; /* 2000 ns. */ + u32 available_bandwidth = dce8_available_bandwidth(wm); + u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; + u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; + u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ + u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + + (wm->num_heads * cursor_line_pair_return_time); + u32 latency = mc_latency + other_heads_data_return_time + dc_latency; + u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; + u32 tmp, dmif_size = 12288; + fixed20_12 a, b, c; + + if (wm->num_heads == 0) + return 0; + + a.full = dfixed_const(2); + b.full = dfixed_const(1); + if ((wm->vsc.full > a.full) || + ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || + (wm->vtaps >= 5) || + ((wm->vsc.full >= a.full) && wm->interlaced)) + max_src_lines_per_dst_line = 4; + else + max_src_lines_per_dst_line = 2; + + a.full = dfixed_const(available_bandwidth); + b.full = dfixed_const(wm->num_heads); + a.full = dfixed_div(a, b); + + b.full = dfixed_const(mc_latency + 512); + c.full = dfixed_const(wm->disp_clk); + b.full = dfixed_div(b, c); + + c.full = dfixed_const(dmif_size); + b.full = dfixed_div(c, b); + + tmp = min(dfixed_trunc(a), dfixed_trunc(b)); + + b.full = dfixed_const(1000); + c.full = dfixed_const(wm->disp_clk); + b.full = dfixed_div(c, b); + c.full = dfixed_const(wm->bytes_per_pixel); + b.full = dfixed_mul(b, c); + + lb_fill_bw = min(tmp, dfixed_trunc(b)); + + a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); + b.full = dfixed_const(1000); + c.full = dfixed_const(lb_fill_bw); + b.full = dfixed_div(c, b); + a.full = dfixed_div(a, b); + line_fill_time = dfixed_trunc(a); + + if (line_fill_time < wm->active_time) + return latency; + else + return latency + (line_fill_time - wm->active_time); + +} + +/** + * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check + * average and available dram bandwidth + * + * @wm: watermark calculation data + * + * Check if the display average bandwidth fits in the display + * dram bandwidth (CIK). + * Used for display watermark bandwidth calculations + * Returns true if the display fits, false if not. + */ +static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm) +{ + if (dce8_average_bandwidth(wm) <= + (dce8_dram_bandwidth_for_display(wm) / wm->num_heads)) + return true; + else + return false; +} + +/** + * dce8_average_bandwidth_vs_available_bandwidth - check + * average and available bandwidth + * + * @wm: watermark calculation data + * + * Check if the display average bandwidth fits in the display + * available bandwidth (CIK). + * Used for display watermark bandwidth calculations + * Returns true if the display fits, false if not. + */ +static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm) +{ + if (dce8_average_bandwidth(wm) <= + (dce8_available_bandwidth(wm) / wm->num_heads)) + return true; + else + return false; +} + +/** + * dce8_check_latency_hiding - check latency hiding + * + * @wm: watermark calculation data + * + * Check latency hiding (CIK). + * Used for display watermark bandwidth calculations + * Returns true if the display fits, false if not. + */ +static bool dce8_check_latency_hiding(struct dce8_wm_params *wm) +{ + u32 lb_partitions = wm->lb_size / wm->src_width; + u32 line_time = wm->active_time + wm->blank_time; + u32 latency_tolerant_lines; + u32 latency_hiding; + fixed20_12 a; + + a.full = dfixed_const(1); + if (wm->vsc.full > a.full) + latency_tolerant_lines = 1; + else { + if (lb_partitions <= (wm->vtaps + 1)) + latency_tolerant_lines = 1; + else + latency_tolerant_lines = 2; + } + + latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); + + if (dce8_latency_watermark(wm) <= latency_hiding) + return true; + else + return false; +} + +/** + * dce8_program_watermarks - program display watermarks + * + * @rdev: radeon_device pointer + * @radeon_crtc: the selected display controller + * @lb_size: line buffer size + * @num_heads: number of display controllers in use + * + * Calculate and program the display watermarks for the + * selected display controller (CIK). + */ +static void dce8_program_watermarks(struct radeon_device *rdev, + struct radeon_crtc *radeon_crtc, + u32 lb_size, u32 num_heads) +{ + struct drm_display_mode *mode = &radeon_crtc->base.mode; + struct dce8_wm_params wm_low, wm_high; + u32 pixel_period; + u32 line_time = 0; + u32 latency_watermark_a = 0, latency_watermark_b = 0; + u32 tmp, wm_mask; + + if (radeon_crtc->base.enabled && num_heads && mode) { + pixel_period = 1000000 / (u32)mode->clock; + line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); + + /* watermark for high clocks */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && + rdev->pm.dpm_enabled) { + wm_high.yclk = + radeon_dpm_get_mclk(rdev, false) * 10; + wm_high.sclk = + radeon_dpm_get_sclk(rdev, false) * 10; + } else { + wm_high.yclk = rdev->pm.current_mclk * 10; + wm_high.sclk = rdev->pm.current_sclk * 10; + } + + wm_high.disp_clk = mode->clock; + wm_high.src_width = mode->crtc_hdisplay; + wm_high.active_time = mode->crtc_hdisplay * pixel_period; + wm_high.blank_time = line_time - wm_high.active_time; + wm_high.interlaced = false; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + wm_high.interlaced = true; + wm_high.vsc = radeon_crtc->vsc; + wm_high.vtaps = 1; + if (radeon_crtc->rmx_type != RMX_OFF) + wm_high.vtaps = 2; + wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ + wm_high.lb_size = lb_size; + wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); + wm_high.num_heads = num_heads; + + /* set for high clocks */ + latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535); + + /* possibly force display priority to high */ + /* should really do this at mode validation time... */ + if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || + !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) || + !dce8_check_latency_hiding(&wm_high) || + (rdev->disp_priority == 2)) { + DRM_DEBUG_KMS("force priority to high\n"); + } + + /* watermark for low clocks */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && + rdev->pm.dpm_enabled) { + wm_low.yclk = + radeon_dpm_get_mclk(rdev, true) * 10; + wm_low.sclk = + radeon_dpm_get_sclk(rdev, true) * 10; + } else { + wm_low.yclk = rdev->pm.current_mclk * 10; + wm_low.sclk = rdev->pm.current_sclk * 10; + } + + wm_low.disp_clk = mode->clock; + wm_low.src_width = mode->crtc_hdisplay; + wm_low.active_time = mode->crtc_hdisplay * pixel_period; + wm_low.blank_time = line_time - wm_low.active_time; + wm_low.interlaced = false; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + wm_low.interlaced = true; + wm_low.vsc = radeon_crtc->vsc; + wm_low.vtaps = 1; + if (radeon_crtc->rmx_type != RMX_OFF) + wm_low.vtaps = 2; + wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ + wm_low.lb_size = lb_size; + wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); + wm_low.num_heads = num_heads; + + /* set for low clocks */ + latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535); + + /* possibly force display priority to high */ + /* should really do this at mode validation time... */ + if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || + !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) || + !dce8_check_latency_hiding(&wm_low) || + (rdev->disp_priority == 2)) { + DRM_DEBUG_KMS("force priority to high\n"); + } + + /* Save number of lines the linebuffer leads before the scanout */ + radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay); + } + + /* select wm A */ + wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); + tmp = wm_mask; + tmp &= ~LATENCY_WATERMARK_MASK(3); + tmp |= LATENCY_WATERMARK_MASK(1); + WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); + WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, + (LATENCY_LOW_WATERMARK(latency_watermark_a) | + LATENCY_HIGH_WATERMARK(line_time))); + /* select wm B */ + tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); + tmp &= ~LATENCY_WATERMARK_MASK(3); + tmp |= LATENCY_WATERMARK_MASK(2); + WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); + WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, + (LATENCY_LOW_WATERMARK(latency_watermark_b) | + LATENCY_HIGH_WATERMARK(line_time))); + /* restore original selection */ + WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask); + + /* save values for DPM */ + radeon_crtc->line_time = line_time; + radeon_crtc->wm_high = latency_watermark_a; + radeon_crtc->wm_low = latency_watermark_b; +} + +/** + * dce8_bandwidth_update - program display watermarks + * + * @rdev: radeon_device pointer + * + * Calculate and program the display watermarks and line + * buffer allocation (CIK). + */ +void dce8_bandwidth_update(struct radeon_device *rdev) +{ + struct drm_display_mode *mode = NULL; + u32 num_heads = 0, lb_size; + int i; + + if (!rdev->mode_info.mode_config_initialized) + return; + + radeon_update_display_priority(rdev); + + for (i = 0; i < rdev->num_crtc; i++) { + if (rdev->mode_info.crtcs[i]->base.enabled) + num_heads++; + } + for (i = 0; i < rdev->num_crtc; i++) { + mode = &rdev->mode_info.crtcs[i]->base.mode; + lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); + dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); + } +} + +/** + * cik_get_gpu_clock_counter - return GPU clock counter snapshot + * + * @rdev: radeon_device pointer + * + * Fetches a GPU clock counter snapshot (SI). + * Returns the 64 bit clock counter snapshot. + */ +uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) +{ + uint64_t clock; + + mutex_lock(&rdev->gpu_clock_mutex); + WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); + clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | + ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); + mutex_unlock(&rdev->gpu_clock_mutex); + return clock; +} + +static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, + u32 cntl_reg, u32 status_reg) +{ + int r, i; + struct atom_clock_dividers dividers; + uint32_t tmp; + + r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, + clock, false, ÷rs); + if (r) + return r; + + tmp = RREG32_SMC(cntl_reg); + tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK); + tmp |= dividers.post_divider; + WREG32_SMC(cntl_reg, tmp); + + for (i = 0; i < 100; i++) { + if (RREG32_SMC(status_reg) & DCLK_STATUS) + break; + mdelay(10); + } + if (i == 100) + return -ETIMEDOUT; + + return 0; +} + +int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) +{ + int r = 0; + + r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); + if (r) + return r; + + r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); + return r; +} + +int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) +{ + int r, i; + struct atom_clock_dividers dividers; + u32 tmp; + + r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, + ecclk, false, ÷rs); + if (r) + return r; + + for (i = 0; i < 100; i++) { + if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) + break; + mdelay(10); + } + if (i == 100) + return -ETIMEDOUT; + + tmp = RREG32_SMC(CG_ECLK_CNTL); + tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK); + tmp |= dividers.post_divider; + WREG32_SMC(CG_ECLK_CNTL, tmp); + + for (i = 0; i < 100; i++) { + if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) + break; + mdelay(10); + } + if (i == 100) + return -ETIMEDOUT; + + return 0; +} + +static void cik_pcie_gen3_enable(struct radeon_device *rdev) +{ + struct pci_dev _root; + struct pci_dev *root; + int bridge_pos, gpu_pos; + u32 speed_cntl, mask, current_data_rate; + int ret, i; + u16 tmp16; + + root = &_root; + root->pc = rdev->pdev->pc; + root->tag = *rdev->ddev->bridgetag; + + if (pci_is_root_bus(rdev->pdev->bus)) + return; + + if (radeon_pcie_gen2 == 0) + return; + + if (rdev->flags & RADEON_IS_IGP) + return; + + if (!(rdev->flags & RADEON_IS_PCIE)) + return; + + ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); + if (ret != 0) + return; + + if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) + return; + + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> + LC_CURRENT_DATA_RATE_SHIFT; + if (mask & DRM_PCIE_SPEED_80) { + if (current_data_rate == 2) { + DRM_INFO("PCIE gen 3 link speeds already enabled\n"); + return; + } + DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); + } else if (mask & DRM_PCIE_SPEED_50) { + if (current_data_rate == 1) { + DRM_INFO("PCIE gen 2 link speeds already enabled\n"); + return; + } + DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); + } + + bridge_pos = pci_pcie_cap(root); + if (!bridge_pos) + return; + + gpu_pos = pci_pcie_cap(rdev->pdev); + if (!gpu_pos) + return; + + if (mask & DRM_PCIE_SPEED_80) { + /* re-try equalization if gen3 is not already enabled */ + if (current_data_rate != 2) { + u16 bridge_cfg, gpu_cfg; + u16 bridge_cfg2, gpu_cfg2; + u32 max_lw, current_lw, tmp; + + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); + + tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; + pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); + + tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; + pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); + + tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1); + max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; + current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; + + if (current_lw < max_lw) { + tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); + if (tmp & LC_RENEGOTIATION_SUPPORT) { + tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); + tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); + tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); + } + } + + for (i = 0; i < 10; i++) { + /* check status */ + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); + if (tmp16 & PCI_EXP_DEVSTA_TRPND) + break; + + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); + + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); + tmp |= LC_SET_QUIESCE; + WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); + tmp |= LC_REDO_EQ; + WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); + + mdelay(100); + + /* linkctl */ + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); + tmp16 &= ~PCI_EXP_LNKCTL_HAWD; + tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); + pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); + + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); + tmp16 &= ~PCI_EXP_LNKCTL_HAWD; + tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); + pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); + + /* linkctl2 */ + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); + tmp16 &= ~((1 << 4) | (7 << 9)); + tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); + pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); + + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); + tmp16 &= ~((1 << 4) | (7 << 9)); + tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); + pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); + tmp &= ~LC_SET_QUIESCE; + WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); + } + } + } + + /* set the link speed */ + speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; + speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); + + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); + tmp16 &= ~0xf; + if (mask & DRM_PCIE_SPEED_80) + tmp16 |= 3; /* gen3 */ + else if (mask & DRM_PCIE_SPEED_50) + tmp16 |= 2; /* gen2 */ + else + tmp16 |= 1; /* gen1 */ + pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); + + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); + + for (i = 0; i < rdev->usec_timeout; i++) { + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) + break; + udelay(1); + } +} + +static void cik_program_aspm(struct radeon_device *rdev) +{ + u32 data, orig; + bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; + bool disable_clkreq = false; + + if (radeon_aspm == 0) + return; + + /* XXX double check IGPs */ + if (rdev->flags & RADEON_IS_IGP) + return; + + if (!(rdev->flags & RADEON_IS_PCIE)) + return; + + orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); + data &= ~LC_XMIT_N_FTS_MASK; + data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); + + orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); + data |= LC_GO_TO_RECOVERY; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); + + orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL); + data |= P_IGNORE_EDB_ERR; + if (orig != data) + WREG32_PCIE_PORT(PCIE_P_CNTL, data); + + orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); + data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); + data |= LC_PMI_TO_L1_DIS; + if (!disable_l0s) + data |= LC_L0S_INACTIVITY(7); + + if (!disable_l1) { + data |= LC_L1_INACTIVITY(7); + data &= ~LC_PMI_TO_L1_DIS; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL, data); + + if (!disable_plloff_in_l1) { + bool clk_req_support; + + orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0); + data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); + if (orig != data) + WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data); + + orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1); + data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); + if (orig != data) + WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data); + + orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0); + data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); + if (orig != data) + WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data); + + orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1); + data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); + if (orig != data) + WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data); + + orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); + data &= ~LC_DYN_LANES_PWR_STATE_MASK; + data |= LC_DYN_LANES_PWR_STATE(3); + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); + + if (!disable_clkreq && + !pci_is_root_bus(rdev->pdev->bus)) { + u32 lnkcap; + struct pci_dev _root; + struct pci_dev *root; + + root = &_root; + root->pc = rdev->pdev->pc; + root->tag = *rdev->ddev->bridgetag; + + clk_req_support = false; + pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); + if (lnkcap & PCI_EXP_LNKCAP_CLKPM) + clk_req_support = true; + } else { + clk_req_support = false; + } + + if (clk_req_support) { + orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); + data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); + + orig = data = RREG32_SMC(THM_CLK_CNTL); + data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); + data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); + if (orig != data) + WREG32_SMC(THM_CLK_CNTL, data); + + orig = data = RREG32_SMC(MISC_CLK_CTRL); + data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); + data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); + if (orig != data) + WREG32_SMC(MISC_CLK_CTRL, data); + + orig = data = RREG32_SMC(CG_CLKPIN_CNTL); + data &= ~BCLK_AS_XCLK; + if (orig != data) + WREG32_SMC(CG_CLKPIN_CNTL, data); + + orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2); + data &= ~FORCE_BIF_REFCLK_EN; + if (orig != data) + WREG32_SMC(CG_CLKPIN_CNTL_2, data); + + orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL); + data &= ~MPLL_CLKOUT_SEL_MASK; + data |= MPLL_CLKOUT_SEL(4); + if (orig != data) + WREG32_SMC(MPLL_BYPASSCLK_SEL, data); + } + } + } else { + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL, data); + } + + orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); + data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; + if (orig != data) + WREG32_PCIE_PORT(PCIE_CNTL2, data); + + if (!disable_l0s) { + data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); + if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { + data = RREG32_PCIE_PORT(PCIE_LC_STATUS1); + if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { + orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); + data &= ~LC_L0S_INACTIVITY_MASK; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL, data); + } + } + } +} diff --git a/sys/dev/pci/drm/radeon/cik_blit_shaders.c b/sys/dev/pci/drm/radeon/cik_blit_shaders.c new file mode 100644 index 00000000000..2edcce04bdc --- /dev/null +++ b/sys/dev/pci/drm/radeon/cik_blit_shaders.c @@ -0,0 +1,244 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Alex Deucher + */ + +#include + +const u32 cik_default_state[] = +{ + 0xc0066900, + 0x00000000, + 0x00000060, /* DB_RENDER_CONTROL */ + 0x00000000, /* DB_COUNT_CONTROL */ + 0x00000000, /* DB_DEPTH_VIEW */ + 0x0000002a, /* DB_RENDER_OVERRIDE */ + 0x00000000, /* DB_RENDER_OVERRIDE2 */ + 0x00000000, /* DB_HTILE_DATA_BASE */ + + 0xc0046900, + 0x00000008, + 0x00000000, /* DB_DEPTH_BOUNDS_MIN */ + 0x00000000, /* DB_DEPTH_BOUNDS_MAX */ + 0x00000000, /* DB_STENCIL_CLEAR */ + 0x00000000, /* DB_DEPTH_CLEAR */ + + 0xc0036900, + 0x0000000f, + 0x00000000, /* DB_DEPTH_INFO */ + 0x00000000, /* DB_Z_INFO */ + 0x00000000, /* DB_STENCIL_INFO */ + + 0xc0016900, + 0x00000080, + 0x00000000, /* PA_SC_WINDOW_OFFSET */ + + 0xc00d6900, + 0x00000083, + 0x0000ffff, /* PA_SC_CLIPRECT_RULE */ + 0x00000000, /* PA_SC_CLIPRECT_0_TL */ + 0x20002000, /* PA_SC_CLIPRECT_0_BR */ + 0x00000000, + 0x20002000, + 0x00000000, + 0x20002000, + 0x00000000, + 0x20002000, + 0xaaaaaaaa, /* PA_SC_EDGERULE */ + 0x00000000, /* PA_SU_HARDWARE_SCREEN_OFFSET */ + 0x0000000f, /* CB_TARGET_MASK */ + 0x0000000f, /* CB_SHADER_MASK */ + + 0xc0226900, + 0x00000094, + 0x80000000, /* PA_SC_VPORT_SCISSOR_0_TL */ + 0x20002000, /* PA_SC_VPORT_SCISSOR_0_BR */ + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x80000000, + 0x20002000, + 0x00000000, /* PA_SC_VPORT_ZMIN_0 */ + 0x3f800000, /* PA_SC_VPORT_ZMAX_0 */ + + 0xc0046900, + 0x00000100, + 0xffffffff, /* VGT_MAX_VTX_INDX */ + 0x00000000, /* VGT_MIN_VTX_INDX */ + 0x00000000, /* VGT_INDX_OFFSET */ + 0x00000000, /* VGT_MULTI_PRIM_IB_RESET_INDX */ + + 0xc0046900, + 0x00000105, + 0x00000000, /* CB_BLEND_RED */ + 0x00000000, /* CB_BLEND_GREEN */ + 0x00000000, /* CB_BLEND_BLUE */ + 0x00000000, /* CB_BLEND_ALPHA */ + + 0xc0016900, + 0x000001e0, + 0x00000000, /* CB_BLEND0_CONTROL */ + + 0xc00c6900, + 0x00000200, + 0x00000000, /* DB_DEPTH_CONTROL */ + 0x00000000, /* DB_EQAA */ + 0x00cc0010, /* CB_COLOR_CONTROL */ + 0x00000210, /* DB_SHADER_CONTROL */ + 0x00010000, /* PA_CL_CLIP_CNTL */ + 0x00000004, /* PA_SU_SC_MODE_CNTL */ + 0x00000100, /* PA_CL_VTE_CNTL */ + 0x00000000, /* PA_CL_VS_OUT_CNTL */ + 0x00000000, /* PA_CL_NANINF_CNTL */ + 0x00000000, /* PA_SU_LINE_STIPPLE_CNTL */ + 0x00000000, /* PA_SU_LINE_STIPPLE_SCALE */ + 0x00000000, /* PA_SU_PRIM_FILTER_CNTL */ + + 0xc0116900, + 0x00000280, + 0x00000000, /* PA_SU_POINT_SIZE */ + 0x00000000, /* PA_SU_POINT_MINMAX */ + 0x00000008, /* PA_SU_LINE_CNTL */ + 0x00000000, /* PA_SC_LINE_STIPPLE */ + 0x00000000, /* VGT_OUTPUT_PATH_CNTL */ + 0x00000000, /* VGT_HOS_CNTL */ + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, /* VGT_GS_MODE */ + + 0xc0026900, + 0x00000292, + 0x00000000, /* PA_SC_MODE_CNTL_0 */ + 0x00000000, /* PA_SC_MODE_CNTL_1 */ + + 0xc0016900, + 0x000002a1, + 0x00000000, /* VGT_PRIMITIVEID_EN */ + + 0xc0016900, + 0x000002a5, + 0x00000000, /* VGT_MULTI_PRIM_IB_RESET_EN */ + + 0xc0026900, + 0x000002a8, + 0x00000000, /* VGT_INSTANCE_STEP_RATE_0 */ + 0x00000000, + + 0xc0026900, + 0x000002ad, + 0x00000000, /* VGT_REUSE_OFF */ + 0x00000000, + + 0xc0016900, + 0x000002d5, + 0x00000000, /* VGT_SHADER_STAGES_EN */ + + 0xc0016900, + 0x000002dc, + 0x0000aa00, /* DB_ALPHA_TO_MASK */ + + 0xc0066900, + 0x000002de, + 0x00000000, /* PA_SU_POLY_OFFSET_DB_FMT_CNTL */ + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + + 0xc0026900, + 0x000002e5, + 0x00000000, /* VGT_STRMOUT_CONFIG */ + 0x00000000, + + 0xc01b6900, + 0x000002f5, + 0x76543210, /* PA_SC_CENTROID_PRIORITY_0 */ + 0xfedcba98, /* PA_SC_CENTROID_PRIORITY_1 */ + 0x00000000, /* PA_SC_LINE_CNTL */ + 0x00000000, /* PA_SC_AA_CONFIG */ + 0x00000005, /* PA_SU_VTX_CNTL */ + 0x3f800000, /* PA_CL_GB_VERT_CLIP_ADJ */ + 0x3f800000, /* PA_CL_GB_VERT_DISC_ADJ */ + 0x3f800000, /* PA_CL_GB_HORZ_CLIP_ADJ */ + 0x3f800000, /* PA_CL_GB_HORZ_DISC_ADJ */ + 0x00000000, /* PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 */ + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0xffffffff, /* PA_SC_AA_MASK_X0Y0_X1Y0 */ + 0xffffffff, + + 0xc0026900, + 0x00000316, + 0x0000000e, /* VGT_VERTEX_REUSE_BLOCK_CNTL */ + 0x00000010, /* */ +}; + +const u32 cik_default_size = ARRAY_SIZE(cik_default_state); diff --git a/sys/dev/pci/drm/radeon/cik_blit_shaders.h b/sys/dev/pci/drm/radeon/cik_blit_shaders.h new file mode 100644 index 00000000000..dfe7314f9ff --- /dev/null +++ b/sys/dev/pci/drm/radeon/cik_blit_shaders.h @@ -0,0 +1,32 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef CIK_BLIT_SHADERS_H +#define CIK_BLIT_SHADERS_H + +extern const u32 cik_default_state[]; + +extern const u32 cik_default_size; + +#endif diff --git a/sys/dev/pci/drm/radeon/cik_reg.h b/sys/dev/pci/drm/radeon/cik_reg.h new file mode 100644 index 00000000000..4e883fdc59d --- /dev/null +++ b/sys/dev/pci/drm/radeon/cik_reg.h @@ -0,0 +1,243 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#ifndef __CIK_REG_H__ +#define __CIK_REG_H__ + +#define CIK_DIDT_IND_INDEX 0xca00 +#define CIK_DIDT_IND_DATA 0xca04 + +#define CIK_DC_GPIO_HPD_MASK 0x65b0 +#define CIK_DC_GPIO_HPD_A 0x65b4 +#define CIK_DC_GPIO_HPD_EN 0x65b8 +#define CIK_DC_GPIO_HPD_Y 0x65bc + +#define CIK_GRPH_CONTROL 0x6804 +# define CIK_GRPH_DEPTH(x) (((x) & 0x3) << 0) +# define CIK_GRPH_DEPTH_8BPP 0 +# define CIK_GRPH_DEPTH_16BPP 1 +# define CIK_GRPH_DEPTH_32BPP 2 +# define CIK_GRPH_NUM_BANKS(x) (((x) & 0x3) << 2) +# define CIK_ADDR_SURF_2_BANK 0 +# define CIK_ADDR_SURF_4_BANK 1 +# define CIK_ADDR_SURF_8_BANK 2 +# define CIK_ADDR_SURF_16_BANK 3 +# define CIK_GRPH_Z(x) (((x) & 0x3) << 4) +# define CIK_GRPH_BANK_WIDTH(x) (((x) & 0x3) << 6) +# define CIK_ADDR_SURF_BANK_WIDTH_1 0 +# define CIK_ADDR_SURF_BANK_WIDTH_2 1 +# define CIK_ADDR_SURF_BANK_WIDTH_4 2 +# define CIK_ADDR_SURF_BANK_WIDTH_8 3 +# define CIK_GRPH_FORMAT(x) (((x) & 0x7) << 8) +/* 8 BPP */ +# define CIK_GRPH_FORMAT_INDEXED 0 +/* 16 BPP */ +# define CIK_GRPH_FORMAT_ARGB1555 0 +# define CIK_GRPH_FORMAT_ARGB565 1 +# define CIK_GRPH_FORMAT_ARGB4444 2 +# define CIK_GRPH_FORMAT_AI88 3 +# define CIK_GRPH_FORMAT_MONO16 4 +# define CIK_GRPH_FORMAT_BGRA5551 5 +/* 32 BPP */ +# define CIK_GRPH_FORMAT_ARGB8888 0 +# define CIK_GRPH_FORMAT_ARGB2101010 1 +# define CIK_GRPH_FORMAT_32BPP_DIG 2 +# define CIK_GRPH_FORMAT_8B_ARGB2101010 3 +# define CIK_GRPH_FORMAT_BGRA1010102 4 +# define CIK_GRPH_FORMAT_8B_BGRA1010102 5 +# define CIK_GRPH_FORMAT_RGB111110 6 +# define CIK_GRPH_FORMAT_BGR101111 7 +# define CIK_GRPH_BANK_HEIGHT(x) (((x) & 0x3) << 11) +# define CIK_ADDR_SURF_BANK_HEIGHT_1 0 +# define CIK_ADDR_SURF_BANK_HEIGHT_2 1 +# define CIK_ADDR_SURF_BANK_HEIGHT_4 2 +# define CIK_ADDR_SURF_BANK_HEIGHT_8 3 +# define CIK_GRPH_TILE_SPLIT(x) (((x) & 0x7) << 13) +# define CIK_ADDR_SURF_TILE_SPLIT_64B 0 +# define CIK_ADDR_SURF_TILE_SPLIT_128B 1 +# define CIK_ADDR_SURF_TILE_SPLIT_256B 2 +# define CIK_ADDR_SURF_TILE_SPLIT_512B 3 +# define CIK_ADDR_SURF_TILE_SPLIT_1KB 4 +# define CIK_ADDR_SURF_TILE_SPLIT_2KB 5 +# define CIK_ADDR_SURF_TILE_SPLIT_4KB 6 +# define CIK_GRPH_MACRO_TILE_ASPECT(x) (((x) & 0x3) << 18) +# define CIK_ADDR_SURF_MACRO_TILE_ASPECT_1 0 +# define CIK_ADDR_SURF_MACRO_TILE_ASPECT_2 1 +# define CIK_ADDR_SURF_MACRO_TILE_ASPECT_4 2 +# define CIK_ADDR_SURF_MACRO_TILE_ASPECT_8 3 +# define CIK_GRPH_ARRAY_MODE(x) (((x) & 0x7) << 20) +# define CIK_GRPH_ARRAY_LINEAR_GENERAL 0 +# define CIK_GRPH_ARRAY_LINEAR_ALIGNED 1 +# define CIK_GRPH_ARRAY_1D_TILED_THIN1 2 +# define CIK_GRPH_ARRAY_2D_TILED_THIN1 4 +# define CIK_GRPH_PIPE_CONFIG(x) (((x) & 0x1f) << 24) +# define CIK_ADDR_SURF_P2 0 +# define CIK_ADDR_SURF_P4_8x16 4 +# define CIK_ADDR_SURF_P4_16x16 5 +# define CIK_ADDR_SURF_P4_16x32 6 +# define CIK_ADDR_SURF_P4_32x32 7 +# define CIK_ADDR_SURF_P8_16x16_8x16 8 +# define CIK_ADDR_SURF_P8_16x32_8x16 9 +# define CIK_ADDR_SURF_P8_32x32_8x16 10 +# define CIK_ADDR_SURF_P8_16x32_16x16 11 +# define CIK_ADDR_SURF_P8_32x32_16x16 12 +# define CIK_ADDR_SURF_P8_32x32_16x32 13 +# define CIK_ADDR_SURF_P8_32x64_32x32 14 +# define CIK_GRPH_MICRO_TILE_MODE(x) (((x) & 0x7) << 29) +# define CIK_DISPLAY_MICRO_TILING 0 +# define CIK_THIN_MICRO_TILING 1 +# define CIK_DEPTH_MICRO_TILING 2 +# define CIK_ROTATED_MICRO_TILING 4 + +/* CUR blocks at 0x6998, 0x7598, 0x10198, 0x10d98, 0x11998, 0x12598 */ +#define CIK_CUR_CONTROL 0x6998 +# define CIK_CURSOR_EN (1 << 0) +# define CIK_CURSOR_MODE(x) (((x) & 0x3) << 8) +# define CIK_CURSOR_MONO 0 +# define CIK_CURSOR_24_1 1 +# define CIK_CURSOR_24_8_PRE_MULT 2 +# define CIK_CURSOR_24_8_UNPRE_MULT 3 +# define CIK_CURSOR_2X_MAGNIFY (1 << 16) +# define CIK_CURSOR_FORCE_MC_ON (1 << 20) +# define CIK_CURSOR_URGENT_CONTROL(x) (((x) & 0x7) << 24) +# define CIK_CURSOR_URGENT_ALWAYS 0 +# define CIK_CURSOR_URGENT_1_8 1 +# define CIK_CURSOR_URGENT_1_4 2 +# define CIK_CURSOR_URGENT_3_8 3 +# define CIK_CURSOR_URGENT_1_2 4 +#define CIK_CUR_SURFACE_ADDRESS 0x699c +# define CIK_CUR_SURFACE_ADDRESS_MASK 0xfffff000 +#define CIK_CUR_SIZE 0x69a0 +#define CIK_CUR_SURFACE_ADDRESS_HIGH 0x69a4 +#define CIK_CUR_POSITION 0x69a8 +#define CIK_CUR_HOT_SPOT 0x69ac +#define CIK_CUR_COLOR1 0x69b0 +#define CIK_CUR_COLOR2 0x69b4 +#define CIK_CUR_UPDATE 0x69b8 +# define CIK_CURSOR_UPDATE_PENDING (1 << 0) +# define CIK_CURSOR_UPDATE_TAKEN (1 << 1) +# define CIK_CURSOR_UPDATE_LOCK (1 << 16) +# define CIK_CURSOR_DISABLE_MULTIPLE_UPDATE (1 << 24) + +#define CIK_ALPHA_CONTROL 0x6af0 +# define CIK_CURSOR_ALPHA_BLND_ENA (1 << 1) + +#define CIK_LB_DATA_FORMAT 0x6b00 +# define CIK_INTERLEAVE_EN (1 << 3) + +#define CIK_LB_DESKTOP_HEIGHT 0x6b0c + +#define KFD_CIK_SDMA_QUEUE_OFFSET 0x200 + +#define SQ_IND_INDEX 0x8DE0 +#define SQ_CMD 0x8DEC +#define SQ_IND_DATA 0x8DE4 + +/* + * The TCP_WATCHx_xxxx addresses that are shown here are in dwords, + * and that's why they are multiplied by 4 + */ +#define TCP_WATCH0_ADDR_H (0x32A0*4) +#define TCP_WATCH1_ADDR_H (0x32A3*4) +#define TCP_WATCH2_ADDR_H (0x32A6*4) +#define TCP_WATCH3_ADDR_H (0x32A9*4) +#define TCP_WATCH0_ADDR_L (0x32A1*4) +#define TCP_WATCH1_ADDR_L (0x32A4*4) +#define TCP_WATCH2_ADDR_L (0x32A7*4) +#define TCP_WATCH3_ADDR_L (0x32AA*4) +#define TCP_WATCH0_CNTL (0x32A2*4) +#define TCP_WATCH1_CNTL (0x32A5*4) +#define TCP_WATCH2_CNTL (0x32A8*4) +#define TCP_WATCH3_CNTL (0x32AB*4) + +#define CPC_INT_CNTL 0xC2D0 + +#define CP_HQD_IQ_RPTR 0xC970u +#define SDMA0_RLC0_RB_CNTL 0xD400u +#define SDMA_RB_VMID(x) (x << 24) +#define SDMA0_RLC0_RB_BASE 0xD404u +#define SDMA0_RLC0_RB_BASE_HI 0xD408u +#define SDMA0_RLC0_RB_RPTR 0xD40Cu +#define SDMA0_RLC0_RB_WPTR 0xD410u +#define SDMA0_RLC0_RB_WPTR_POLL_CNTL 0xD414u +#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI 0xD418u +#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO 0xD41Cu +#define SDMA0_RLC0_RB_RPTR_ADDR_HI 0xD420u +#define SDMA0_RLC0_RB_RPTR_ADDR_LO 0xD424u +#define SDMA0_RLC0_IB_CNTL 0xD428u +#define SDMA0_RLC0_IB_RPTR 0xD42Cu +#define SDMA0_RLC0_IB_OFFSET 0xD430u +#define SDMA0_RLC0_IB_BASE_LO 0xD434u +#define SDMA0_RLC0_IB_BASE_HI 0xD438u +#define SDMA0_RLC0_IB_SIZE 0xD43Cu +#define SDMA0_RLC0_SKIP_CNTL 0xD440u +#define SDMA0_RLC0_CONTEXT_STATUS 0xD444u +#define SDMA_RLC_IDLE (1 << 2) +#define SDMA0_RLC0_DOORBELL 0xD448u +#define SDMA_OFFSET(x) (x << 0) +#define SDMA_DB_ENABLE (1 << 28) +#define SDMA0_RLC0_VIRTUAL_ADDR 0xD49Cu +#define SDMA_ATC (1 << 0) +#define SDMA_VA_PTR32 (1 << 4) +#define SDMA_VA_SHARED_BASE(x) (x << 8) +#define SDMA0_RLC0_APE1_CNTL 0xD4A0u +#define SDMA0_RLC0_DOORBELL_LOG 0xD4A4u +#define SDMA0_RLC0_WATERMARK 0xD4A8u +#define SDMA0_CNTL 0xD010 +#define SDMA1_CNTL 0xD810 + +enum { + MAX_TRAPID = 8, /* 3 bits in the bitfield. */ + MAX_WATCH_ADDRESSES = 4 +}; + +enum { + ADDRESS_WATCH_REG_ADDR_HI = 0, + ADDRESS_WATCH_REG_ADDR_LO, + ADDRESS_WATCH_REG_CNTL, + ADDRESS_WATCH_REG_MAX +}; + +enum { /* not defined in the CI/KV reg file */ + ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL, + ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF, + ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000, + /* extend the mask to 26 bits in order to match the low address field */ + ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6, + ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF +}; + +union TCP_WATCH_CNTL_BITS { + struct { + uint32_t mask:24; + uint32_t vmid:4; + uint32_t atc:1; + uint32_t mode:2; + uint32_t valid:1; + } bitfields, bits; + uint32_t u32All; + signed int i32All; + float f32All; +}; + +#endif diff --git a/sys/dev/pci/drm/radeon/cik_sdma.c b/sys/dev/pci/drm/radeon/cik_sdma.c new file mode 100644 index 00000000000..3f36fa1e3d5 --- /dev/null +++ b/sys/dev/pci/drm/radeon/cik_sdma.c @@ -0,0 +1,995 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#include +#include "radeon.h" +#include "radeon_ucode.h" +#include "radeon_asic.h" +#include "radeon_trace.h" +#include "cikd.h" + +/* sdma */ +#define CIK_SDMA_UCODE_SIZE 1050 +#define CIK_SDMA_UCODE_VERSION 64 + +u32 cik_gpu_check_soft_reset(struct radeon_device *rdev); + +/* + * sDMA - System DMA + * Starting with CIK, the GPU has new asynchronous + * DMA engines. These engines are used for compute + * and gfx. There are two DMA engines (SDMA0, SDMA1) + * and each one supports 1 ring buffer used for gfx + * and 2 queues used for compute. + * + * The programming model is very similar to the CP + * (ring buffer, IBs, etc.), but sDMA has it's own + * packet format that is different from the PM4 format + * used by the CP. sDMA supports copying data, writing + * embedded data, solid fills, and a number of other + * things. It also has support for tiling/detiling of + * buffers. + */ + +/** + * cik_sdma_get_rptr - get the current read pointer + * + * @rdev: radeon_device pointer + * @ring: radeon ring pointer + * + * Get the current rptr from the hardware (CIK+). + */ +uint32_t cik_sdma_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 rptr, reg; + + if (rdev->wb.enabled) { + rptr = rdev->wb.wb[ring->rptr_offs/4]; + } else { + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + reg = SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET; + else + reg = SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET; + + rptr = RREG32(reg); + } + + return (rptr & 0x3fffc) >> 2; +} + +/** + * cik_sdma_get_wptr - get the current write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon ring pointer + * + * Get the current wptr from the hardware (CIK+). + */ +uint32_t cik_sdma_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 reg; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + reg = SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET; + else + reg = SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET; + + return (RREG32(reg) & 0x3fffc) >> 2; +} + +/** + * cik_sdma_set_wptr - commit the write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon ring pointer + * + * Write the wptr back to the hardware (CIK+). + */ +void cik_sdma_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 reg; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + reg = SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET; + else + reg = SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET; + + WREG32(reg, (ring->wptr << 2) & 0x3fffc); + (void)RREG32(reg); +} + +/** + * cik_sdma_ring_ib_execute - Schedule an IB on the DMA engine + * + * @rdev: radeon_device pointer + * @ib: IB object to schedule + * + * Schedule an IB in the DMA ring (CIK). + */ +void cik_sdma_ring_ib_execute(struct radeon_device *rdev, + struct radeon_ib *ib) +{ + struct radeon_ring *ring = &rdev->ring[ib->ring]; + u32 extra_bits = (ib->vm ? ib->vm->ids[ib->ring].id : 0) & 0xf; + + if (rdev->wb.enabled) { + u32 next_rptr = ring->wptr + 5; + while ((next_rptr & 7) != 4) + next_rptr++; + next_rptr += 4; + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); + radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); + radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); + radeon_ring_write(ring, 1); /* number of DWs to follow */ + radeon_ring_write(ring, next_rptr); + } + + /* IB packet must end on a 8 DW boundary */ + while ((ring->wptr & 7) != 4) + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits)); + radeon_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */ + radeon_ring_write(ring, upper_32_bits(ib->gpu_addr)); + radeon_ring_write(ring, ib->length_dw); + +} + +/** + * cik_sdma_hdp_flush_ring_emit - emit an hdp flush on the DMA ring + * + * @rdev: radeon_device pointer + * @ridx: radeon ring index + * + * Emit an hdp flush packet on the requested DMA ring. + */ +static void cik_sdma_hdp_flush_ring_emit(struct radeon_device *rdev, + int ridx) +{ + struct radeon_ring *ring = &rdev->ring[ridx]; + u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) | + SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */ + u32 ref_and_mask; + + if (ridx == R600_RING_TYPE_DMA_INDEX) + ref_and_mask = SDMA0; + else + ref_and_mask = SDMA1; + + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits)); + radeon_ring_write(ring, GPU_HDP_FLUSH_DONE); + radeon_ring_write(ring, GPU_HDP_FLUSH_REQ); + radeon_ring_write(ring, ref_and_mask); /* reference */ + radeon_ring_write(ring, ref_and_mask); /* mask */ + radeon_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */ +} + +/** + * cik_sdma_fence_ring_emit - emit a fence on the DMA ring + * + * @rdev: radeon_device pointer + * @fence: radeon fence object + * + * Add a DMA fence packet to the ring to write + * the fence seq number and DMA trap packet to generate + * an interrupt if needed (CIK). + */ +void cik_sdma_fence_ring_emit(struct radeon_device *rdev, + struct radeon_fence *fence) +{ + struct radeon_ring *ring = &rdev->ring[fence->ring]; + u64 addr = rdev->fence_drv[fence->ring].gpu_addr; + + /* write the fence */ + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0)); + radeon_ring_write(ring, lower_32_bits(addr)); + radeon_ring_write(ring, upper_32_bits(addr)); + radeon_ring_write(ring, fence->seq); + /* generate an interrupt */ + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0)); + /* flush HDP */ + cik_sdma_hdp_flush_ring_emit(rdev, fence->ring); +} + +/** + * cik_sdma_semaphore_ring_emit - emit a semaphore on the dma ring + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * @semaphore: radeon semaphore object + * @emit_wait: wait or signal semaphore + * + * Add a DMA semaphore packet to the ring wait on or signal + * other rings (CIK). + */ +bool cik_sdma_semaphore_ring_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait) +{ + u64 addr = semaphore->gpu_addr; + u32 extra_bits = emit_wait ? 0 : SDMA_SEMAPHORE_EXTRA_S; + + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits)); + radeon_ring_write(ring, addr & 0xfffffff8); + radeon_ring_write(ring, upper_32_bits(addr)); + + return true; +} + +/** + * cik_sdma_gfx_stop - stop the gfx async dma engines + * + * @rdev: radeon_device pointer + * + * Stop the gfx async dma ring buffers (CIK). + */ +static void cik_sdma_gfx_stop(struct radeon_device *rdev) +{ + u32 rb_cntl, reg_offset; + int i; + + if ((rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) || + (rdev->asic->copy.copy_ring_index == CAYMAN_RING_TYPE_DMA1_INDEX)) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + + for (i = 0; i < 2; i++) { + if (i == 0) + reg_offset = SDMA0_REGISTER_OFFSET; + else + reg_offset = SDMA1_REGISTER_OFFSET; + rb_cntl = RREG32(SDMA0_GFX_RB_CNTL + reg_offset); + rb_cntl &= ~SDMA_RB_ENABLE; + WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl); + WREG32(SDMA0_GFX_IB_CNTL + reg_offset, 0); + } + rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false; + rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false; + + /* FIXME use something else than big hammer but after few days can not + * seem to find good combination so reset SDMA blocks as it seems we + * do not shut them down properly. This fix hibernation and does not + * affect suspend to ram. + */ + WREG32(SRBM_SOFT_RESET, SOFT_RESET_SDMA | SOFT_RESET_SDMA1); + (void)RREG32(SRBM_SOFT_RESET); + udelay(50); + WREG32(SRBM_SOFT_RESET, 0); + (void)RREG32(SRBM_SOFT_RESET); +} + +/** + * cik_sdma_rlc_stop - stop the compute async dma engines + * + * @rdev: radeon_device pointer + * + * Stop the compute async dma queues (CIK). + */ +static void cik_sdma_rlc_stop(struct radeon_device *rdev) +{ + /* XXX todo */ +} + +/** + * cik_sdma_ctx_switch_enable - enable/disable sdma engine preemption + * + * @rdev: radeon_device pointer + * @enable: enable/disable preemption. + * + * Halt or unhalt the async dma engines (CIK). + */ +static void cik_sdma_ctx_switch_enable(struct radeon_device *rdev, bool enable) +{ + uint32_t reg_offset, value; + int i; + + for (i = 0; i < 2; i++) { + if (i == 0) + reg_offset = SDMA0_REGISTER_OFFSET; + else + reg_offset = SDMA1_REGISTER_OFFSET; + value = RREG32(SDMA0_CNTL + reg_offset); + if (enable) + value |= AUTO_CTXSW_ENABLE; + else + value &= ~AUTO_CTXSW_ENABLE; + WREG32(SDMA0_CNTL + reg_offset, value); + } +} + +/** + * cik_sdma_enable - stop the async dma engines + * + * @rdev: radeon_device pointer + * @enable: enable/disable the DMA MEs. + * + * Halt or unhalt the async dma engines (CIK). + */ +void cik_sdma_enable(struct radeon_device *rdev, bool enable) +{ + u32 me_cntl, reg_offset; + int i; + + if (enable == false) { + cik_sdma_gfx_stop(rdev); + cik_sdma_rlc_stop(rdev); + } + + for (i = 0; i < 2; i++) { + if (i == 0) + reg_offset = SDMA0_REGISTER_OFFSET; + else + reg_offset = SDMA1_REGISTER_OFFSET; + me_cntl = RREG32(SDMA0_ME_CNTL + reg_offset); + if (enable) + me_cntl &= ~SDMA_HALT; + else + me_cntl |= SDMA_HALT; + WREG32(SDMA0_ME_CNTL + reg_offset, me_cntl); + } + + cik_sdma_ctx_switch_enable(rdev, enable); +} + +/** + * cik_sdma_gfx_resume - setup and start the async dma engines + * + * @rdev: radeon_device pointer + * + * Set up the gfx DMA ring buffers and enable them (CIK). + * Returns 0 for success, error for failure. + */ +static int cik_sdma_gfx_resume(struct radeon_device *rdev) +{ + struct radeon_ring *ring; + u32 rb_cntl, ib_cntl; + u32 rb_bufsz; + u32 reg_offset, wb_offset; + int i, r; + + for (i = 0; i < 2; i++) { + if (i == 0) { + ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; + reg_offset = SDMA0_REGISTER_OFFSET; + wb_offset = R600_WB_DMA_RPTR_OFFSET; + } else { + ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; + reg_offset = SDMA1_REGISTER_OFFSET; + wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET; + } + + WREG32(SDMA0_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0); + WREG32(SDMA0_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0); + + /* Set ring buffer size in dwords */ + rb_bufsz = order_base_2(ring->ring_size / 4); + rb_cntl = rb_bufsz << 1; +#ifdef __BIG_ENDIAN + rb_cntl |= SDMA_RB_SWAP_ENABLE | SDMA_RPTR_WRITEBACK_SWAP_ENABLE; +#endif + WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl); + + /* Initialize the ring buffer's read and write pointers */ + WREG32(SDMA0_GFX_RB_RPTR + reg_offset, 0); + WREG32(SDMA0_GFX_RB_WPTR + reg_offset, 0); + + /* set the wb address whether it's enabled or not */ + WREG32(SDMA0_GFX_RB_RPTR_ADDR_HI + reg_offset, + upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF); + WREG32(SDMA0_GFX_RB_RPTR_ADDR_LO + reg_offset, + ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC)); + + if (rdev->wb.enabled) + rb_cntl |= SDMA_RPTR_WRITEBACK_ENABLE; + + WREG32(SDMA0_GFX_RB_BASE + reg_offset, ring->gpu_addr >> 8); + WREG32(SDMA0_GFX_RB_BASE_HI + reg_offset, ring->gpu_addr >> 40); + + ring->wptr = 0; + WREG32(SDMA0_GFX_RB_WPTR + reg_offset, ring->wptr << 2); + + /* enable DMA RB */ + WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl | SDMA_RB_ENABLE); + + ib_cntl = SDMA_IB_ENABLE; +#ifdef __BIG_ENDIAN + ib_cntl |= SDMA_IB_SWAP_ENABLE; +#endif + /* enable DMA IBs */ + WREG32(SDMA0_GFX_IB_CNTL + reg_offset, ib_cntl); + + ring->ready = true; + + r = radeon_ring_test(rdev, ring->idx, ring); + if (r) { + ring->ready = false; + return r; + } + } + + if ((rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) || + (rdev->asic->copy.copy_ring_index == CAYMAN_RING_TYPE_DMA1_INDEX)) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); + + return 0; +} + +/** + * cik_sdma_rlc_resume - setup and start the async dma engines + * + * @rdev: radeon_device pointer + * + * Set up the compute DMA queues and enable them (CIK). + * Returns 0 for success, error for failure. + */ +static int cik_sdma_rlc_resume(struct radeon_device *rdev) +{ + /* XXX todo */ + return 0; +} + +/** + * cik_sdma_load_microcode - load the sDMA ME ucode + * + * @rdev: radeon_device pointer + * + * Loads the sDMA0/1 ucode. + * Returns 0 for success, -EINVAL if the ucode is not available. + */ +static int cik_sdma_load_microcode(struct radeon_device *rdev) +{ + int i; + + if (!rdev->sdma_fw) + return -EINVAL; + + /* halt the MEs */ + cik_sdma_enable(rdev, false); + + if (rdev->new_fw) { + const struct sdma_firmware_header_v1_0 *hdr = + (const struct sdma_firmware_header_v1_0 *)rdev->sdma_fw->data; + const __le32 *fw_data; + u32 fw_size; + + radeon_ucode_print_sdma_hdr(&hdr->header); + + /* sdma0 */ + fw_data = (const __le32 *) + (rdev->sdma_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; + WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0); + for (i = 0; i < fw_size; i++) + WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, le32_to_cpup(fw_data++)); + WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION); + + /* sdma1 */ + fw_data = (const __le32 *) + (rdev->sdma_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; + WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0); + for (i = 0; i < fw_size; i++) + WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, le32_to_cpup(fw_data++)); + WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION); + } else { + const __be32 *fw_data; + + /* sdma0 */ + fw_data = (const __be32 *)rdev->sdma_fw->data; + WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0); + for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++) + WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, be32_to_cpup(fw_data++)); + WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION); + + /* sdma1 */ + fw_data = (const __be32 *)rdev->sdma_fw->data; + WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0); + for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++) + WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, be32_to_cpup(fw_data++)); + WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION); + } + + WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0); + WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0); + return 0; +} + +/** + * cik_sdma_resume - setup and start the async dma engines + * + * @rdev: radeon_device pointer + * + * Set up the DMA engines and enable them (CIK). + * Returns 0 for success, error for failure. + */ +int cik_sdma_resume(struct radeon_device *rdev) +{ + int r; + + r = cik_sdma_load_microcode(rdev); + if (r) + return r; + + /* unhalt the MEs */ + cik_sdma_enable(rdev, true); + + /* start the gfx rings and rlc compute queues */ + r = cik_sdma_gfx_resume(rdev); + if (r) + return r; + r = cik_sdma_rlc_resume(rdev); + if (r) + return r; + + return 0; +} + +/** + * cik_sdma_fini - tear down the async dma engines + * + * @rdev: radeon_device pointer + * + * Stop the async dma engines and free the rings (CIK). + */ +void cik_sdma_fini(struct radeon_device *rdev) +{ + /* halt the MEs */ + cik_sdma_enable(rdev, false); + radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]); + radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]); + /* XXX - compute dma queue tear down */ +} + +/** + * cik_copy_dma - copy pages using the DMA engine + * + * @rdev: radeon_device pointer + * @src_offset: src GPU address + * @dst_offset: dst GPU address + * @num_gpu_pages: number of GPU pages to xfer + * @resv: reservation object to sync to + * + * Copy GPU paging using the DMA engine (CIK). + * Used by the radeon ttm implementation to move pages if + * registered as the asic copy callback. + */ +struct radeon_fence *cik_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv) +{ + struct radeon_fence *fence; + struct radeon_sync sync; + int ring_index = rdev->asic->copy.dma_ring_index; + struct radeon_ring *ring = &rdev->ring[ring_index]; + u32 size_in_bytes, cur_size_in_bytes; + int i, num_loops; + int r = 0; + + radeon_sync_create(&sync); + + size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); + num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); + r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14); + if (r) { + DRM_ERROR("radeon: moving bo (%d).\n", r); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); + + for (i = 0; i < num_loops; i++) { + cur_size_in_bytes = size_in_bytes; + if (cur_size_in_bytes > 0x1fffff) + cur_size_in_bytes = 0x1fffff; + size_in_bytes -= cur_size_in_bytes; + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0)); + radeon_ring_write(ring, cur_size_in_bytes); + radeon_ring_write(ring, 0); /* src/dst endian swap */ + radeon_ring_write(ring, lower_32_bits(src_offset)); + radeon_ring_write(ring, upper_32_bits(src_offset)); + radeon_ring_write(ring, lower_32_bits(dst_offset)); + radeon_ring_write(ring, upper_32_bits(dst_offset)); + src_offset += cur_size_in_bytes; + dst_offset += cur_size_in_bytes; + } + + r = radeon_fence_emit(rdev, &fence, ring->idx); + if (r) { + radeon_ring_unlock_undo(rdev, ring); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_ring_unlock_commit(rdev, ring, false); + radeon_sync_free(rdev, &sync, fence); + + return fence; +} + +/** + * cik_sdma_ring_test - simple async dma engine test + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Test the DMA engine by writing using it to write an + * value to memory. (CIK). + * Returns 0 for success, error for failure. + */ +int cik_sdma_ring_test(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + unsigned i; + int r; + unsigned index; + u32 tmp; + u64 gpu_addr; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + index = R600_WB_DMA_RING_TEST_OFFSET; + else + index = CAYMAN_WB_DMA1_RING_TEST_OFFSET; + + gpu_addr = rdev->wb.gpu_addr + index; + + tmp = 0xCAFEDEAD; + rdev->wb.wb[index/4] = cpu_to_le32(tmp); + + r = radeon_ring_lock(rdev, ring, 5); + if (r) { + DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r); + return r; + } + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); + radeon_ring_write(ring, lower_32_bits(gpu_addr)); + radeon_ring_write(ring, upper_32_bits(gpu_addr)); + radeon_ring_write(ring, 1); /* number of DWs to follow */ + radeon_ring_write(ring, 0xDEADBEEF); + radeon_ring_unlock_commit(rdev, ring, false); + + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = le32_to_cpu(rdev->wb.wb[index/4]); + if (tmp == 0xDEADBEEF) + break; + DRM_UDELAY(1); + } + + if (i < rdev->usec_timeout) { + DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); + } else { + DRM_ERROR("radeon: ring %d test failed (0x%08X)\n", + ring->idx, tmp); + r = -EINVAL; + } + return r; +} + +/** + * cik_sdma_ib_test - test an IB on the DMA engine + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Test a simple IB in the DMA ring (CIK). + * Returns 0 on success, error on failure. + */ +int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) +{ + struct radeon_ib ib; + unsigned i; + unsigned index; + int r; + u32 tmp = 0; + u64 gpu_addr; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + index = R600_WB_DMA_RING_TEST_OFFSET; + else + index = CAYMAN_WB_DMA1_RING_TEST_OFFSET; + + gpu_addr = rdev->wb.gpu_addr + index; + + tmp = 0xCAFEDEAD; + rdev->wb.wb[index/4] = cpu_to_le32(tmp); + + r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); + if (r) { + DRM_ERROR("radeon: failed to get ib (%d).\n", r); + return r; + } + + ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0); + ib.ptr[1] = lower_32_bits(gpu_addr); + ib.ptr[2] = upper_32_bits(gpu_addr); + ib.ptr[3] = 1; + ib.ptr[4] = 0xDEADBEEF; + ib.length_dw = 5; + + r = radeon_ib_schedule(rdev, &ib, NULL, false); + if (r) { + radeon_ib_free(rdev, &ib); + DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); + return r; + } + r = radeon_fence_wait(ib.fence, false); + if (r) { + DRM_ERROR("radeon: fence wait failed (%d).\n", r); + return r; + } + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = le32_to_cpu(rdev->wb.wb[index/4]); + if (tmp == 0xDEADBEEF) + break; + DRM_UDELAY(1); + } + if (i < rdev->usec_timeout) { + DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); + } else { + DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp); + r = -EINVAL; + } + radeon_ib_free(rdev, &ib); + return r; +} + +/** + * cik_sdma_is_lockup - Check if the DMA engine is locked up + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Check if the async DMA engine is locked up (CIK). + * Returns true if the engine appears to be locked up, false if not. + */ +bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +{ + u32 reset_mask = cik_gpu_check_soft_reset(rdev); + u32 mask; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + mask = RADEON_RESET_DMA; + else + mask = RADEON_RESET_DMA1; + + if (!(reset_mask & mask)) { + radeon_ring_lockup_update(rdev, ring); + return false; + } + return radeon_ring_test_lockup(rdev, ring); +} + +/** + * cik_sdma_vm_copy_pages - update PTEs by copying them from the GART + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @src: src addr to copy from + * @count: number of page entries to update + * + * Update PTEs by copying them from the GART using sDMA (CIK). + */ +void cik_sdma_vm_copy_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, uint64_t src, + unsigned count) +{ + while (count) { + unsigned bytes = count * 8; + if (bytes > 0x1FFFF8) + bytes = 0x1FFFF8; + + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, + SDMA_WRITE_SUB_OPCODE_LINEAR, 0); + ib->ptr[ib->length_dw++] = bytes; + ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ + ib->ptr[ib->length_dw++] = lower_32_bits(src); + ib->ptr[ib->length_dw++] = upper_32_bits(src); + ib->ptr[ib->length_dw++] = lower_32_bits(pe); + ib->ptr[ib->length_dw++] = upper_32_bits(pe); + + pe += bytes; + src += bytes; + count -= bytes / 8; + } +} + +/** + * cik_sdma_vm_write_pages - update PTEs by writing them manually + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: access flags + * + * Update PTEs by writing them manually using sDMA (CIK). + */ +void cik_sdma_vm_write_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags) +{ + uint64_t value; + unsigned ndw; + + while (count) { + ndw = count * 2; + if (ndw > 0xFFFFE) + ndw = 0xFFFFE; + + /* for non-physically contiguous pages (system) */ + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE, + SDMA_WRITE_SUB_OPCODE_LINEAR, 0); + ib->ptr[ib->length_dw++] = pe; + ib->ptr[ib->length_dw++] = upper_32_bits(pe); + ib->ptr[ib->length_dw++] = ndw; + for (; ndw > 0; ndw -= 2, --count, pe += 8) { + if (flags & R600_PTE_SYSTEM) { + value = radeon_vm_map_gart(rdev, addr); + } else if (flags & R600_PTE_VALID) { + value = addr; + } else { + value = 0; + } + addr += incr; + value |= flags; + ib->ptr[ib->length_dw++] = value; + ib->ptr[ib->length_dw++] = upper_32_bits(value); + } + } +} + +/** + * cik_sdma_vm_set_pages - update the page tables using sDMA + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: access flags + * + * Update the page tables using sDMA (CIK). + */ +void cik_sdma_vm_set_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags) +{ + uint64_t value; + unsigned ndw; + + while (count) { + ndw = count; + if (ndw > 0x7FFFF) + ndw = 0x7FFFF; + + if (flags & R600_PTE_VALID) + value = addr; + else + value = 0; + + /* for physically contiguous pages (vram) */ + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0); + ib->ptr[ib->length_dw++] = pe; /* dst addr */ + ib->ptr[ib->length_dw++] = upper_32_bits(pe); + ib->ptr[ib->length_dw++] = flags; /* mask */ + ib->ptr[ib->length_dw++] = 0; + ib->ptr[ib->length_dw++] = value; /* value */ + ib->ptr[ib->length_dw++] = upper_32_bits(value); + ib->ptr[ib->length_dw++] = incr; /* increment size */ + ib->ptr[ib->length_dw++] = 0; + ib->ptr[ib->length_dw++] = ndw; /* number of entries */ + + pe += ndw * 8; + addr += ndw * incr; + count -= ndw; + } +} + +/** + * cik_sdma_vm_pad_ib - pad the IB to the required number of dw + * + * @ib: indirect buffer to fill with padding + * + */ +void cik_sdma_vm_pad_ib(struct radeon_ib *ib) +{ + while (ib->length_dw & 0x7) + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0); +} + +/** + * cik_dma_vm_flush - cik vm flush using sDMA + * + * @rdev: radeon_device pointer + * + * Update the page table base and flush the VM TLB + * using sDMA (CIK). + */ +void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) +{ + u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) | + SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */ + + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + if (vm_id < 8) { + radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); + } else { + radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); + } + radeon_ring_write(ring, pd_addr >> 12); + + /* update SH_MEM_* regs */ + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); + radeon_ring_write(ring, VMID(vm_id)); + + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + radeon_ring_write(ring, SH_MEM_BASES >> 2); + radeon_ring_write(ring, 0); + + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + radeon_ring_write(ring, SH_MEM_CONFIG >> 2); + radeon_ring_write(ring, 0); + + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + radeon_ring_write(ring, SH_MEM_APE1_BASE >> 2); + radeon_ring_write(ring, 1); + + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + radeon_ring_write(ring, SH_MEM_APE1_LIMIT >> 2); + radeon_ring_write(ring, 0); + + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); + radeon_ring_write(ring, VMID(0)); + + /* flush HDP */ + cik_sdma_hdp_flush_ring_emit(rdev, ring->idx); + + /* flush TLB */ + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); + radeon_ring_write(ring, 1 << vm_id); + + radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits)); + radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, 0); /* reference */ + radeon_ring_write(ring, 0); /* mask */ + radeon_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */ +} + diff --git a/sys/dev/pci/drm/radeon/cikd.h b/sys/dev/pci/drm/radeon/cikd.h new file mode 100644 index 00000000000..391ff9d5d70 --- /dev/null +++ b/sys/dev/pci/drm/radeon/cikd.h @@ -0,0 +1,2172 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#ifndef CIK_H +#define CIK_H + +#define BONAIRE_GB_ADDR_CONFIG_GOLDEN 0x12010001 +#define HAWAII_GB_ADDR_CONFIG_GOLDEN 0x12011003 + +#define CIK_RB_BITMAP_WIDTH_PER_SH 2 +#define HAWAII_RB_BITMAP_WIDTH_PER_SH 4 + +#define RADEON_NUM_OF_VMIDS 8 + +/* DIDT IND registers */ +#define DIDT_SQ_CTRL0 0x0 +# define DIDT_CTRL_EN (1 << 0) +#define DIDT_DB_CTRL0 0x20 +#define DIDT_TD_CTRL0 0x40 +#define DIDT_TCP_CTRL0 0x60 + +/* SMC IND registers */ +#define DPM_TABLE_475 0x3F768 +# define SamuBootLevel(x) ((x) << 0) +# define SamuBootLevel_MASK 0x000000ff +# define SamuBootLevel_SHIFT 0 +# define AcpBootLevel(x) ((x) << 8) +# define AcpBootLevel_MASK 0x0000ff00 +# define AcpBootLevel_SHIFT 8 +# define VceBootLevel(x) ((x) << 16) +# define VceBootLevel_MASK 0x00ff0000 +# define VceBootLevel_SHIFT 16 +# define UvdBootLevel(x) ((x) << 24) +# define UvdBootLevel_MASK 0xff000000 +# define UvdBootLevel_SHIFT 24 + +#define FIRMWARE_FLAGS 0x3F800 +# define INTERRUPTS_ENABLED (1 << 0) + +#define NB_DPM_CONFIG_1 0x3F9E8 +# define Dpm0PgNbPsLo(x) ((x) << 0) +# define Dpm0PgNbPsLo_MASK 0x000000ff +# define Dpm0PgNbPsLo_SHIFT 0 +# define Dpm0PgNbPsHi(x) ((x) << 8) +# define Dpm0PgNbPsHi_MASK 0x0000ff00 +# define Dpm0PgNbPsHi_SHIFT 8 +# define DpmXNbPsLo(x) ((x) << 16) +# define DpmXNbPsLo_MASK 0x00ff0000 +# define DpmXNbPsLo_SHIFT 16 +# define DpmXNbPsHi(x) ((x) << 24) +# define DpmXNbPsHi_MASK 0xff000000 +# define DpmXNbPsHi_SHIFT 24 + +#define SMC_SYSCON_RESET_CNTL 0x80000000 +# define RST_REG (1 << 0) +#define SMC_SYSCON_CLOCK_CNTL_0 0x80000004 +# define CK_DISABLE (1 << 0) +# define CKEN (1 << 24) + +#define SMC_SYSCON_MISC_CNTL 0x80000010 + +#define SMC_SYSCON_MSG_ARG_0 0x80000068 + +#define SMC_PC_C 0x80000370 + +#define SMC_SCRATCH9 0x80000424 + +#define RCU_UC_EVENTS 0xC0000004 +# define BOOT_SEQ_DONE (1 << 7) + +#define GENERAL_PWRMGT 0xC0200000 +# define GLOBAL_PWRMGT_EN (1 << 0) +# define STATIC_PM_EN (1 << 1) +# define THERMAL_PROTECTION_DIS (1 << 2) +# define THERMAL_PROTECTION_TYPE (1 << 3) +# define SW_SMIO_INDEX(x) ((x) << 6) +# define SW_SMIO_INDEX_MASK (1 << 6) +# define SW_SMIO_INDEX_SHIFT 6 +# define VOLT_PWRMGT_EN (1 << 10) +# define GPU_COUNTER_CLK (1 << 15) +# define DYN_SPREAD_SPECTRUM_EN (1 << 23) + +#define CNB_PWRMGT_CNTL 0xC0200004 +# define GNB_SLOW_MODE(x) ((x) << 0) +# define GNB_SLOW_MODE_MASK (3 << 0) +# define GNB_SLOW_MODE_SHIFT 0 +# define GNB_SLOW (1 << 2) +# define FORCE_NB_PS1 (1 << 3) +# define DPM_ENABLED (1 << 4) + +#define SCLK_PWRMGT_CNTL 0xC0200008 +# define SCLK_PWRMGT_OFF (1 << 0) +# define RESET_BUSY_CNT (1 << 4) +# define RESET_SCLK_CNT (1 << 5) +# define DYNAMIC_PM_EN (1 << 21) + +#define TARGET_AND_CURRENT_PROFILE_INDEX 0xC0200014 +# define CURRENT_STATE_MASK (0xf << 4) +# define CURRENT_STATE_SHIFT 4 +# define CURR_MCLK_INDEX_MASK (0xf << 8) +# define CURR_MCLK_INDEX_SHIFT 8 +# define CURR_SCLK_INDEX_MASK (0x1f << 16) +# define CURR_SCLK_INDEX_SHIFT 16 + +#define CG_SSP 0xC0200044 +# define SST(x) ((x) << 0) +# define SST_MASK (0xffff << 0) +# define SSTU(x) ((x) << 16) +# define SSTU_MASK (0xf << 16) + +#define CG_DISPLAY_GAP_CNTL 0xC0200060 +# define DISP_GAP(x) ((x) << 0) +# define DISP_GAP_MASK (3 << 0) +# define VBI_TIMER_COUNT(x) ((x) << 4) +# define VBI_TIMER_COUNT_MASK (0x3fff << 4) +# define VBI_TIMER_UNIT(x) ((x) << 20) +# define VBI_TIMER_UNIT_MASK (7 << 20) +# define DISP_GAP_MCHG(x) ((x) << 24) +# define DISP_GAP_MCHG_MASK (3 << 24) + +#define SMU_VOLTAGE_STATUS 0xC0200094 +# define SMU_VOLTAGE_CURRENT_LEVEL_MASK (0xff << 1) +# define SMU_VOLTAGE_CURRENT_LEVEL_SHIFT 1 + +#define TARGET_AND_CURRENT_PROFILE_INDEX_1 0xC02000F0 +# define CURR_PCIE_INDEX_MASK (0xf << 24) +# define CURR_PCIE_INDEX_SHIFT 24 + +#define CG_ULV_PARAMETER 0xC0200158 + +#define CG_FTV_0 0xC02001A8 +#define CG_FTV_1 0xC02001AC +#define CG_FTV_2 0xC02001B0 +#define CG_FTV_3 0xC02001B4 +#define CG_FTV_4 0xC02001B8 +#define CG_FTV_5 0xC02001BC +#define CG_FTV_6 0xC02001C0 +#define CG_FTV_7 0xC02001C4 + +#define CG_DISPLAY_GAP_CNTL2 0xC0200230 + +#define LCAC_SX0_OVR_SEL 0xC0400D04 +#define LCAC_SX0_OVR_VAL 0xC0400D08 + +#define LCAC_MC0_CNTL 0xC0400D30 +#define LCAC_MC0_OVR_SEL 0xC0400D34 +#define LCAC_MC0_OVR_VAL 0xC0400D38 +#define LCAC_MC1_CNTL 0xC0400D3C +#define LCAC_MC1_OVR_SEL 0xC0400D40 +#define LCAC_MC1_OVR_VAL 0xC0400D44 + +#define LCAC_MC2_OVR_SEL 0xC0400D4C +#define LCAC_MC2_OVR_VAL 0xC0400D50 + +#define LCAC_MC3_OVR_SEL 0xC0400D58 +#define LCAC_MC3_OVR_VAL 0xC0400D5C + +#define LCAC_CPL_CNTL 0xC0400D80 +#define LCAC_CPL_OVR_SEL 0xC0400D84 +#define LCAC_CPL_OVR_VAL 0xC0400D88 + +/* dGPU */ +#define CG_THERMAL_CTRL 0xC0300004 +#define DPM_EVENT_SRC(x) ((x) << 0) +#define DPM_EVENT_SRC_MASK (7 << 0) +#define DIG_THERM_DPM(x) ((x) << 14) +#define DIG_THERM_DPM_MASK 0x003FC000 +#define DIG_THERM_DPM_SHIFT 14 +#define CG_THERMAL_STATUS 0xC0300008 +#define FDO_PWM_DUTY(x) ((x) << 9) +#define FDO_PWM_DUTY_MASK (0xff << 9) +#define FDO_PWM_DUTY_SHIFT 9 +#define CG_THERMAL_INT 0xC030000C +#define CI_DIG_THERM_INTH(x) ((x) << 8) +#define CI_DIG_THERM_INTH_MASK 0x0000FF00 +#define CI_DIG_THERM_INTH_SHIFT 8 +#define CI_DIG_THERM_INTL(x) ((x) << 16) +#define CI_DIG_THERM_INTL_MASK 0x00FF0000 +#define CI_DIG_THERM_INTL_SHIFT 16 +#define THERM_INT_MASK_HIGH (1 << 24) +#define THERM_INT_MASK_LOW (1 << 25) +#define CG_MULT_THERMAL_CTRL 0xC0300010 +#define TEMP_SEL(x) ((x) << 20) +#define TEMP_SEL_MASK (0xff << 20) +#define TEMP_SEL_SHIFT 20 +#define CG_MULT_THERMAL_STATUS 0xC0300014 +#define ASIC_MAX_TEMP(x) ((x) << 0) +#define ASIC_MAX_TEMP_MASK 0x000001ff +#define ASIC_MAX_TEMP_SHIFT 0 +#define CTF_TEMP(x) ((x) << 9) +#define CTF_TEMP_MASK 0x0003fe00 +#define CTF_TEMP_SHIFT 9 + +#define CG_FDO_CTRL0 0xC0300064 +#define FDO_STATIC_DUTY(x) ((x) << 0) +#define FDO_STATIC_DUTY_MASK 0x000000FF +#define FDO_STATIC_DUTY_SHIFT 0 +#define CG_FDO_CTRL1 0xC0300068 +#define FMAX_DUTY100(x) ((x) << 0) +#define FMAX_DUTY100_MASK 0x000000FF +#define FMAX_DUTY100_SHIFT 0 +#define CG_FDO_CTRL2 0xC030006C +#define TMIN(x) ((x) << 0) +#define TMIN_MASK 0x000000FF +#define TMIN_SHIFT 0 +#define FDO_PWM_MODE(x) ((x) << 11) +#define FDO_PWM_MODE_MASK (7 << 11) +#define FDO_PWM_MODE_SHIFT 11 +#define TACH_PWM_RESP_RATE(x) ((x) << 25) +#define TACH_PWM_RESP_RATE_MASK (0x7f << 25) +#define TACH_PWM_RESP_RATE_SHIFT 25 +#define CG_TACH_CTRL 0xC0300070 +# define EDGE_PER_REV(x) ((x) << 0) +# define EDGE_PER_REV_MASK (0x7 << 0) +# define EDGE_PER_REV_SHIFT 0 +# define TARGET_PERIOD(x) ((x) << 3) +# define TARGET_PERIOD_MASK 0xfffffff8 +# define TARGET_PERIOD_SHIFT 3 +#define CG_TACH_STATUS 0xC0300074 +# define TACH_PERIOD(x) ((x) << 0) +# define TACH_PERIOD_MASK 0xffffffff +# define TACH_PERIOD_SHIFT 0 + +#define CG_ECLK_CNTL 0xC05000AC +# define ECLK_DIVIDER_MASK 0x7f +# define ECLK_DIR_CNTL_EN (1 << 8) +#define CG_ECLK_STATUS 0xC05000B0 +# define ECLK_STATUS (1 << 0) + +#define CG_SPLL_FUNC_CNTL 0xC0500140 +#define SPLL_RESET (1 << 0) +#define SPLL_PWRON (1 << 1) +#define SPLL_BYPASS_EN (1 << 3) +#define SPLL_REF_DIV(x) ((x) << 5) +#define SPLL_REF_DIV_MASK (0x3f << 5) +#define SPLL_PDIV_A(x) ((x) << 20) +#define SPLL_PDIV_A_MASK (0x7f << 20) +#define SPLL_PDIV_A_SHIFT 20 +#define CG_SPLL_FUNC_CNTL_2 0xC0500144 +#define SCLK_MUX_SEL(x) ((x) << 0) +#define SCLK_MUX_SEL_MASK (0x1ff << 0) +#define CG_SPLL_FUNC_CNTL_3 0xC0500148 +#define SPLL_FB_DIV(x) ((x) << 0) +#define SPLL_FB_DIV_MASK (0x3ffffff << 0) +#define SPLL_FB_DIV_SHIFT 0 +#define SPLL_DITHEN (1 << 28) +#define CG_SPLL_FUNC_CNTL_4 0xC050014C + +#define CG_SPLL_SPREAD_SPECTRUM 0xC0500164 +#define SSEN (1 << 0) +#define CLK_S(x) ((x) << 4) +#define CLK_S_MASK (0xfff << 4) +#define CLK_S_SHIFT 4 +#define CG_SPLL_SPREAD_SPECTRUM_2 0xC0500168 +#define CLK_V(x) ((x) << 0) +#define CLK_V_MASK (0x3ffffff << 0) +#define CLK_V_SHIFT 0 + +#define MPLL_BYPASSCLK_SEL 0xC050019C +# define MPLL_CLKOUT_SEL(x) ((x) << 8) +# define MPLL_CLKOUT_SEL_MASK 0xFF00 +#define CG_CLKPIN_CNTL 0xC05001A0 +# define XTALIN_DIVIDE (1 << 1) +# define BCLK_AS_XCLK (1 << 2) +#define CG_CLKPIN_CNTL_2 0xC05001A4 +# define FORCE_BIF_REFCLK_EN (1 << 3) +# define MUX_TCLK_TO_XCLK (1 << 8) +#define THM_CLK_CNTL 0xC05001A8 +# define CMON_CLK_SEL(x) ((x) << 0) +# define CMON_CLK_SEL_MASK 0xFF +# define TMON_CLK_SEL(x) ((x) << 8) +# define TMON_CLK_SEL_MASK 0xFF00 +#define MISC_CLK_CTRL 0xC05001AC +# define DEEP_SLEEP_CLK_SEL(x) ((x) << 0) +# define DEEP_SLEEP_CLK_SEL_MASK 0xFF +# define ZCLK_SEL(x) ((x) << 8) +# define ZCLK_SEL_MASK 0xFF00 + +/* KV/KB */ +#define CG_THERMAL_INT_CTRL 0xC2100028 +#define DIG_THERM_INTH(x) ((x) << 0) +#define DIG_THERM_INTH_MASK 0x000000FF +#define DIG_THERM_INTH_SHIFT 0 +#define DIG_THERM_INTL(x) ((x) << 8) +#define DIG_THERM_INTL_MASK 0x0000FF00 +#define DIG_THERM_INTL_SHIFT 8 +#define THERM_INTH_MASK (1 << 24) +#define THERM_INTL_MASK (1 << 25) + +/* PCIE registers idx/data 0x38/0x3c */ +#define PB0_PIF_PWRDOWN_0 0x1100012 /* PCIE */ +# define PLL_POWER_STATE_IN_TXS2_0(x) ((x) << 7) +# define PLL_POWER_STATE_IN_TXS2_0_MASK (0x7 << 7) +# define PLL_POWER_STATE_IN_TXS2_0_SHIFT 7 +# define PLL_POWER_STATE_IN_OFF_0(x) ((x) << 10) +# define PLL_POWER_STATE_IN_OFF_0_MASK (0x7 << 10) +# define PLL_POWER_STATE_IN_OFF_0_SHIFT 10 +# define PLL_RAMP_UP_TIME_0(x) ((x) << 24) +# define PLL_RAMP_UP_TIME_0_MASK (0x7 << 24) +# define PLL_RAMP_UP_TIME_0_SHIFT 24 +#define PB0_PIF_PWRDOWN_1 0x1100013 /* PCIE */ +# define PLL_POWER_STATE_IN_TXS2_1(x) ((x) << 7) +# define PLL_POWER_STATE_IN_TXS2_1_MASK (0x7 << 7) +# define PLL_POWER_STATE_IN_TXS2_1_SHIFT 7 +# define PLL_POWER_STATE_IN_OFF_1(x) ((x) << 10) +# define PLL_POWER_STATE_IN_OFF_1_MASK (0x7 << 10) +# define PLL_POWER_STATE_IN_OFF_1_SHIFT 10 +# define PLL_RAMP_UP_TIME_1(x) ((x) << 24) +# define PLL_RAMP_UP_TIME_1_MASK (0x7 << 24) +# define PLL_RAMP_UP_TIME_1_SHIFT 24 + +#define PCIE_CNTL2 0x1001001c /* PCIE */ +# define SLV_MEM_LS_EN (1 << 16) +# define SLV_MEM_AGGRESSIVE_LS_EN (1 << 17) +# define MST_MEM_LS_EN (1 << 18) +# define REPLAY_MEM_LS_EN (1 << 19) + +#define PCIE_LC_STATUS1 0x1400028 /* PCIE */ +# define LC_REVERSE_RCVR (1 << 0) +# define LC_REVERSE_XMIT (1 << 1) +# define LC_OPERATING_LINK_WIDTH_MASK (0x7 << 2) +# define LC_OPERATING_LINK_WIDTH_SHIFT 2 +# define LC_DETECTED_LINK_WIDTH_MASK (0x7 << 5) +# define LC_DETECTED_LINK_WIDTH_SHIFT 5 + +#define PCIE_P_CNTL 0x1400040 /* PCIE */ +# define P_IGNORE_EDB_ERR (1 << 6) + +#define PB1_PIF_PWRDOWN_0 0x2100012 /* PCIE */ +#define PB1_PIF_PWRDOWN_1 0x2100013 /* PCIE */ + +#define PCIE_LC_CNTL 0x100100A0 /* PCIE */ +# define LC_L0S_INACTIVITY(x) ((x) << 8) +# define LC_L0S_INACTIVITY_MASK (0xf << 8) +# define LC_L0S_INACTIVITY_SHIFT 8 +# define LC_L1_INACTIVITY(x) ((x) << 12) +# define LC_L1_INACTIVITY_MASK (0xf << 12) +# define LC_L1_INACTIVITY_SHIFT 12 +# define LC_PMI_TO_L1_DIS (1 << 16) +# define LC_ASPM_TO_L1_DIS (1 << 24) + +#define PCIE_LC_LINK_WIDTH_CNTL 0x100100A2 /* PCIE */ +# define LC_LINK_WIDTH_SHIFT 0 +# define LC_LINK_WIDTH_MASK 0x7 +# define LC_LINK_WIDTH_X0 0 +# define LC_LINK_WIDTH_X1 1 +# define LC_LINK_WIDTH_X2 2 +# define LC_LINK_WIDTH_X4 3 +# define LC_LINK_WIDTH_X8 4 +# define LC_LINK_WIDTH_X16 6 +# define LC_LINK_WIDTH_RD_SHIFT 4 +# define LC_LINK_WIDTH_RD_MASK 0x70 +# define LC_RECONFIG_ARC_MISSING_ESCAPE (1 << 7) +# define LC_RECONFIG_NOW (1 << 8) +# define LC_RENEGOTIATION_SUPPORT (1 << 9) +# define LC_RENEGOTIATE_EN (1 << 10) +# define LC_SHORT_RECONFIG_EN (1 << 11) +# define LC_UPCONFIGURE_SUPPORT (1 << 12) +# define LC_UPCONFIGURE_DIS (1 << 13) +# define LC_DYN_LANES_PWR_STATE(x) ((x) << 21) +# define LC_DYN_LANES_PWR_STATE_MASK (0x3 << 21) +# define LC_DYN_LANES_PWR_STATE_SHIFT 21 +#define PCIE_LC_N_FTS_CNTL 0x100100a3 /* PCIE */ +# define LC_XMIT_N_FTS(x) ((x) << 0) +# define LC_XMIT_N_FTS_MASK (0xff << 0) +# define LC_XMIT_N_FTS_SHIFT 0 +# define LC_XMIT_N_FTS_OVERRIDE_EN (1 << 8) +# define LC_N_FTS_MASK (0xff << 24) +#define PCIE_LC_SPEED_CNTL 0x100100A4 /* PCIE */ +# define LC_GEN2_EN_STRAP (1 << 0) +# define LC_GEN3_EN_STRAP (1 << 1) +# define LC_TARGET_LINK_SPEED_OVERRIDE_EN (1 << 2) +# define LC_TARGET_LINK_SPEED_OVERRIDE_MASK (0x3 << 3) +# define LC_TARGET_LINK_SPEED_OVERRIDE_SHIFT 3 +# define LC_FORCE_EN_SW_SPEED_CHANGE (1 << 5) +# define LC_FORCE_DIS_SW_SPEED_CHANGE (1 << 6) +# define LC_FORCE_EN_HW_SPEED_CHANGE (1 << 7) +# define LC_FORCE_DIS_HW_SPEED_CHANGE (1 << 8) +# define LC_INITIATE_LINK_SPEED_CHANGE (1 << 9) +# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK (0x3 << 10) +# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT 10 +# define LC_CURRENT_DATA_RATE_MASK (0x3 << 13) /* 0/1/2 = gen1/2/3 */ +# define LC_CURRENT_DATA_RATE_SHIFT 13 +# define LC_CLR_FAILED_SPD_CHANGE_CNT (1 << 16) +# define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 18) +# define LC_OTHER_SIDE_SUPPORTS_GEN2 (1 << 19) +# define LC_OTHER_SIDE_EVER_SENT_GEN3 (1 << 20) +# define LC_OTHER_SIDE_SUPPORTS_GEN3 (1 << 21) + +#define PCIE_LC_CNTL2 0x100100B1 /* PCIE */ +# define LC_ALLOW_PDWN_IN_L1 (1 << 17) +# define LC_ALLOW_PDWN_IN_L23 (1 << 18) + +#define PCIE_LC_CNTL3 0x100100B5 /* PCIE */ +# define LC_GO_TO_RECOVERY (1 << 30) +#define PCIE_LC_CNTL4 0x100100B6 /* PCIE */ +# define LC_REDO_EQ (1 << 5) +# define LC_SET_QUIESCE (1 << 13) + +/* direct registers */ +#define PCIE_INDEX 0x38 +#define PCIE_DATA 0x3C + +#define SMC_IND_INDEX_0 0x200 +#define SMC_IND_DATA_0 0x204 + +#define SMC_IND_ACCESS_CNTL 0x240 +#define AUTO_INCREMENT_IND_0 (1 << 0) + +#define SMC_MESSAGE_0 0x250 +#define SMC_MSG_MASK 0xffff +#define SMC_RESP_0 0x254 +#define SMC_RESP_MASK 0xffff + +#define SMC_MSG_ARG_0 0x290 + +#define VGA_HDP_CONTROL 0x328 +#define VGA_MEMORY_DISABLE (1 << 4) + +#define DMIF_ADDR_CALC 0xC00 + +#define PIPE0_DMIF_BUFFER_CONTROL 0x0ca0 +# define DMIF_BUFFERS_ALLOCATED(x) ((x) << 0) +# define DMIF_BUFFERS_ALLOCATED_COMPLETED (1 << 4) + +#define SRBM_GFX_CNTL 0xE44 +#define PIPEID(x) ((x) << 0) +#define MEID(x) ((x) << 2) +#define VMID(x) ((x) << 4) +#define QUEUEID(x) ((x) << 8) + +#define SRBM_STATUS2 0xE4C +#define SDMA_BUSY (1 << 5) +#define SDMA1_BUSY (1 << 6) +#define SRBM_STATUS 0xE50 +#define UVD_RQ_PENDING (1 << 1) +#define GRBM_RQ_PENDING (1 << 5) +#define VMC_BUSY (1 << 8) +#define MCB_BUSY (1 << 9) +#define MCB_NON_DISPLAY_BUSY (1 << 10) +#define MCC_BUSY (1 << 11) +#define MCD_BUSY (1 << 12) +#define SEM_BUSY (1 << 14) +#define IH_BUSY (1 << 17) +#define UVD_BUSY (1 << 19) + +#define SRBM_SOFT_RESET 0xE60 +#define SOFT_RESET_BIF (1 << 1) +#define SOFT_RESET_R0PLL (1 << 4) +#define SOFT_RESET_DC (1 << 5) +#define SOFT_RESET_SDMA1 (1 << 6) +#define SOFT_RESET_GRBM (1 << 8) +#define SOFT_RESET_HDP (1 << 9) +#define SOFT_RESET_IH (1 << 10) +#define SOFT_RESET_MC (1 << 11) +#define SOFT_RESET_ROM (1 << 14) +#define SOFT_RESET_SEM (1 << 15) +#define SOFT_RESET_VMC (1 << 17) +#define SOFT_RESET_SDMA (1 << 20) +#define SOFT_RESET_TST (1 << 21) +#define SOFT_RESET_REGBB (1 << 22) +#define SOFT_RESET_ORB (1 << 23) +#define SOFT_RESET_VCE (1 << 24) + +#define SRBM_READ_ERROR 0xE98 +#define SRBM_INT_CNTL 0xEA0 +#define SRBM_INT_ACK 0xEA8 + +#define VM_L2_CNTL 0x1400 +#define ENABLE_L2_CACHE (1 << 0) +#define ENABLE_L2_FRAGMENT_PROCESSING (1 << 1) +#define L2_CACHE_PTE_ENDIAN_SWAP_MODE(x) ((x) << 2) +#define L2_CACHE_PDE_ENDIAN_SWAP_MODE(x) ((x) << 4) +#define ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE (1 << 9) +#define ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE (1 << 10) +#define EFFECTIVE_L2_QUEUE_SIZE(x) (((x) & 7) << 15) +#define CONTEXT1_IDENTITY_ACCESS_MODE(x) (((x) & 3) << 19) +#define VM_L2_CNTL2 0x1404 +#define INVALIDATE_ALL_L1_TLBS (1 << 0) +#define INVALIDATE_L2_CACHE (1 << 1) +#define INVALIDATE_CACHE_MODE(x) ((x) << 26) +#define INVALIDATE_PTE_AND_PDE_CACHES 0 +#define INVALIDATE_ONLY_PTE_CACHES 1 +#define INVALIDATE_ONLY_PDE_CACHES 2 +#define VM_L2_CNTL3 0x1408 +#define BANK_SELECT(x) ((x) << 0) +#define L2_CACHE_UPDATE_MODE(x) ((x) << 6) +#define L2_CACHE_BIGK_FRAGMENT_SIZE(x) ((x) << 15) +#define L2_CACHE_BIGK_ASSOCIATIVITY (1 << 20) +#define VM_L2_STATUS 0x140C +#define L2_BUSY (1 << 0) +#define VM_CONTEXT0_CNTL 0x1410 +#define ENABLE_CONTEXT (1 << 0) +#define PAGE_TABLE_DEPTH(x) (((x) & 3) << 1) +#define RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT (1 << 3) +#define RANGE_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 4) +#define DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT (1 << 6) +#define DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 7) +#define PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT (1 << 9) +#define PDE0_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 10) +#define VALID_PROTECTION_FAULT_ENABLE_INTERRUPT (1 << 12) +#define VALID_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 13) +#define READ_PROTECTION_FAULT_ENABLE_INTERRUPT (1 << 15) +#define READ_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 16) +#define WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT (1 << 18) +#define WRITE_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 19) +#define PAGE_TABLE_BLOCK_SIZE(x) (((x) & 0xF) << 24) +#define VM_CONTEXT1_CNTL 0x1414 +#define VM_CONTEXT0_CNTL2 0x1430 +#define VM_CONTEXT1_CNTL2 0x1434 +#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR 0x1438 +#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR 0x143c +#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR 0x1440 +#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR 0x1444 +#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR 0x1448 +#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR 0x144c +#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR 0x1450 +#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR 0x1454 + +#define VM_INVALIDATE_REQUEST 0x1478 +#define VM_INVALIDATE_RESPONSE 0x147c + +#define VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x14DC +#define PROTECTIONS_MASK (0xf << 0) +#define PROTECTIONS_SHIFT 0 + /* bit 0: range + * bit 1: pde0 + * bit 2: valid + * bit 3: read + * bit 4: write + */ +#define MEMORY_CLIENT_ID_MASK (0xff << 12) +#define HAWAII_MEMORY_CLIENT_ID_MASK (0x1ff << 12) +#define MEMORY_CLIENT_ID_SHIFT 12 +#define MEMORY_CLIENT_RW_MASK (1 << 24) +#define MEMORY_CLIENT_RW_SHIFT 24 +#define FAULT_VMID_MASK (0xf << 25) +#define FAULT_VMID_SHIFT 25 + +#define VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT 0x14E4 + +#define VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x14FC + +#define VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR 0x1518 +#define VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR 0x151c + +#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR 0x153c +#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR 0x1540 +#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR 0x1544 +#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR 0x1548 +#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR 0x154c +#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR 0x1550 +#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR 0x1554 +#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR 0x1558 +#define VM_CONTEXT0_PAGE_TABLE_START_ADDR 0x155c +#define VM_CONTEXT1_PAGE_TABLE_START_ADDR 0x1560 + +#define VM_CONTEXT0_PAGE_TABLE_END_ADDR 0x157C +#define VM_CONTEXT1_PAGE_TABLE_END_ADDR 0x1580 + +#define VM_L2_CG 0x15c0 +#define MC_CG_ENABLE (1 << 18) +#define MC_LS_ENABLE (1 << 19) + +#define MC_SHARED_CHMAP 0x2004 +#define NOOFCHAN_SHIFT 12 +#define NOOFCHAN_MASK 0x0000f000 +#define MC_SHARED_CHREMAP 0x2008 + +#define CHUB_CONTROL 0x1864 +#define BYPASS_VM (1 << 0) + +#define MC_VM_FB_LOCATION 0x2024 +#define MC_VM_AGP_TOP 0x2028 +#define MC_VM_AGP_BOT 0x202C +#define MC_VM_AGP_BASE 0x2030 +#define MC_VM_SYSTEM_APERTURE_LOW_ADDR 0x2034 +#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR 0x2038 +#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR 0x203C + +#define MC_VM_MX_L1_TLB_CNTL 0x2064 +#define ENABLE_L1_TLB (1 << 0) +#define ENABLE_L1_FRAGMENT_PROCESSING (1 << 1) +#define SYSTEM_ACCESS_MODE_PA_ONLY (0 << 3) +#define SYSTEM_ACCESS_MODE_USE_SYS_MAP (1 << 3) +#define SYSTEM_ACCESS_MODE_IN_SYS (2 << 3) +#define SYSTEM_ACCESS_MODE_NOT_IN_SYS (3 << 3) +#define SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU (0 << 5) +#define ENABLE_ADVANCED_DRIVER_MODEL (1 << 6) +#define MC_VM_FB_OFFSET 0x2068 + +#define MC_SHARED_BLACKOUT_CNTL 0x20ac + +#define MC_HUB_MISC_HUB_CG 0x20b8 +#define MC_HUB_MISC_VM_CG 0x20bc + +#define MC_HUB_MISC_SIP_CG 0x20c0 + +#define MC_XPB_CLK_GAT 0x2478 + +#define MC_CITF_MISC_RD_CG 0x2648 +#define MC_CITF_MISC_WR_CG 0x264c +#define MC_CITF_MISC_VM_CG 0x2650 + +#define MC_ARB_RAMCFG 0x2760 +#define NOOFBANK_SHIFT 0 +#define NOOFBANK_MASK 0x00000003 +#define NOOFRANK_SHIFT 2 +#define NOOFRANK_MASK 0x00000004 +#define NOOFROWS_SHIFT 3 +#define NOOFROWS_MASK 0x00000038 +#define NOOFCOLS_SHIFT 6 +#define NOOFCOLS_MASK 0x000000C0 +#define CHANSIZE_SHIFT 8 +#define CHANSIZE_MASK 0x00000100 +#define NOOFGROUPS_SHIFT 12 +#define NOOFGROUPS_MASK 0x00001000 + +#define MC_ARB_DRAM_TIMING 0x2774 +#define MC_ARB_DRAM_TIMING2 0x2778 + +#define MC_ARB_BURST_TIME 0x2808 +#define STATE0(x) ((x) << 0) +#define STATE0_MASK (0x1f << 0) +#define STATE0_SHIFT 0 +#define STATE1(x) ((x) << 5) +#define STATE1_MASK (0x1f << 5) +#define STATE1_SHIFT 5 +#define STATE2(x) ((x) << 10) +#define STATE2_MASK (0x1f << 10) +#define STATE2_SHIFT 10 +#define STATE3(x) ((x) << 15) +#define STATE3_MASK (0x1f << 15) +#define STATE3_SHIFT 15 + +#define MC_SEQ_RAS_TIMING 0x28a0 +#define MC_SEQ_CAS_TIMING 0x28a4 +#define MC_SEQ_MISC_TIMING 0x28a8 +#define MC_SEQ_MISC_TIMING2 0x28ac +#define MC_SEQ_PMG_TIMING 0x28b0 +#define MC_SEQ_RD_CTL_D0 0x28b4 +#define MC_SEQ_RD_CTL_D1 0x28b8 +#define MC_SEQ_WR_CTL_D0 0x28bc +#define MC_SEQ_WR_CTL_D1 0x28c0 + +#define MC_SEQ_SUP_CNTL 0x28c8 +#define RUN_MASK (1 << 0) +#define MC_SEQ_SUP_PGM 0x28cc +#define MC_PMG_AUTO_CMD 0x28d0 + +#define MC_SEQ_TRAIN_WAKEUP_CNTL 0x28e8 +#define TRAIN_DONE_D0 (1 << 30) +#define TRAIN_DONE_D1 (1 << 31) + +#define MC_IO_PAD_CNTL_D0 0x29d0 +#define MEM_FALL_OUT_CMD (1 << 8) + +#define MC_SEQ_MISC0 0x2a00 +#define MC_SEQ_MISC0_VEN_ID_SHIFT 8 +#define MC_SEQ_MISC0_VEN_ID_MASK 0x00000f00 +#define MC_SEQ_MISC0_VEN_ID_VALUE 3 +#define MC_SEQ_MISC0_REV_ID_SHIFT 12 +#define MC_SEQ_MISC0_REV_ID_MASK 0x0000f000 +#define MC_SEQ_MISC0_REV_ID_VALUE 1 +#define MC_SEQ_MISC0_GDDR5_SHIFT 28 +#define MC_SEQ_MISC0_GDDR5_MASK 0xf0000000 +#define MC_SEQ_MISC0_GDDR5_VALUE 5 +#define MC_SEQ_MISC1 0x2a04 +#define MC_SEQ_RESERVE_M 0x2a08 +#define MC_PMG_CMD_EMRS 0x2a0c + +#define MC_SEQ_IO_DEBUG_INDEX 0x2a44 +#define MC_SEQ_IO_DEBUG_DATA 0x2a48 + +#define MC_SEQ_MISC5 0x2a54 +#define MC_SEQ_MISC6 0x2a58 + +#define MC_SEQ_MISC7 0x2a64 + +#define MC_SEQ_RAS_TIMING_LP 0x2a6c +#define MC_SEQ_CAS_TIMING_LP 0x2a70 +#define MC_SEQ_MISC_TIMING_LP 0x2a74 +#define MC_SEQ_MISC_TIMING2_LP 0x2a78 +#define MC_SEQ_WR_CTL_D0_LP 0x2a7c +#define MC_SEQ_WR_CTL_D1_LP 0x2a80 +#define MC_SEQ_PMG_CMD_EMRS_LP 0x2a84 +#define MC_SEQ_PMG_CMD_MRS_LP 0x2a88 + +#define MC_PMG_CMD_MRS 0x2aac + +#define MC_SEQ_RD_CTL_D0_LP 0x2b1c +#define MC_SEQ_RD_CTL_D1_LP 0x2b20 + +#define MC_PMG_CMD_MRS1 0x2b44 +#define MC_SEQ_PMG_CMD_MRS1_LP 0x2b48 +#define MC_SEQ_PMG_TIMING_LP 0x2b4c + +#define MC_SEQ_WR_CTL_2 0x2b54 +#define MC_SEQ_WR_CTL_2_LP 0x2b58 +#define MC_PMG_CMD_MRS2 0x2b5c +#define MC_SEQ_PMG_CMD_MRS2_LP 0x2b60 + +#define MCLK_PWRMGT_CNTL 0x2ba0 +# define DLL_SPEED(x) ((x) << 0) +# define DLL_SPEED_MASK (0x1f << 0) +# define DLL_READY (1 << 6) +# define MC_INT_CNTL (1 << 7) +# define MRDCK0_PDNB (1 << 8) +# define MRDCK1_PDNB (1 << 9) +# define MRDCK0_RESET (1 << 16) +# define MRDCK1_RESET (1 << 17) +# define DLL_READY_READ (1 << 24) +#define DLL_CNTL 0x2ba4 +# define MRDCK0_BYPASS (1 << 24) +# define MRDCK1_BYPASS (1 << 25) + +#define MPLL_FUNC_CNTL 0x2bb4 +#define BWCTRL(x) ((x) << 20) +#define BWCTRL_MASK (0xff << 20) +#define MPLL_FUNC_CNTL_1 0x2bb8 +#define VCO_MODE(x) ((x) << 0) +#define VCO_MODE_MASK (3 << 0) +#define CLKFRAC(x) ((x) << 4) +#define CLKFRAC_MASK (0xfff << 4) +#define CLKF(x) ((x) << 16) +#define CLKF_MASK (0xfff << 16) +#define MPLL_FUNC_CNTL_2 0x2bbc +#define MPLL_AD_FUNC_CNTL 0x2bc0 +#define YCLK_POST_DIV(x) ((x) << 0) +#define YCLK_POST_DIV_MASK (7 << 0) +#define MPLL_DQ_FUNC_CNTL 0x2bc4 +#define YCLK_SEL(x) ((x) << 4) +#define YCLK_SEL_MASK (1 << 4) + +#define MPLL_SS1 0x2bcc +#define CLKV(x) ((x) << 0) +#define CLKV_MASK (0x3ffffff << 0) +#define MPLL_SS2 0x2bd0 +#define CLKS(x) ((x) << 0) +#define CLKS_MASK (0xfff << 0) + +#define HDP_HOST_PATH_CNTL 0x2C00 +#define CLOCK_GATING_DIS (1 << 23) +#define HDP_NONSURFACE_BASE 0x2C04 +#define HDP_NONSURFACE_INFO 0x2C08 +#define HDP_NONSURFACE_SIZE 0x2C0C + +#define HDP_ADDR_CONFIG 0x2F48 +#define HDP_MISC_CNTL 0x2F4C +#define HDP_FLUSH_INVALIDATE_CACHE (1 << 0) +#define HDP_MEM_POWER_LS 0x2F50 +#define HDP_LS_ENABLE (1 << 0) + +#define ATC_MISC_CG 0x3350 + +#define GMCON_RENG_EXECUTE 0x3508 +#define RENG_EXECUTE_ON_PWR_UP (1 << 0) +#define GMCON_MISC 0x350c +#define RENG_EXECUTE_ON_REG_UPDATE (1 << 11) +#define STCTRL_STUTTER_EN (1 << 16) + +#define GMCON_PGFSM_CONFIG 0x3538 +#define GMCON_PGFSM_WRITE 0x353c +#define GMCON_PGFSM_READ 0x3540 +#define GMCON_MISC3 0x3544 + +#define MC_SEQ_CNTL_3 0x3600 +# define CAC_EN (1 << 31) +#define MC_SEQ_G5PDX_CTRL 0x3604 +#define MC_SEQ_G5PDX_CTRL_LP 0x3608 +#define MC_SEQ_G5PDX_CMD0 0x360c +#define MC_SEQ_G5PDX_CMD0_LP 0x3610 +#define MC_SEQ_G5PDX_CMD1 0x3614 +#define MC_SEQ_G5PDX_CMD1_LP 0x3618 + +#define MC_SEQ_PMG_DVS_CTL 0x3628 +#define MC_SEQ_PMG_DVS_CTL_LP 0x362c +#define MC_SEQ_PMG_DVS_CMD 0x3630 +#define MC_SEQ_PMG_DVS_CMD_LP 0x3634 +#define MC_SEQ_DLL_STBY 0x3638 +#define MC_SEQ_DLL_STBY_LP 0x363c + +#define IH_RB_CNTL 0x3e00 +# define IH_RB_ENABLE (1 << 0) +# define IH_RB_SIZE(x) ((x) << 1) /* log2 */ +# define IH_RB_FULL_DRAIN_ENABLE (1 << 6) +# define IH_WPTR_WRITEBACK_ENABLE (1 << 8) +# define IH_WPTR_WRITEBACK_TIMER(x) ((x) << 9) /* log2 */ +# define IH_WPTR_OVERFLOW_ENABLE (1 << 16) +# define IH_WPTR_OVERFLOW_CLEAR (1 << 31) +#define IH_RB_BASE 0x3e04 +#define IH_RB_RPTR 0x3e08 +#define IH_RB_WPTR 0x3e0c +# define RB_OVERFLOW (1 << 0) +# define WPTR_OFFSET_MASK 0x3fffc +#define IH_RB_WPTR_ADDR_HI 0x3e10 +#define IH_RB_WPTR_ADDR_LO 0x3e14 +#define IH_CNTL 0x3e18 +# define ENABLE_INTR (1 << 0) +# define IH_MC_SWAP(x) ((x) << 1) +# define IH_MC_SWAP_NONE 0 +# define IH_MC_SWAP_16BIT 1 +# define IH_MC_SWAP_32BIT 2 +# define IH_MC_SWAP_64BIT 3 +# define RPTR_REARM (1 << 4) +# define MC_WRREQ_CREDIT(x) ((x) << 15) +# define MC_WR_CLEAN_CNT(x) ((x) << 20) +# define MC_VMID(x) ((x) << 25) + +#define BIF_LNCNT_RESET 0x5220 +# define RESET_LNCNT_EN (1 << 0) + +#define CONFIG_MEMSIZE 0x5428 + +#define INTERRUPT_CNTL 0x5468 +# define IH_DUMMY_RD_OVERRIDE (1 << 0) +# define IH_DUMMY_RD_EN (1 << 1) +# define IH_REQ_NONSNOOP_EN (1 << 3) +# define GEN_IH_INT_EN (1 << 8) +#define INTERRUPT_CNTL2 0x546c + +#define HDP_MEM_COHERENCY_FLUSH_CNTL 0x5480 + +#define BIF_FB_EN 0x5490 +#define FB_READ_EN (1 << 0) +#define FB_WRITE_EN (1 << 1) + +#define HDP_REG_COHERENCY_FLUSH_CNTL 0x54A0 + +#define GPU_HDP_FLUSH_REQ 0x54DC +#define GPU_HDP_FLUSH_DONE 0x54E0 +#define CP0 (1 << 0) +#define CP1 (1 << 1) +#define CP2 (1 << 2) +#define CP3 (1 << 3) +#define CP4 (1 << 4) +#define CP5 (1 << 5) +#define CP6 (1 << 6) +#define CP7 (1 << 7) +#define CP8 (1 << 8) +#define CP9 (1 << 9) +#define SDMA0 (1 << 10) +#define SDMA1 (1 << 11) + +/* 0x6b04, 0x7704, 0x10304, 0x10f04, 0x11b04, 0x12704 */ +#define LB_MEMORY_CTRL 0x6b04 +#define LB_MEMORY_SIZE(x) ((x) << 0) +#define LB_MEMORY_CONFIG(x) ((x) << 20) + +#define DPG_WATERMARK_MASK_CONTROL 0x6cc8 +# define LATENCY_WATERMARK_MASK(x) ((x) << 8) +#define DPG_PIPE_LATENCY_CONTROL 0x6ccc +# define LATENCY_LOW_WATERMARK(x) ((x) << 0) +# define LATENCY_HIGH_WATERMARK(x) ((x) << 16) + +/* 0x6b24, 0x7724, 0x10324, 0x10f24, 0x11b24, 0x12724 */ +#define LB_VLINE_STATUS 0x6b24 +# define VLINE_OCCURRED (1 << 0) +# define VLINE_ACK (1 << 4) +# define VLINE_STAT (1 << 12) +# define VLINE_INTERRUPT (1 << 16) +# define VLINE_INTERRUPT_TYPE (1 << 17) +/* 0x6b2c, 0x772c, 0x1032c, 0x10f2c, 0x11b2c, 0x1272c */ +#define LB_VBLANK_STATUS 0x6b2c +# define VBLANK_OCCURRED (1 << 0) +# define VBLANK_ACK (1 << 4) +# define VBLANK_STAT (1 << 12) +# define VBLANK_INTERRUPT (1 << 16) +# define VBLANK_INTERRUPT_TYPE (1 << 17) + +/* 0x6b20, 0x7720, 0x10320, 0x10f20, 0x11b20, 0x12720 */ +#define LB_INTERRUPT_MASK 0x6b20 +# define VBLANK_INTERRUPT_MASK (1 << 0) +# define VLINE_INTERRUPT_MASK (1 << 4) +# define VLINE2_INTERRUPT_MASK (1 << 8) + +#define DISP_INTERRUPT_STATUS 0x60f4 +# define LB_D1_VLINE_INTERRUPT (1 << 2) +# define LB_D1_VBLANK_INTERRUPT (1 << 3) +# define DC_HPD1_INTERRUPT (1 << 17) +# define DC_HPD1_RX_INTERRUPT (1 << 18) +# define DACA_AUTODETECT_INTERRUPT (1 << 22) +# define DACB_AUTODETECT_INTERRUPT (1 << 23) +# define DC_I2C_SW_DONE_INTERRUPT (1 << 24) +# define DC_I2C_HW_DONE_INTERRUPT (1 << 25) +#define DISP_INTERRUPT_STATUS_CONTINUE 0x60f8 +# define LB_D2_VLINE_INTERRUPT (1 << 2) +# define LB_D2_VBLANK_INTERRUPT (1 << 3) +# define DC_HPD2_INTERRUPT (1 << 17) +# define DC_HPD2_RX_INTERRUPT (1 << 18) +# define DISP_TIMER_INTERRUPT (1 << 24) +#define DISP_INTERRUPT_STATUS_CONTINUE2 0x60fc +# define LB_D3_VLINE_INTERRUPT (1 << 2) +# define LB_D3_VBLANK_INTERRUPT (1 << 3) +# define DC_HPD3_INTERRUPT (1 << 17) +# define DC_HPD3_RX_INTERRUPT (1 << 18) +#define DISP_INTERRUPT_STATUS_CONTINUE3 0x6100 +# define LB_D4_VLINE_INTERRUPT (1 << 2) +# define LB_D4_VBLANK_INTERRUPT (1 << 3) +# define DC_HPD4_INTERRUPT (1 << 17) +# define DC_HPD4_RX_INTERRUPT (1 << 18) +#define DISP_INTERRUPT_STATUS_CONTINUE4 0x614c +# define LB_D5_VLINE_INTERRUPT (1 << 2) +# define LB_D5_VBLANK_INTERRUPT (1 << 3) +# define DC_HPD5_INTERRUPT (1 << 17) +# define DC_HPD5_RX_INTERRUPT (1 << 18) +#define DISP_INTERRUPT_STATUS_CONTINUE5 0x6150 +# define LB_D6_VLINE_INTERRUPT (1 << 2) +# define LB_D6_VBLANK_INTERRUPT (1 << 3) +# define DC_HPD6_INTERRUPT (1 << 17) +# define DC_HPD6_RX_INTERRUPT (1 << 18) +#define DISP_INTERRUPT_STATUS_CONTINUE6 0x6780 + +/* 0x6858, 0x7458, 0x10058, 0x10c58, 0x11858, 0x12458 */ +#define GRPH_INT_STATUS 0x6858 +# define GRPH_PFLIP_INT_OCCURRED (1 << 0) +# define GRPH_PFLIP_INT_CLEAR (1 << 8) +/* 0x685c, 0x745c, 0x1005c, 0x10c5c, 0x1185c, 0x1245c */ +#define GRPH_INT_CONTROL 0x685c +# define GRPH_PFLIP_INT_MASK (1 << 0) +# define GRPH_PFLIP_INT_TYPE (1 << 8) + +#define DAC_AUTODETECT_INT_CONTROL 0x67c8 + +#define DC_HPD1_INT_STATUS 0x601c +#define DC_HPD2_INT_STATUS 0x6028 +#define DC_HPD3_INT_STATUS 0x6034 +#define DC_HPD4_INT_STATUS 0x6040 +#define DC_HPD5_INT_STATUS 0x604c +#define DC_HPD6_INT_STATUS 0x6058 +# define DC_HPDx_INT_STATUS (1 << 0) +# define DC_HPDx_SENSE (1 << 1) +# define DC_HPDx_SENSE_DELAYED (1 << 4) +# define DC_HPDx_RX_INT_STATUS (1 << 8) + +#define DC_HPD1_INT_CONTROL 0x6020 +#define DC_HPD2_INT_CONTROL 0x602c +#define DC_HPD3_INT_CONTROL 0x6038 +#define DC_HPD4_INT_CONTROL 0x6044 +#define DC_HPD5_INT_CONTROL 0x6050 +#define DC_HPD6_INT_CONTROL 0x605c +# define DC_HPDx_INT_ACK (1 << 0) +# define DC_HPDx_INT_POLARITY (1 << 8) +# define DC_HPDx_INT_EN (1 << 16) +# define DC_HPDx_RX_INT_ACK (1 << 20) +# define DC_HPDx_RX_INT_EN (1 << 24) + +#define DC_HPD1_CONTROL 0x6024 +#define DC_HPD2_CONTROL 0x6030 +#define DC_HPD3_CONTROL 0x603c +#define DC_HPD4_CONTROL 0x6048 +#define DC_HPD5_CONTROL 0x6054 +#define DC_HPD6_CONTROL 0x6060 +# define DC_HPDx_CONNECTION_TIMER(x) ((x) << 0) +# define DC_HPDx_RX_INT_TIMER(x) ((x) << 16) +# define DC_HPDx_EN (1 << 28) + +#define DPG_PIPE_STUTTER_CONTROL 0x6cd4 +# define STUTTER_ENABLE (1 << 0) + +/* DCE8 FMT blocks */ +#define FMT_DYNAMIC_EXP_CNTL 0x6fb4 +# define FMT_DYNAMIC_EXP_EN (1 << 0) +# define FMT_DYNAMIC_EXP_MODE (1 << 4) + /* 0 = 10bit -> 12bit, 1 = 8bit -> 12bit */ +#define FMT_CONTROL 0x6fb8 +# define FMT_PIXEL_ENCODING (1 << 16) + /* 0 = RGB 4:4:4 or YCbCr 4:4:4, 1 = YCbCr 4:2:2 */ +#define FMT_BIT_DEPTH_CONTROL 0x6fc8 +# define FMT_TRUNCATE_EN (1 << 0) +# define FMT_TRUNCATE_MODE (1 << 1) +# define FMT_TRUNCATE_DEPTH(x) ((x) << 4) /* 0 - 18bpp, 1 - 24bpp, 2 - 30bpp */ +# define FMT_SPATIAL_DITHER_EN (1 << 8) +# define FMT_SPATIAL_DITHER_MODE(x) ((x) << 9) +# define FMT_SPATIAL_DITHER_DEPTH(x) ((x) << 11) /* 0 - 18bpp, 1 - 24bpp, 2 - 30bpp */ +# define FMT_FRAME_RANDOM_ENABLE (1 << 13) +# define FMT_RGB_RANDOM_ENABLE (1 << 14) +# define FMT_HIGHPASS_RANDOM_ENABLE (1 << 15) +# define FMT_TEMPORAL_DITHER_EN (1 << 16) +# define FMT_TEMPORAL_DITHER_DEPTH(x) ((x) << 17) /* 0 - 18bpp, 1 - 24bpp, 2 - 30bpp */ +# define FMT_TEMPORAL_DITHER_OFFSET(x) ((x) << 21) +# define FMT_TEMPORAL_LEVEL (1 << 24) +# define FMT_TEMPORAL_DITHER_RESET (1 << 25) +# define FMT_25FRC_SEL(x) ((x) << 26) +# define FMT_50FRC_SEL(x) ((x) << 28) +# define FMT_75FRC_SEL(x) ((x) << 30) +#define FMT_CLAMP_CONTROL 0x6fe4 +# define FMT_CLAMP_DATA_EN (1 << 0) +# define FMT_CLAMP_COLOR_FORMAT(x) ((x) << 16) +# define FMT_CLAMP_6BPC 0 +# define FMT_CLAMP_8BPC 1 +# define FMT_CLAMP_10BPC 2 + +#define GRBM_CNTL 0x8000 +#define GRBM_READ_TIMEOUT(x) ((x) << 0) + +#define GRBM_STATUS2 0x8008 +#define ME0PIPE1_CMDFIFO_AVAIL_MASK 0x0000000F +#define ME0PIPE1_CF_RQ_PENDING (1 << 4) +#define ME0PIPE1_PF_RQ_PENDING (1 << 5) +#define ME1PIPE0_RQ_PENDING (1 << 6) +#define ME1PIPE1_RQ_PENDING (1 << 7) +#define ME1PIPE2_RQ_PENDING (1 << 8) +#define ME1PIPE3_RQ_PENDING (1 << 9) +#define ME2PIPE0_RQ_PENDING (1 << 10) +#define ME2PIPE1_RQ_PENDING (1 << 11) +#define ME2PIPE2_RQ_PENDING (1 << 12) +#define ME2PIPE3_RQ_PENDING (1 << 13) +#define RLC_RQ_PENDING (1 << 14) +#define RLC_BUSY (1 << 24) +#define TC_BUSY (1 << 25) +#define CPF_BUSY (1 << 28) +#define CPC_BUSY (1 << 29) +#define CPG_BUSY (1 << 30) + +#define GRBM_STATUS 0x8010 +#define ME0PIPE0_CMDFIFO_AVAIL_MASK 0x0000000F +#define SRBM_RQ_PENDING (1 << 5) +#define ME0PIPE0_CF_RQ_PENDING (1 << 7) +#define ME0PIPE0_PF_RQ_PENDING (1 << 8) +#define GDS_DMA_RQ_PENDING (1 << 9) +#define DB_CLEAN (1 << 12) +#define CB_CLEAN (1 << 13) +#define TA_BUSY (1 << 14) +#define GDS_BUSY (1 << 15) +#define WD_BUSY_NO_DMA (1 << 16) +#define VGT_BUSY (1 << 17) +#define IA_BUSY_NO_DMA (1 << 18) +#define IA_BUSY (1 << 19) +#define SX_BUSY (1 << 20) +#define WD_BUSY (1 << 21) +#define SPI_BUSY (1 << 22) +#define BCI_BUSY (1 << 23) +#define SC_BUSY (1 << 24) +#define PA_BUSY (1 << 25) +#define DB_BUSY (1 << 26) +#define CP_COHERENCY_BUSY (1 << 28) +#define CP_BUSY (1 << 29) +#define CB_BUSY (1 << 30) +#define GUI_ACTIVE (1 << 31) +#define GRBM_STATUS_SE0 0x8014 +#define GRBM_STATUS_SE1 0x8018 +#define GRBM_STATUS_SE2 0x8038 +#define GRBM_STATUS_SE3 0x803C +#define SE_DB_CLEAN (1 << 1) +#define SE_CB_CLEAN (1 << 2) +#define SE_BCI_BUSY (1 << 22) +#define SE_VGT_BUSY (1 << 23) +#define SE_PA_BUSY (1 << 24) +#define SE_TA_BUSY (1 << 25) +#define SE_SX_BUSY (1 << 26) +#define SE_SPI_BUSY (1 << 27) +#define SE_SC_BUSY (1 << 29) +#define SE_DB_BUSY (1 << 30) +#define SE_CB_BUSY (1 << 31) + +#define GRBM_SOFT_RESET 0x8020 +#define SOFT_RESET_CP (1 << 0) /* All CP blocks */ +#define SOFT_RESET_RLC (1 << 2) /* RLC */ +#define SOFT_RESET_GFX (1 << 16) /* GFX */ +#define SOFT_RESET_CPF (1 << 17) /* CP fetcher shared by gfx and compute */ +#define SOFT_RESET_CPC (1 << 18) /* CP Compute (MEC1/2) */ +#define SOFT_RESET_CPG (1 << 19) /* CP GFX (PFP, ME, CE) */ + +#define GRBM_INT_CNTL 0x8060 +# define RDERR_INT_ENABLE (1 << 0) +# define GUI_IDLE_INT_ENABLE (1 << 19) + +#define CP_CPC_STATUS 0x8210 +#define CP_CPC_BUSY_STAT 0x8214 +#define CP_CPC_STALLED_STAT1 0x8218 +#define CP_CPF_STATUS 0x821c +#define CP_CPF_BUSY_STAT 0x8220 +#define CP_CPF_STALLED_STAT1 0x8224 + +#define CP_MEC_CNTL 0x8234 +#define MEC_ME2_HALT (1 << 28) +#define MEC_ME1_HALT (1 << 30) + +#define CP_MEC_CNTL 0x8234 +#define MEC_ME2_HALT (1 << 28) +#define MEC_ME1_HALT (1 << 30) + +#define CP_STALLED_STAT3 0x8670 +#define CP_STALLED_STAT1 0x8674 +#define CP_STALLED_STAT2 0x8678 + +#define CP_STAT 0x8680 + +#define CP_ME_CNTL 0x86D8 +#define CP_CE_HALT (1 << 24) +#define CP_PFP_HALT (1 << 26) +#define CP_ME_HALT (1 << 28) + +#define CP_RB0_RPTR 0x8700 +#define CP_RB_WPTR_DELAY 0x8704 +#define CP_RB_WPTR_POLL_CNTL 0x8708 +#define IDLE_POLL_COUNT(x) ((x) << 16) +#define IDLE_POLL_COUNT_MASK (0xffff << 16) + +#define CP_MEQ_THRESHOLDS 0x8764 +#define MEQ1_START(x) ((x) << 0) +#define MEQ2_START(x) ((x) << 8) + +#define VGT_VTX_VECT_EJECT_REG 0x88B0 + +#define VGT_CACHE_INVALIDATION 0x88C4 +#define CACHE_INVALIDATION(x) ((x) << 0) +#define VC_ONLY 0 +#define TC_ONLY 1 +#define VC_AND_TC 2 +#define AUTO_INVLD_EN(x) ((x) << 6) +#define NO_AUTO 0 +#define ES_AUTO 1 +#define GS_AUTO 2 +#define ES_AND_GS_AUTO 3 + +#define VGT_GS_VERTEX_REUSE 0x88D4 + +#define CC_GC_SHADER_ARRAY_CONFIG 0x89bc +#define INACTIVE_CUS_MASK 0xFFFF0000 +#define INACTIVE_CUS_SHIFT 16 +#define GC_USER_SHADER_ARRAY_CONFIG 0x89c0 + +#define PA_CL_ENHANCE 0x8A14 +#define CLIP_VTX_REORDER_ENA (1 << 0) +#define NUM_CLIP_SEQ(x) ((x) << 1) + +#define PA_SC_FORCE_EOV_MAX_CNTS 0x8B24 +#define FORCE_EOV_MAX_CLK_CNT(x) ((x) << 0) +#define FORCE_EOV_MAX_REZ_CNT(x) ((x) << 16) + +#define PA_SC_FIFO_SIZE 0x8BCC +#define SC_FRONTEND_PRIM_FIFO_SIZE(x) ((x) << 0) +#define SC_BACKEND_PRIM_FIFO_SIZE(x) ((x) << 6) +#define SC_HIZ_TILE_FIFO_SIZE(x) ((x) << 15) +#define SC_EARLYZ_TILE_FIFO_SIZE(x) ((x) << 23) + +#define PA_SC_ENHANCE 0x8BF0 +#define ENABLE_PA_SC_OUT_OF_ORDER (1 << 0) +#define DISABLE_PA_SC_GUIDANCE (1 << 13) + +#define SQ_CONFIG 0x8C00 + +#define SH_MEM_BASES 0x8C28 +/* if PTR32, these are the bases for scratch and lds */ +#define PRIVATE_BASE(x) ((x) << 0) /* scratch */ +#define SHARED_BASE(x) ((x) << 16) /* LDS */ +#define SH_MEM_APE1_BASE 0x8C2C +/* if PTR32, this is the base location of GPUVM */ +#define SH_MEM_APE1_LIMIT 0x8C30 +/* if PTR32, this is the upper limit of GPUVM */ +#define SH_MEM_CONFIG 0x8C34 +#define PTR32 (1 << 0) +#define ALIGNMENT_MODE(x) ((x) << 2) +#define SH_MEM_ALIGNMENT_MODE_DWORD 0 +#define SH_MEM_ALIGNMENT_MODE_DWORD_STRICT 1 +#define SH_MEM_ALIGNMENT_MODE_STRICT 2 +#define SH_MEM_ALIGNMENT_MODE_UNALIGNED 3 +#define DEFAULT_MTYPE(x) ((x) << 4) +#define APE1_MTYPE(x) ((x) << 7) +/* valid for both DEFAULT_MTYPE and APE1_MTYPE */ +#define MTYPE_CACHED 0 +#define MTYPE_NONCACHED 3 + +#define SX_DEBUG_1 0x9060 + +#define SPI_CONFIG_CNTL 0x9100 + +#define SPI_CONFIG_CNTL_1 0x913C +#define VTX_DONE_DELAY(x) ((x) << 0) +#define INTERP_ONE_PRIM_PER_ROW (1 << 4) + +#define TA_CNTL_AUX 0x9508 + +#define DB_DEBUG 0x9830 +#define DB_DEBUG2 0x9834 +#define DB_DEBUG3 0x9838 + +#define CC_RB_BACKEND_DISABLE 0x98F4 +#define BACKEND_DISABLE(x) ((x) << 16) +#define GB_ADDR_CONFIG 0x98F8 +#define NUM_PIPES(x) ((x) << 0) +#define NUM_PIPES_MASK 0x00000007 +#define NUM_PIPES_SHIFT 0 +#define PIPE_INTERLEAVE_SIZE(x) ((x) << 4) +#define PIPE_INTERLEAVE_SIZE_MASK 0x00000070 +#define PIPE_INTERLEAVE_SIZE_SHIFT 4 +#define NUM_SHADER_ENGINES(x) ((x) << 12) +#define NUM_SHADER_ENGINES_MASK 0x00003000 +#define NUM_SHADER_ENGINES_SHIFT 12 +#define SHADER_ENGINE_TILE_SIZE(x) ((x) << 16) +#define SHADER_ENGINE_TILE_SIZE_MASK 0x00070000 +#define SHADER_ENGINE_TILE_SIZE_SHIFT 16 +#define ROW_SIZE(x) ((x) << 28) +#define ROW_SIZE_MASK 0x30000000 +#define ROW_SIZE_SHIFT 28 + +#define GB_TILE_MODE0 0x9910 +# define ARRAY_MODE(x) ((x) << 2) +# define ARRAY_LINEAR_GENERAL 0 +# define ARRAY_LINEAR_ALIGNED 1 +# define ARRAY_1D_TILED_THIN1 2 +# define ARRAY_2D_TILED_THIN1 4 +# define ARRAY_PRT_TILED_THIN1 5 +# define ARRAY_PRT_2D_TILED_THIN1 6 +# define PIPE_CONFIG(x) ((x) << 6) +# define ADDR_SURF_P2 0 +# define ADDR_SURF_P4_8x16 4 +# define ADDR_SURF_P4_16x16 5 +# define ADDR_SURF_P4_16x32 6 +# define ADDR_SURF_P4_32x32 7 +# define ADDR_SURF_P8_16x16_8x16 8 +# define ADDR_SURF_P8_16x32_8x16 9 +# define ADDR_SURF_P8_32x32_8x16 10 +# define ADDR_SURF_P8_16x32_16x16 11 +# define ADDR_SURF_P8_32x32_16x16 12 +# define ADDR_SURF_P8_32x32_16x32 13 +# define ADDR_SURF_P8_32x64_32x32 14 +# define ADDR_SURF_P16_32x32_8x16 16 +# define ADDR_SURF_P16_32x32_16x16 17 +# define TILE_SPLIT(x) ((x) << 11) +# define ADDR_SURF_TILE_SPLIT_64B 0 +# define ADDR_SURF_TILE_SPLIT_128B 1 +# define ADDR_SURF_TILE_SPLIT_256B 2 +# define ADDR_SURF_TILE_SPLIT_512B 3 +# define ADDR_SURF_TILE_SPLIT_1KB 4 +# define ADDR_SURF_TILE_SPLIT_2KB 5 +# define ADDR_SURF_TILE_SPLIT_4KB 6 +# define MICRO_TILE_MODE_NEW(x) ((x) << 22) +# define ADDR_SURF_DISPLAY_MICRO_TILING 0 +# define ADDR_SURF_THIN_MICRO_TILING 1 +# define ADDR_SURF_DEPTH_MICRO_TILING 2 +# define ADDR_SURF_ROTATED_MICRO_TILING 3 +# define SAMPLE_SPLIT(x) ((x) << 25) +# define ADDR_SURF_SAMPLE_SPLIT_1 0 +# define ADDR_SURF_SAMPLE_SPLIT_2 1 +# define ADDR_SURF_SAMPLE_SPLIT_4 2 +# define ADDR_SURF_SAMPLE_SPLIT_8 3 + +#define GB_MACROTILE_MODE0 0x9990 +# define BANK_WIDTH(x) ((x) << 0) +# define ADDR_SURF_BANK_WIDTH_1 0 +# define ADDR_SURF_BANK_WIDTH_2 1 +# define ADDR_SURF_BANK_WIDTH_4 2 +# define ADDR_SURF_BANK_WIDTH_8 3 +# define BANK_HEIGHT(x) ((x) << 2) +# define ADDR_SURF_BANK_HEIGHT_1 0 +# define ADDR_SURF_BANK_HEIGHT_2 1 +# define ADDR_SURF_BANK_HEIGHT_4 2 +# define ADDR_SURF_BANK_HEIGHT_8 3 +# define MACRO_TILE_ASPECT(x) ((x) << 4) +# define ADDR_SURF_MACRO_ASPECT_1 0 +# define ADDR_SURF_MACRO_ASPECT_2 1 +# define ADDR_SURF_MACRO_ASPECT_4 2 +# define ADDR_SURF_MACRO_ASPECT_8 3 +# define NUM_BANKS(x) ((x) << 6) +# define ADDR_SURF_2_BANK 0 +# define ADDR_SURF_4_BANK 1 +# define ADDR_SURF_8_BANK 2 +# define ADDR_SURF_16_BANK 3 + +#define CB_HW_CONTROL 0x9A10 + +#define GC_USER_RB_BACKEND_DISABLE 0x9B7C +#define BACKEND_DISABLE_MASK 0x00FF0000 +#define BACKEND_DISABLE_SHIFT 16 + +#define TCP_CHAN_STEER_LO 0xac0c +#define TCP_CHAN_STEER_HI 0xac10 + +#define TC_CFG_L1_LOAD_POLICY0 0xAC68 +#define TC_CFG_L1_LOAD_POLICY1 0xAC6C +#define TC_CFG_L1_STORE_POLICY 0xAC70 +#define TC_CFG_L2_LOAD_POLICY0 0xAC74 +#define TC_CFG_L2_LOAD_POLICY1 0xAC78 +#define TC_CFG_L2_STORE_POLICY0 0xAC7C +#define TC_CFG_L2_STORE_POLICY1 0xAC80 +#define TC_CFG_L2_ATOMIC_POLICY 0xAC84 +#define TC_CFG_L1_VOLATILE 0xAC88 +#define TC_CFG_L2_VOLATILE 0xAC8C + +#define CP_RB0_BASE 0xC100 +#define CP_RB0_CNTL 0xC104 +#define RB_BUFSZ(x) ((x) << 0) +#define RB_BLKSZ(x) ((x) << 8) +#define BUF_SWAP_32BIT (2 << 16) +#define RB_NO_UPDATE (1 << 27) +#define RB_RPTR_WR_ENA (1 << 31) + +#define CP_RB0_RPTR_ADDR 0xC10C +#define RB_RPTR_SWAP_32BIT (2 << 0) +#define CP_RB0_RPTR_ADDR_HI 0xC110 +#define CP_RB0_WPTR 0xC114 + +#define CP_DEVICE_ID 0xC12C +#define CP_ENDIAN_SWAP 0xC140 +#define CP_RB_VMID 0xC144 + +#define CP_PFP_UCODE_ADDR 0xC150 +#define CP_PFP_UCODE_DATA 0xC154 +#define CP_ME_RAM_RADDR 0xC158 +#define CP_ME_RAM_WADDR 0xC15C +#define CP_ME_RAM_DATA 0xC160 + +#define CP_CE_UCODE_ADDR 0xC168 +#define CP_CE_UCODE_DATA 0xC16C +#define CP_MEC_ME1_UCODE_ADDR 0xC170 +#define CP_MEC_ME1_UCODE_DATA 0xC174 +#define CP_MEC_ME2_UCODE_ADDR 0xC178 +#define CP_MEC_ME2_UCODE_DATA 0xC17C + +#define CP_INT_CNTL_RING0 0xC1A8 +# define CNTX_BUSY_INT_ENABLE (1 << 19) +# define CNTX_EMPTY_INT_ENABLE (1 << 20) +# define PRIV_INSTR_INT_ENABLE (1 << 22) +# define PRIV_REG_INT_ENABLE (1 << 23) +# define OPCODE_ERROR_INT_ENABLE (1 << 24) +# define TIME_STAMP_INT_ENABLE (1 << 26) +# define CP_RINGID2_INT_ENABLE (1 << 29) +# define CP_RINGID1_INT_ENABLE (1 << 30) +# define CP_RINGID0_INT_ENABLE (1 << 31) + +#define CP_INT_STATUS_RING0 0xC1B4 +# define PRIV_INSTR_INT_STAT (1 << 22) +# define PRIV_REG_INT_STAT (1 << 23) +# define TIME_STAMP_INT_STAT (1 << 26) +# define CP_RINGID2_INT_STAT (1 << 29) +# define CP_RINGID1_INT_STAT (1 << 30) +# define CP_RINGID0_INT_STAT (1 << 31) + +#define CP_MEM_SLP_CNTL 0xC1E4 +# define CP_MEM_LS_EN (1 << 0) + +#define CP_CPF_DEBUG 0xC200 + +#define CP_PQ_WPTR_POLL_CNTL 0xC20C +#define WPTR_POLL_EN (1 << 31) + +#define CP_ME1_PIPE0_INT_CNTL 0xC214 +#define CP_ME1_PIPE1_INT_CNTL 0xC218 +#define CP_ME1_PIPE2_INT_CNTL 0xC21C +#define CP_ME1_PIPE3_INT_CNTL 0xC220 +#define CP_ME2_PIPE0_INT_CNTL 0xC224 +#define CP_ME2_PIPE1_INT_CNTL 0xC228 +#define CP_ME2_PIPE2_INT_CNTL 0xC22C +#define CP_ME2_PIPE3_INT_CNTL 0xC230 +# define DEQUEUE_REQUEST_INT_ENABLE (1 << 13) +# define WRM_POLL_TIMEOUT_INT_ENABLE (1 << 17) +# define PRIV_REG_INT_ENABLE (1 << 23) +# define TIME_STAMP_INT_ENABLE (1 << 26) +# define GENERIC2_INT_ENABLE (1 << 29) +# define GENERIC1_INT_ENABLE (1 << 30) +# define GENERIC0_INT_ENABLE (1 << 31) +#define CP_ME1_PIPE0_INT_STATUS 0xC214 +#define CP_ME1_PIPE1_INT_STATUS 0xC218 +#define CP_ME1_PIPE2_INT_STATUS 0xC21C +#define CP_ME1_PIPE3_INT_STATUS 0xC220 +#define CP_ME2_PIPE0_INT_STATUS 0xC224 +#define CP_ME2_PIPE1_INT_STATUS 0xC228 +#define CP_ME2_PIPE2_INT_STATUS 0xC22C +#define CP_ME2_PIPE3_INT_STATUS 0xC230 +# define DEQUEUE_REQUEST_INT_STATUS (1 << 13) +# define WRM_POLL_TIMEOUT_INT_STATUS (1 << 17) +# define PRIV_REG_INT_STATUS (1 << 23) +# define TIME_STAMP_INT_STATUS (1 << 26) +# define GENERIC2_INT_STATUS (1 << 29) +# define GENERIC1_INT_STATUS (1 << 30) +# define GENERIC0_INT_STATUS (1 << 31) + +#define CP_MAX_CONTEXT 0xC2B8 + +#define CP_RB0_BASE_HI 0xC2C4 + +#define RLC_CNTL 0xC300 +# define RLC_ENABLE (1 << 0) + +#define RLC_MC_CNTL 0xC30C + +#define RLC_MEM_SLP_CNTL 0xC318 +# define RLC_MEM_LS_EN (1 << 0) + +#define RLC_LB_CNTR_MAX 0xC348 + +#define RLC_LB_CNTL 0xC364 +# define LOAD_BALANCE_ENABLE (1 << 0) + +#define RLC_LB_CNTR_INIT 0xC36C + +#define RLC_SAVE_AND_RESTORE_BASE 0xC374 +#define RLC_DRIVER_DMA_STATUS 0xC378 /* dGPU */ +#define RLC_CP_TABLE_RESTORE 0xC378 /* APU */ +#define RLC_PG_DELAY_2 0xC37C + +#define RLC_GPM_UCODE_ADDR 0xC388 +#define RLC_GPM_UCODE_DATA 0xC38C +#define RLC_GPU_CLOCK_COUNT_LSB 0xC390 +#define RLC_GPU_CLOCK_COUNT_MSB 0xC394 +#define RLC_CAPTURE_GPU_CLOCK_COUNT 0xC398 +#define RLC_UCODE_CNTL 0xC39C + +#define RLC_GPM_STAT 0xC400 +# define RLC_GPM_BUSY (1 << 0) +# define GFX_POWER_STATUS (1 << 1) +# define GFX_CLOCK_STATUS (1 << 2) + +#define RLC_PG_CNTL 0xC40C +# define GFX_PG_ENABLE (1 << 0) +# define GFX_PG_SRC (1 << 1) +# define DYN_PER_CU_PG_ENABLE (1 << 2) +# define STATIC_PER_CU_PG_ENABLE (1 << 3) +# define DISABLE_GDS_PG (1 << 13) +# define DISABLE_CP_PG (1 << 15) +# define SMU_CLK_SLOWDOWN_ON_PU_ENABLE (1 << 17) +# define SMU_CLK_SLOWDOWN_ON_PD_ENABLE (1 << 18) + +#define RLC_CGTT_MGCG_OVERRIDE 0xC420 +#define RLC_CGCG_CGLS_CTRL 0xC424 +# define CGCG_EN (1 << 0) +# define CGLS_EN (1 << 1) + +#define RLC_PG_DELAY 0xC434 + +#define RLC_LB_INIT_CU_MASK 0xC43C + +#define RLC_LB_PARAMS 0xC444 + +#define RLC_PG_AO_CU_MASK 0xC44C + +#define RLC_MAX_PG_CU 0xC450 +# define MAX_PU_CU(x) ((x) << 0) +# define MAX_PU_CU_MASK (0xff << 0) +#define RLC_AUTO_PG_CTRL 0xC454 +# define AUTO_PG_EN (1 << 0) +# define GRBM_REG_SGIT(x) ((x) << 3) +# define GRBM_REG_SGIT_MASK (0xffff << 3) + +#define RLC_SERDES_WR_CU_MASTER_MASK 0xC474 +#define RLC_SERDES_WR_NONCU_MASTER_MASK 0xC478 +#define RLC_SERDES_WR_CTRL 0xC47C +#define BPM_ADDR(x) ((x) << 0) +#define BPM_ADDR_MASK (0xff << 0) +#define CGLS_ENABLE (1 << 16) +#define CGCG_OVERRIDE_0 (1 << 20) +#define MGCG_OVERRIDE_0 (1 << 22) +#define MGCG_OVERRIDE_1 (1 << 23) + +#define RLC_SERDES_CU_MASTER_BUSY 0xC484 +#define RLC_SERDES_NONCU_MASTER_BUSY 0xC488 +# define SE_MASTER_BUSY_MASK 0x0000ffff +# define GC_MASTER_BUSY (1 << 16) +# define TC0_MASTER_BUSY (1 << 17) +# define TC1_MASTER_BUSY (1 << 18) + +#define RLC_GPM_SCRATCH_ADDR 0xC4B0 +#define RLC_GPM_SCRATCH_DATA 0xC4B4 + +#define RLC_GPR_REG2 0xC4E8 +#define REQ 0x00000001 +#define MESSAGE(x) ((x) << 1) +#define MESSAGE_MASK 0x0000001e +#define MSG_ENTER_RLC_SAFE_MODE 1 +#define MSG_EXIT_RLC_SAFE_MODE 0 + +#define CP_HPD_EOP_BASE_ADDR 0xC904 +#define CP_HPD_EOP_BASE_ADDR_HI 0xC908 +#define CP_HPD_EOP_VMID 0xC90C +#define CP_HPD_EOP_CONTROL 0xC910 +#define EOP_SIZE(x) ((x) << 0) +#define EOP_SIZE_MASK (0x3f << 0) +#define CP_MQD_BASE_ADDR 0xC914 +#define CP_MQD_BASE_ADDR_HI 0xC918 +#define CP_HQD_ACTIVE 0xC91C +#define CP_HQD_VMID 0xC920 + +#define CP_HQD_PERSISTENT_STATE 0xC924u +#define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8) + +#define CP_HQD_PIPE_PRIORITY 0xC928u +#define CP_HQD_QUEUE_PRIORITY 0xC92Cu +#define CP_HQD_QUANTUM 0xC930u +#define QUANTUM_EN 1U +#define QUANTUM_SCALE_1MS (1U << 4) +#define QUANTUM_DURATION(x) ((x) << 8) + +#define CP_HQD_PQ_BASE 0xC934 +#define CP_HQD_PQ_BASE_HI 0xC938 +#define CP_HQD_PQ_RPTR 0xC93C +#define CP_HQD_PQ_RPTR_REPORT_ADDR 0xC940 +#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI 0xC944 +#define CP_HQD_PQ_WPTR_POLL_ADDR 0xC948 +#define CP_HQD_PQ_WPTR_POLL_ADDR_HI 0xC94C +#define CP_HQD_PQ_DOORBELL_CONTROL 0xC950 +#define DOORBELL_OFFSET(x) ((x) << 2) +#define DOORBELL_OFFSET_MASK (0x1fffff << 2) +#define DOORBELL_SOURCE (1 << 28) +#define DOORBELL_SCHD_HIT (1 << 29) +#define DOORBELL_EN (1 << 30) +#define DOORBELL_HIT (1 << 31) +#define CP_HQD_PQ_WPTR 0xC954 +#define CP_HQD_PQ_CONTROL 0xC958 +#define QUEUE_SIZE(x) ((x) << 0) +#define QUEUE_SIZE_MASK (0x3f << 0) +#define RPTR_BLOCK_SIZE(x) ((x) << 8) +#define RPTR_BLOCK_SIZE_MASK (0x3f << 8) +#define PQ_VOLATILE (1 << 26) +#define NO_UPDATE_RPTR (1 << 27) +#define UNORD_DISPATCH (1 << 28) +#define ROQ_PQ_IB_FLIP (1 << 29) +#define PRIV_STATE (1 << 30) +#define KMD_QUEUE (1 << 31) + +#define CP_HQD_IB_BASE_ADDR 0xC95Cu +#define CP_HQD_IB_BASE_ADDR_HI 0xC960u +#define CP_HQD_IB_RPTR 0xC964u +#define CP_HQD_IB_CONTROL 0xC968u +#define IB_ATC_EN (1U << 23) +#define DEFAULT_MIN_IB_AVAIL_SIZE (3U << 20) + +#define CP_HQD_DEQUEUE_REQUEST 0xC974 +#define DEQUEUE_REQUEST_DRAIN 1 +#define DEQUEUE_REQUEST_RESET 2 + +#define CP_MQD_CONTROL 0xC99C +#define MQD_VMID(x) ((x) << 0) +#define MQD_VMID_MASK (0xf << 0) + +#define CP_HQD_SEMA_CMD 0xC97Cu +#define CP_HQD_MSG_TYPE 0xC980u +#define CP_HQD_ATOMIC0_PREOP_LO 0xC984u +#define CP_HQD_ATOMIC0_PREOP_HI 0xC988u +#define CP_HQD_ATOMIC1_PREOP_LO 0xC98Cu +#define CP_HQD_ATOMIC1_PREOP_HI 0xC990u +#define CP_HQD_HQ_SCHEDULER0 0xC994u +#define CP_HQD_HQ_SCHEDULER1 0xC998u + +#define SH_STATIC_MEM_CONFIG 0x9604u + +#define DB_RENDER_CONTROL 0x28000 + +#define PA_SC_RASTER_CONFIG 0x28350 +# define RASTER_CONFIG_RB_MAP_0 0 +# define RASTER_CONFIG_RB_MAP_1 1 +# define RASTER_CONFIG_RB_MAP_2 2 +# define RASTER_CONFIG_RB_MAP_3 3 +#define PKR_MAP(x) ((x) << 8) + +#define VGT_EVENT_INITIATOR 0x28a90 +# define SAMPLE_STREAMOUTSTATS1 (1 << 0) +# define SAMPLE_STREAMOUTSTATS2 (2 << 0) +# define SAMPLE_STREAMOUTSTATS3 (3 << 0) +# define CACHE_FLUSH_TS (4 << 0) +# define CACHE_FLUSH (6 << 0) +# define CS_PARTIAL_FLUSH (7 << 0) +# define VGT_STREAMOUT_RESET (10 << 0) +# define END_OF_PIPE_INCR_DE (11 << 0) +# define END_OF_PIPE_IB_END (12 << 0) +# define RST_PIX_CNT (13 << 0) +# define VS_PARTIAL_FLUSH (15 << 0) +# define PS_PARTIAL_FLUSH (16 << 0) +# define CACHE_FLUSH_AND_INV_TS_EVENT (20 << 0) +# define ZPASS_DONE (21 << 0) +# define CACHE_FLUSH_AND_INV_EVENT (22 << 0) +# define PERFCOUNTER_START (23 << 0) +# define PERFCOUNTER_STOP (24 << 0) +# define PIPELINESTAT_START (25 << 0) +# define PIPELINESTAT_STOP (26 << 0) +# define PERFCOUNTER_SAMPLE (27 << 0) +# define SAMPLE_PIPELINESTAT (30 << 0) +# define SO_VGT_STREAMOUT_FLUSH (31 << 0) +# define SAMPLE_STREAMOUTSTATS (32 << 0) +# define RESET_VTX_CNT (33 << 0) +# define VGT_FLUSH (36 << 0) +# define BOTTOM_OF_PIPE_TS (40 << 0) +# define DB_CACHE_FLUSH_AND_INV (42 << 0) +# define FLUSH_AND_INV_DB_DATA_TS (43 << 0) +# define FLUSH_AND_INV_DB_META (44 << 0) +# define FLUSH_AND_INV_CB_DATA_TS (45 << 0) +# define FLUSH_AND_INV_CB_META (46 << 0) +# define CS_DONE (47 << 0) +# define PS_DONE (48 << 0) +# define FLUSH_AND_INV_CB_PIXEL_DATA (49 << 0) +# define THREAD_TRACE_START (51 << 0) +# define THREAD_TRACE_STOP (52 << 0) +# define THREAD_TRACE_FLUSH (54 << 0) +# define THREAD_TRACE_FINISH (55 << 0) +# define PIXEL_PIPE_STAT_CONTROL (56 << 0) +# define PIXEL_PIPE_STAT_DUMP (57 << 0) +# define PIXEL_PIPE_STAT_RESET (58 << 0) + +#define SCRATCH_REG0 0x30100 +#define SCRATCH_REG1 0x30104 +#define SCRATCH_REG2 0x30108 +#define SCRATCH_REG3 0x3010C +#define SCRATCH_REG4 0x30110 +#define SCRATCH_REG5 0x30114 +#define SCRATCH_REG6 0x30118 +#define SCRATCH_REG7 0x3011C + +#define SCRATCH_UMSK 0x30140 +#define SCRATCH_ADDR 0x30144 + +#define CP_SEM_WAIT_TIMER 0x301BC + +#define CP_SEM_INCOMPLETE_TIMER_CNTL 0x301C8 + +#define CP_WAIT_REG_MEM_TIMEOUT 0x301D0 + +#define GRBM_GFX_INDEX 0x30800 +#define INSTANCE_INDEX(x) ((x) << 0) +#define SH_INDEX(x) ((x) << 8) +#define SE_INDEX(x) ((x) << 16) +#define SH_BROADCAST_WRITES (1 << 29) +#define INSTANCE_BROADCAST_WRITES (1 << 30) +#define SE_BROADCAST_WRITES (1 << 31) + +#define VGT_ESGS_RING_SIZE 0x30900 +#define VGT_GSVS_RING_SIZE 0x30904 +#define VGT_PRIMITIVE_TYPE 0x30908 +#define VGT_INDEX_TYPE 0x3090C + +#define VGT_NUM_INDICES 0x30930 +#define VGT_NUM_INSTANCES 0x30934 +#define VGT_TF_RING_SIZE 0x30938 +#define VGT_HS_OFFCHIP_PARAM 0x3093C +#define VGT_TF_MEMORY_BASE 0x30940 + +#define PA_SU_LINE_STIPPLE_VALUE 0x30a00 +#define PA_SC_LINE_STIPPLE_STATE 0x30a04 + +#define SQC_CACHES 0x30d20 + +#define CP_PERFMON_CNTL 0x36020 + +#define CGTS_SM_CTRL_REG 0x3c000 +#define SM_MODE(x) ((x) << 17) +#define SM_MODE_MASK (0x7 << 17) +#define SM_MODE_ENABLE (1 << 20) +#define CGTS_OVERRIDE (1 << 21) +#define CGTS_LS_OVERRIDE (1 << 22) +#define ON_MONITOR_ADD_EN (1 << 23) +#define ON_MONITOR_ADD(x) ((x) << 24) +#define ON_MONITOR_ADD_MASK (0xff << 24) + +#define CGTS_TCC_DISABLE 0x3c00c +#define CGTS_USER_TCC_DISABLE 0x3c010 +#define TCC_DISABLE_MASK 0xFFFF0000 +#define TCC_DISABLE_SHIFT 16 + +#define CB_CGTT_SCLK_CTRL 0x3c2a0 + +/* + * PM4 + */ +#define PACKET_TYPE0 0 +#define PACKET_TYPE1 1 +#define PACKET_TYPE2 2 +#define PACKET_TYPE3 3 + +#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) +#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) +#define CP_PACKET0_GET_REG(h) (((h) & 0xFFFF) << 2) +#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) +#define PACKET0(reg, n) ((PACKET_TYPE0 << 30) | \ + (((reg) >> 2) & 0xFFFF) | \ + ((n) & 0x3FFF) << 16) +#define CP_PACKET2 0x80000000 +#define PACKET2_PAD_SHIFT 0 +#define PACKET2_PAD_MASK (0x3fffffff << 0) + +#define PACKET2(v) (CP_PACKET2 | REG_SET(PACKET2_PAD, (v))) + +#define PACKET3(op, n) ((PACKET_TYPE3 << 30) | \ + (((op) & 0xFF) << 8) | \ + ((n) & 0x3FFF) << 16) + +#define PACKET3_COMPUTE(op, n) (PACKET3(op, n) | 1 << 1) + +/* Packet 3 types */ +#define PACKET3_NOP 0x10 +#define PACKET3_SET_BASE 0x11 +#define PACKET3_BASE_INDEX(x) ((x) << 0) +#define CE_PARTITION_BASE 3 +#define PACKET3_CLEAR_STATE 0x12 +#define PACKET3_INDEX_BUFFER_SIZE 0x13 +#define PACKET3_DISPATCH_DIRECT 0x15 +#define PACKET3_DISPATCH_INDIRECT 0x16 +#define PACKET3_ATOMIC_GDS 0x1D +#define PACKET3_ATOMIC_MEM 0x1E +#define PACKET3_OCCLUSION_QUERY 0x1F +#define PACKET3_SET_PREDICATION 0x20 +#define PACKET3_REG_RMW 0x21 +#define PACKET3_COND_EXEC 0x22 +#define PACKET3_PRED_EXEC 0x23 +#define PACKET3_DRAW_INDIRECT 0x24 +#define PACKET3_DRAW_INDEX_INDIRECT 0x25 +#define PACKET3_INDEX_BASE 0x26 +#define PACKET3_DRAW_INDEX_2 0x27 +#define PACKET3_CONTEXT_CONTROL 0x28 +#define PACKET3_INDEX_TYPE 0x2A +#define PACKET3_DRAW_INDIRECT_MULTI 0x2C +#define PACKET3_DRAW_INDEX_AUTO 0x2D +#define PACKET3_NUM_INSTANCES 0x2F +#define PACKET3_DRAW_INDEX_MULTI_AUTO 0x30 +#define PACKET3_INDIRECT_BUFFER_CONST 0x33 +#define PACKET3_STRMOUT_BUFFER_UPDATE 0x34 +#define PACKET3_DRAW_INDEX_OFFSET_2 0x35 +#define PACKET3_DRAW_PREAMBLE 0x36 +#define PACKET3_WRITE_DATA 0x37 +#define WRITE_DATA_DST_SEL(x) ((x) << 8) + /* 0 - register + * 1 - memory (sync - via GRBM) + * 2 - gl2 + * 3 - gds + * 4 - reserved + * 5 - memory (async - direct) + */ +#define WR_ONE_ADDR (1 << 16) +#define WR_CONFIRM (1 << 20) +#define WRITE_DATA_CACHE_POLICY(x) ((x) << 25) + /* 0 - LRU + * 1 - Stream + */ +#define WRITE_DATA_ENGINE_SEL(x) ((x) << 30) + /* 0 - me + * 1 - pfp + * 2 - ce + */ +#define PACKET3_DRAW_INDEX_INDIRECT_MULTI 0x38 +#define PACKET3_MEM_SEMAPHORE 0x39 +# define PACKET3_SEM_USE_MAILBOX (0x1 << 16) +# define PACKET3_SEM_SEL_SIGNAL_TYPE (0x1 << 20) /* 0 = increment, 1 = write 1 */ +# define PACKET3_SEM_CLIENT_CODE ((x) << 24) /* 0 = CP, 1 = CB, 2 = DB */ +# define PACKET3_SEM_SEL_SIGNAL (0x6 << 29) +# define PACKET3_SEM_SEL_WAIT (0x7 << 29) +#define PACKET3_COPY_DW 0x3B +#define PACKET3_WAIT_REG_MEM 0x3C +#define WAIT_REG_MEM_FUNCTION(x) ((x) << 0) + /* 0 - always + * 1 - < + * 2 - <= + * 3 - == + * 4 - != + * 5 - >= + * 6 - > + */ +#define WAIT_REG_MEM_MEM_SPACE(x) ((x) << 4) + /* 0 - reg + * 1 - mem + */ +#define WAIT_REG_MEM_OPERATION(x) ((x) << 6) + /* 0 - wait_reg_mem + * 1 - wr_wait_wr_reg + */ +#define WAIT_REG_MEM_ENGINE(x) ((x) << 8) + /* 0 - me + * 1 - pfp + */ +#define PACKET3_INDIRECT_BUFFER 0x3F +#define INDIRECT_BUFFER_TCL2_VOLATILE (1 << 22) +#define INDIRECT_BUFFER_VALID (1 << 23) +#define INDIRECT_BUFFER_CACHE_POLICY(x) ((x) << 28) + /* 0 - LRU + * 1 - Stream + * 2 - Bypass + */ +#define PACKET3_COPY_DATA 0x40 +#define PACKET3_PFP_SYNC_ME 0x42 +#define PACKET3_SURFACE_SYNC 0x43 +# define PACKET3_DEST_BASE_0_ENA (1 << 0) +# define PACKET3_DEST_BASE_1_ENA (1 << 1) +# define PACKET3_CB0_DEST_BASE_ENA (1 << 6) +# define PACKET3_CB1_DEST_BASE_ENA (1 << 7) +# define PACKET3_CB2_DEST_BASE_ENA (1 << 8) +# define PACKET3_CB3_DEST_BASE_ENA (1 << 9) +# define PACKET3_CB4_DEST_BASE_ENA (1 << 10) +# define PACKET3_CB5_DEST_BASE_ENA (1 << 11) +# define PACKET3_CB6_DEST_BASE_ENA (1 << 12) +# define PACKET3_CB7_DEST_BASE_ENA (1 << 13) +# define PACKET3_DB_DEST_BASE_ENA (1 << 14) +# define PACKET3_TCL1_VOL_ACTION_ENA (1 << 15) +# define PACKET3_TC_VOL_ACTION_ENA (1 << 16) /* L2 */ +# define PACKET3_TC_WB_ACTION_ENA (1 << 18) /* L2 */ +# define PACKET3_DEST_BASE_2_ENA (1 << 19) +# define PACKET3_DEST_BASE_3_ENA (1 << 21) +# define PACKET3_TCL1_ACTION_ENA (1 << 22) +# define PACKET3_TC_ACTION_ENA (1 << 23) /* L2 */ +# define PACKET3_CB_ACTION_ENA (1 << 25) +# define PACKET3_DB_ACTION_ENA (1 << 26) +# define PACKET3_SH_KCACHE_ACTION_ENA (1 << 27) +# define PACKET3_SH_KCACHE_VOL_ACTION_ENA (1 << 28) +# define PACKET3_SH_ICACHE_ACTION_ENA (1 << 29) +#define PACKET3_COND_WRITE 0x45 +#define PACKET3_EVENT_WRITE 0x46 +#define EVENT_TYPE(x) ((x) << 0) +#define EVENT_INDEX(x) ((x) << 8) + /* 0 - any non-TS event + * 1 - ZPASS_DONE, PIXEL_PIPE_STAT_* + * 2 - SAMPLE_PIPELINESTAT + * 3 - SAMPLE_STREAMOUTSTAT* + * 4 - *S_PARTIAL_FLUSH + * 5 - EOP events + * 6 - EOS events + */ +#define PACKET3_EVENT_WRITE_EOP 0x47 +#define EOP_TCL1_VOL_ACTION_EN (1 << 12) +#define EOP_TC_VOL_ACTION_EN (1 << 13) /* L2 */ +#define EOP_TC_WB_ACTION_EN (1 << 15) /* L2 */ +#define EOP_TCL1_ACTION_EN (1 << 16) +#define EOP_TC_ACTION_EN (1 << 17) /* L2 */ +#define EOP_TCL2_VOLATILE (1 << 24) +#define EOP_CACHE_POLICY(x) ((x) << 25) + /* 0 - LRU + * 1 - Stream + * 2 - Bypass + */ +#define DATA_SEL(x) ((x) << 29) + /* 0 - discard + * 1 - send low 32bit data + * 2 - send 64bit data + * 3 - send 64bit GPU counter value + * 4 - send 64bit sys counter value + */ +#define INT_SEL(x) ((x) << 24) + /* 0 - none + * 1 - interrupt only (DATA_SEL = 0) + * 2 - interrupt when data write is confirmed + */ +#define DST_SEL(x) ((x) << 16) + /* 0 - MC + * 1 - TC/L2 + */ +#define PACKET3_EVENT_WRITE_EOS 0x48 +#define PACKET3_RELEASE_MEM 0x49 +#define PACKET3_PREAMBLE_CNTL 0x4A +# define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE (2 << 28) +# define PACKET3_PREAMBLE_END_CLEAR_STATE (3 << 28) +#define PACKET3_DMA_DATA 0x50 +/* 1. header + * 2. CONTROL + * 3. SRC_ADDR_LO or DATA [31:0] + * 4. SRC_ADDR_HI [31:0] + * 5. DST_ADDR_LO [31:0] + * 6. DST_ADDR_HI [7:0] + * 7. COMMAND [30:21] | BYTE_COUNT [20:0] + */ +/* CONTROL */ +# define PACKET3_DMA_DATA_ENGINE(x) ((x) << 0) + /* 0 - ME + * 1 - PFP + */ +# define PACKET3_DMA_DATA_SRC_CACHE_POLICY(x) ((x) << 13) + /* 0 - LRU + * 1 - Stream + * 2 - Bypass + */ +# define PACKET3_DMA_DATA_SRC_VOLATILE (1 << 15) +# define PACKET3_DMA_DATA_DST_SEL(x) ((x) << 20) + /* 0 - DST_ADDR using DAS + * 1 - GDS + * 3 - DST_ADDR using L2 + */ +# define PACKET3_DMA_DATA_DST_CACHE_POLICY(x) ((x) << 25) + /* 0 - LRU + * 1 - Stream + * 2 - Bypass + */ +# define PACKET3_DMA_DATA_DST_VOLATILE (1 << 27) +# define PACKET3_DMA_DATA_SRC_SEL(x) ((x) << 29) + /* 0 - SRC_ADDR using SAS + * 1 - GDS + * 2 - DATA + * 3 - SRC_ADDR using L2 + */ +# define PACKET3_DMA_DATA_CP_SYNC (1 << 31) +/* COMMAND */ +# define PACKET3_DMA_DATA_DIS_WC (1 << 21) +# define PACKET3_DMA_DATA_CMD_SRC_SWAP(x) ((x) << 22) + /* 0 - none + * 1 - 8 in 16 + * 2 - 8 in 32 + * 3 - 8 in 64 + */ +# define PACKET3_DMA_DATA_CMD_DST_SWAP(x) ((x) << 24) + /* 0 - none + * 1 - 8 in 16 + * 2 - 8 in 32 + * 3 - 8 in 64 + */ +# define PACKET3_DMA_DATA_CMD_SAS (1 << 26) + /* 0 - memory + * 1 - register + */ +# define PACKET3_DMA_DATA_CMD_DAS (1 << 27) + /* 0 - memory + * 1 - register + */ +# define PACKET3_DMA_DATA_CMD_SAIC (1 << 28) +# define PACKET3_DMA_DATA_CMD_DAIC (1 << 29) +# define PACKET3_DMA_DATA_CMD_RAW_WAIT (1 << 30) +#define PACKET3_AQUIRE_MEM 0x58 +#define PACKET3_REWIND 0x59 +#define PACKET3_LOAD_UCONFIG_REG 0x5E +#define PACKET3_LOAD_SH_REG 0x5F +#define PACKET3_LOAD_CONFIG_REG 0x60 +#define PACKET3_LOAD_CONTEXT_REG 0x61 +#define PACKET3_SET_CONFIG_REG 0x68 +#define PACKET3_SET_CONFIG_REG_START 0x00008000 +#define PACKET3_SET_CONFIG_REG_END 0x0000b000 +#define PACKET3_SET_CONTEXT_REG 0x69 +#define PACKET3_SET_CONTEXT_REG_START 0x00028000 +#define PACKET3_SET_CONTEXT_REG_END 0x00029000 +#define PACKET3_SET_CONTEXT_REG_INDIRECT 0x73 +#define PACKET3_SET_SH_REG 0x76 +#define PACKET3_SET_SH_REG_START 0x0000b000 +#define PACKET3_SET_SH_REG_END 0x0000c000 +#define PACKET3_SET_SH_REG_OFFSET 0x77 +#define PACKET3_SET_QUEUE_REG 0x78 +#define PACKET3_SET_UCONFIG_REG 0x79 +#define PACKET3_SET_UCONFIG_REG_START 0x00030000 +#define PACKET3_SET_UCONFIG_REG_END 0x00031000 +#define PACKET3_SCRATCH_RAM_WRITE 0x7D +#define PACKET3_SCRATCH_RAM_READ 0x7E +#define PACKET3_LOAD_CONST_RAM 0x80 +#define PACKET3_WRITE_CONST_RAM 0x81 +#define PACKET3_DUMP_CONST_RAM 0x83 +#define PACKET3_INCREMENT_CE_COUNTER 0x84 +#define PACKET3_INCREMENT_DE_COUNTER 0x85 +#define PACKET3_WAIT_ON_CE_COUNTER 0x86 +#define PACKET3_WAIT_ON_DE_COUNTER_DIFF 0x88 +#define PACKET3_SWITCH_BUFFER 0x8B + +/* SDMA - first instance at 0xd000, second at 0xd800 */ +#define SDMA0_REGISTER_OFFSET 0x0 /* not a register */ +#define SDMA1_REGISTER_OFFSET 0x800 /* not a register */ + +#define SDMA0_UCODE_ADDR 0xD000 +#define SDMA0_UCODE_DATA 0xD004 +#define SDMA0_POWER_CNTL 0xD008 +#define SDMA0_CLK_CTRL 0xD00C + +#define SDMA0_CNTL 0xD010 +# define TRAP_ENABLE (1 << 0) +# define SEM_INCOMPLETE_INT_ENABLE (1 << 1) +# define SEM_WAIT_INT_ENABLE (1 << 2) +# define DATA_SWAP_ENABLE (1 << 3) +# define FENCE_SWAP_ENABLE (1 << 4) +# define AUTO_CTXSW_ENABLE (1 << 18) +# define CTXEMPTY_INT_ENABLE (1 << 28) + +#define SDMA0_TILING_CONFIG 0xD018 + +#define SDMA0_SEM_INCOMPLETE_TIMER_CNTL 0xD020 +#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL 0xD024 + +#define SDMA0_STATUS_REG 0xd034 +# define SDMA_IDLE (1 << 0) + +#define SDMA0_ME_CNTL 0xD048 +# define SDMA_HALT (1 << 0) + +#define SDMA0_GFX_RB_CNTL 0xD200 +# define SDMA_RB_ENABLE (1 << 0) +# define SDMA_RB_SIZE(x) ((x) << 1) /* log2 */ +# define SDMA_RB_SWAP_ENABLE (1 << 9) /* 8IN32 */ +# define SDMA_RPTR_WRITEBACK_ENABLE (1 << 12) +# define SDMA_RPTR_WRITEBACK_SWAP_ENABLE (1 << 13) /* 8IN32 */ +# define SDMA_RPTR_WRITEBACK_TIMER(x) ((x) << 16) /* log2 */ +#define SDMA0_GFX_RB_BASE 0xD204 +#define SDMA0_GFX_RB_BASE_HI 0xD208 +#define SDMA0_GFX_RB_RPTR 0xD20C +#define SDMA0_GFX_RB_WPTR 0xD210 + +#define SDMA0_GFX_RB_RPTR_ADDR_HI 0xD220 +#define SDMA0_GFX_RB_RPTR_ADDR_LO 0xD224 +#define SDMA0_GFX_IB_CNTL 0xD228 +# define SDMA_IB_ENABLE (1 << 0) +# define SDMA_IB_SWAP_ENABLE (1 << 4) +# define SDMA_SWITCH_INSIDE_IB (1 << 8) +# define SDMA_CMD_VMID(x) ((x) << 16) + +#define SDMA0_GFX_VIRTUAL_ADDR 0xD29C +#define SDMA0_GFX_APE1_CNTL 0xD2A0 + +#define SDMA_PACKET(op, sub_op, e) ((((e) & 0xFFFF) << 16) | \ + (((sub_op) & 0xFF) << 8) | \ + (((op) & 0xFF) << 0)) +/* sDMA opcodes */ +#define SDMA_OPCODE_NOP 0 +#define SDMA_OPCODE_COPY 1 +# define SDMA_COPY_SUB_OPCODE_LINEAR 0 +# define SDMA_COPY_SUB_OPCODE_TILED 1 +# define SDMA_COPY_SUB_OPCODE_SOA 3 +# define SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW 4 +# define SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW 5 +# define SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW 6 +#define SDMA_OPCODE_WRITE 2 +# define SDMA_WRITE_SUB_OPCODE_LINEAR 0 +# define SDMA_WRTIE_SUB_OPCODE_TILED 1 +#define SDMA_OPCODE_INDIRECT_BUFFER 4 +#define SDMA_OPCODE_FENCE 5 +#define SDMA_OPCODE_TRAP 6 +#define SDMA_OPCODE_SEMAPHORE 7 +# define SDMA_SEMAPHORE_EXTRA_O (1 << 13) + /* 0 - increment + * 1 - write 1 + */ +# define SDMA_SEMAPHORE_EXTRA_S (1 << 14) + /* 0 - wait + * 1 - signal + */ +# define SDMA_SEMAPHORE_EXTRA_M (1 << 15) + /* mailbox */ +#define SDMA_OPCODE_POLL_REG_MEM 8 +# define SDMA_POLL_REG_MEM_EXTRA_OP(x) ((x) << 10) + /* 0 - wait_reg_mem + * 1 - wr_wait_wr_reg + */ +# define SDMA_POLL_REG_MEM_EXTRA_FUNC(x) ((x) << 12) + /* 0 - always + * 1 - < + * 2 - <= + * 3 - == + * 4 - != + * 5 - >= + * 6 - > + */ +# define SDMA_POLL_REG_MEM_EXTRA_M (1 << 15) + /* 0 = register + * 1 = memory + */ +#define SDMA_OPCODE_COND_EXEC 9 +#define SDMA_OPCODE_CONSTANT_FILL 11 +# define SDMA_CONSTANT_FILL_EXTRA_SIZE(x) ((x) << 14) + /* 0 = byte fill + * 2 = DW fill + */ +#define SDMA_OPCODE_GENERATE_PTE_PDE 12 +#define SDMA_OPCODE_TIMESTAMP 13 +# define SDMA_TIMESTAMP_SUB_OPCODE_SET_LOCAL 0 +# define SDMA_TIMESTAMP_SUB_OPCODE_GET_LOCAL 1 +# define SDMA_TIMESTAMP_SUB_OPCODE_GET_GLOBAL 2 +#define SDMA_OPCODE_SRBM_WRITE 14 +# define SDMA_SRBM_WRITE_EXTRA_BYTE_ENABLE(x) ((x) << 12) + /* byte mask */ + +/* UVD */ + +#define UVD_UDEC_ADDR_CONFIG 0xef4c +#define UVD_UDEC_DB_ADDR_CONFIG 0xef50 +#define UVD_UDEC_DBW_ADDR_CONFIG 0xef54 + +#define UVD_LMI_EXT40_ADDR 0xf498 +#define UVD_LMI_ADDR_EXT 0xf594 +#define UVD_VCPU_CACHE_OFFSET0 0xf608 +#define UVD_VCPU_CACHE_SIZE0 0xf60c +#define UVD_VCPU_CACHE_OFFSET1 0xf610 +#define UVD_VCPU_CACHE_SIZE1 0xf614 +#define UVD_VCPU_CACHE_OFFSET2 0xf618 +#define UVD_VCPU_CACHE_SIZE2 0xf61c + +#define UVD_RBC_RB_RPTR 0xf690 +#define UVD_RBC_RB_WPTR 0xf694 + +#define UVD_CGC_CTRL 0xF4B0 +# define DCM (1 << 0) +# define CG_DT(x) ((x) << 2) +# define CG_DT_MASK (0xf << 2) +# define CLK_OD(x) ((x) << 6) +# define CLK_OD_MASK (0x1f << 6) + +#define UVD_STATUS 0xf6bc + +/* UVD clocks */ + +#define CG_DCLK_CNTL 0xC050009C +# define DCLK_DIVIDER_MASK 0x7f +# define DCLK_DIR_CNTL_EN (1 << 8) +#define CG_DCLK_STATUS 0xC05000A0 +# define DCLK_STATUS (1 << 0) +#define CG_VCLK_CNTL 0xC05000A4 +#define CG_VCLK_STATUS 0xC05000A8 + +/* UVD CTX indirect */ +#define UVD_CGC_MEM_CTRL 0xC0 + +/* VCE */ + +#define VCE_VCPU_CACHE_OFFSET0 0x20024 +#define VCE_VCPU_CACHE_SIZE0 0x20028 +#define VCE_VCPU_CACHE_OFFSET1 0x2002c +#define VCE_VCPU_CACHE_SIZE1 0x20030 +#define VCE_VCPU_CACHE_OFFSET2 0x20034 +#define VCE_VCPU_CACHE_SIZE2 0x20038 +#define VCE_RB_RPTR2 0x20178 +#define VCE_RB_WPTR2 0x2017c +#define VCE_RB_RPTR 0x2018c +#define VCE_RB_WPTR 0x20190 +#define VCE_CLOCK_GATING_A 0x202f8 +# define CGC_CLK_GATE_DLY_TIMER_MASK (0xf << 0) +# define CGC_CLK_GATE_DLY_TIMER(x) ((x) << 0) +# define CGC_CLK_GATER_OFF_DLY_TIMER_MASK (0xff << 4) +# define CGC_CLK_GATER_OFF_DLY_TIMER(x) ((x) << 4) +# define CGC_UENC_WAIT_AWAKE (1 << 18) +#define VCE_CLOCK_GATING_B 0x202fc +#define VCE_CGTT_CLK_OVERRIDE 0x207a0 +#define VCE_UENC_CLOCK_GATING 0x207bc +# define CLOCK_ON_DELAY_MASK (0xf << 0) +# define CLOCK_ON_DELAY(x) ((x) << 0) +# define CLOCK_OFF_DELAY_MASK (0xff << 4) +# define CLOCK_OFF_DELAY(x) ((x) << 4) +#define VCE_UENC_REG_CLOCK_GATING 0x207c0 +#define VCE_SYS_INT_EN 0x21300 +# define VCE_SYS_INT_TRAP_INTERRUPT_EN (1 << 3) +#define VCE_LMI_VCPU_CACHE_40BIT_BAR 0x2145c +#define VCE_LMI_CTRL2 0x21474 +#define VCE_LMI_CTRL 0x21498 +#define VCE_LMI_VM_CTRL 0x214a0 +#define VCE_LMI_SWAP_CNTL 0x214b4 +#define VCE_LMI_SWAP_CNTL1 0x214b8 +#define VCE_LMI_CACHE_CTRL 0x214f4 + +#define VCE_CMD_NO_OP 0x00000000 +#define VCE_CMD_END 0x00000001 +#define VCE_CMD_IB 0x00000002 +#define VCE_CMD_FENCE 0x00000003 +#define VCE_CMD_TRAP 0x00000004 +#define VCE_CMD_IB_AUTO 0x00000005 +#define VCE_CMD_SEMAPHORE 0x00000006 + +#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x3398u +#define ATC_VMID0_PASID_MAPPING 0x339Cu +#define ATC_VMID_PASID_MAPPING_PASID_MASK (0xFFFF) +#define ATC_VMID_PASID_MAPPING_PASID_SHIFT 0 +#define ATC_VMID_PASID_MAPPING_VALID_MASK (0x1 << 31) +#define ATC_VMID_PASID_MAPPING_VALID_SHIFT 31 + +#define ATC_VM_APERTURE0_CNTL 0x3310u +#define ATS_ACCESS_MODE_NEVER 0 +#define ATS_ACCESS_MODE_ALWAYS 1 + +#define ATC_VM_APERTURE0_CNTL2 0x3318u +#define ATC_VM_APERTURE0_HIGH_ADDR 0x3308u +#define ATC_VM_APERTURE0_LOW_ADDR 0x3300u +#define ATC_VM_APERTURE1_CNTL 0x3314u +#define ATC_VM_APERTURE1_CNTL2 0x331Cu +#define ATC_VM_APERTURE1_HIGH_ADDR 0x330Cu +#define ATC_VM_APERTURE1_LOW_ADDR 0x3304u + +#define IH_VMID_0_LUT 0x3D40u + +#endif diff --git a/sys/dev/pci/drm/radeon/clearstate_cayman.h b/sys/dev/pci/drm/radeon/clearstate_cayman.h new file mode 100644 index 00000000000..e48a14037b7 --- /dev/null +++ b/sys/dev/pci/drm/radeon/clearstate_cayman.h @@ -0,0 +1,1081 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +static const u32 SECT_CONTEXT_def_1[] = +{ + 0x00000000, // DB_RENDER_CONTROL + 0x00000000, // DB_COUNT_CONTROL + 0x00000000, // DB_DEPTH_VIEW + 0x00000000, // DB_RENDER_OVERRIDE + 0x00000000, // DB_RENDER_OVERRIDE2 + 0x00000000, // DB_HTILE_DATA_BASE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_STENCIL_CLEAR + 0x00000000, // DB_DEPTH_CLEAR + 0x00000000, // PA_SC_SCREEN_SCISSOR_TL + 0x40004000, // PA_SC_SCREEN_SCISSOR_BR + 0, // HOLE + 0x00000000, // DB_DEPTH_INFO + 0x00000000, // DB_Z_INFO + 0x00000000, // DB_STENCIL_INFO + 0x00000000, // DB_Z_READ_BASE + 0x00000000, // DB_STENCIL_READ_BASE + 0x00000000, // DB_Z_WRITE_BASE + 0x00000000, // DB_STENCIL_WRITE_BASE + 0x00000000, // DB_DEPTH_SIZE + 0x00000000, // DB_DEPTH_SLICE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_15 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_15 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_15 + 0x00000000, // PA_SC_WINDOW_OFFSET + 0x80000000, // PA_SC_WINDOW_SCISSOR_TL + 0x40004000, // PA_SC_WINDOW_SCISSOR_BR + 0x0000ffff, // PA_SC_CLIPRECT_RULE + 0x00000000, // PA_SC_CLIPRECT_0_TL + 0x40004000, // PA_SC_CLIPRECT_0_BR + 0x00000000, // PA_SC_CLIPRECT_1_TL + 0x40004000, // PA_SC_CLIPRECT_1_BR + 0x00000000, // PA_SC_CLIPRECT_2_TL + 0x40004000, // PA_SC_CLIPRECT_2_BR + 0x00000000, // PA_SC_CLIPRECT_3_TL + 0x40004000, // PA_SC_CLIPRECT_3_BR + 0xaa99aaaa, // PA_SC_EDGERULE + 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET + 0xffffffff, // CB_TARGET_MASK + 0xffffffff, // CB_SHADER_MASK + 0x80000000, // PA_SC_GENERIC_SCISSOR_TL + 0x40004000, // PA_SC_GENERIC_SCISSOR_BR + 0x00000000, // COHER_DEST_BASE_0 + 0x00000000, // COHER_DEST_BASE_1 + 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR + 0x00000000, // PA_SC_VPORT_ZMIN_0 + 0x3f800000, // PA_SC_VPORT_ZMAX_0 + 0x00000000, // PA_SC_VPORT_ZMIN_1 + 0x3f800000, // PA_SC_VPORT_ZMAX_1 + 0x00000000, // PA_SC_VPORT_ZMIN_2 + 0x3f800000, // PA_SC_VPORT_ZMAX_2 + 0x00000000, // PA_SC_VPORT_ZMIN_3 + 0x3f800000, // PA_SC_VPORT_ZMAX_3 + 0x00000000, // PA_SC_VPORT_ZMIN_4 + 0x3f800000, // PA_SC_VPORT_ZMAX_4 + 0x00000000, // PA_SC_VPORT_ZMIN_5 + 0x3f800000, // PA_SC_VPORT_ZMAX_5 + 0x00000000, // PA_SC_VPORT_ZMIN_6 + 0x3f800000, // PA_SC_VPORT_ZMAX_6 + 0x00000000, // PA_SC_VPORT_ZMIN_7 + 0x3f800000, // PA_SC_VPORT_ZMAX_7 + 0x00000000, // PA_SC_VPORT_ZMIN_8 + 0x3f800000, // PA_SC_VPORT_ZMAX_8 + 0x00000000, // PA_SC_VPORT_ZMIN_9 + 0x3f800000, // PA_SC_VPORT_ZMAX_9 + 0x00000000, // PA_SC_VPORT_ZMIN_10 + 0x3f800000, // PA_SC_VPORT_ZMAX_10 + 0x00000000, // PA_SC_VPORT_ZMIN_11 + 0x3f800000, // PA_SC_VPORT_ZMAX_11 + 0x00000000, // PA_SC_VPORT_ZMIN_12 + 0x3f800000, // PA_SC_VPORT_ZMAX_12 + 0x00000000, // PA_SC_VPORT_ZMIN_13 + 0x3f800000, // PA_SC_VPORT_ZMAX_13 + 0x00000000, // PA_SC_VPORT_ZMIN_14 + 0x3f800000, // PA_SC_VPORT_ZMAX_14 + 0x00000000, // PA_SC_VPORT_ZMIN_15 + 0x3f800000, // PA_SC_VPORT_ZMAX_15 + 0x00000000, // SX_MISC + 0x00000000, // SX_SURFACE_SYNC + 0x00000000, // SX_SCATTER_EXPORT_BASE + 0x00000000, // SX_SCATTER_EXPORT_SIZE + 0x00000000, // CP_PERFMON_CNTX_CNTL + 0x00000000, // CP_RINGID + 0x00000000, // CP_VMID + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_VTX_SEMANTIC_0 + 0x00000000, // SQ_VTX_SEMANTIC_1 + 0x00000000, // SQ_VTX_SEMANTIC_2 + 0x00000000, // SQ_VTX_SEMANTIC_3 + 0x00000000, // SQ_VTX_SEMANTIC_4 + 0x00000000, // SQ_VTX_SEMANTIC_5 + 0x00000000, // SQ_VTX_SEMANTIC_6 + 0x00000000, // SQ_VTX_SEMANTIC_7 + 0x00000000, // SQ_VTX_SEMANTIC_8 + 0x00000000, // SQ_VTX_SEMANTIC_9 + 0x00000000, // SQ_VTX_SEMANTIC_10 + 0x00000000, // SQ_VTX_SEMANTIC_11 + 0x00000000, // SQ_VTX_SEMANTIC_12 + 0x00000000, // SQ_VTX_SEMANTIC_13 + 0x00000000, // SQ_VTX_SEMANTIC_14 + 0x00000000, // SQ_VTX_SEMANTIC_15 + 0x00000000, // SQ_VTX_SEMANTIC_16 + 0x00000000, // SQ_VTX_SEMANTIC_17 + 0x00000000, // SQ_VTX_SEMANTIC_18 + 0x00000000, // SQ_VTX_SEMANTIC_19 + 0x00000000, // SQ_VTX_SEMANTIC_20 + 0x00000000, // SQ_VTX_SEMANTIC_21 + 0x00000000, // SQ_VTX_SEMANTIC_22 + 0x00000000, // SQ_VTX_SEMANTIC_23 + 0x00000000, // SQ_VTX_SEMANTIC_24 + 0x00000000, // SQ_VTX_SEMANTIC_25 + 0x00000000, // SQ_VTX_SEMANTIC_26 + 0x00000000, // SQ_VTX_SEMANTIC_27 + 0x00000000, // SQ_VTX_SEMANTIC_28 + 0x00000000, // SQ_VTX_SEMANTIC_29 + 0x00000000, // SQ_VTX_SEMANTIC_30 + 0x00000000, // SQ_VTX_SEMANTIC_31 + 0xffffffff, // VGT_MAX_VTX_INDX + 0x00000000, // VGT_MIN_VTX_INDX + 0x00000000, // VGT_INDX_OFFSET + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX + 0x00000000, // SX_ALPHA_TEST_CONTROL + 0x00000000, // CB_BLEND_RED + 0x00000000, // CB_BLEND_GREEN + 0x00000000, // CB_BLEND_BLUE + 0x00000000, // CB_BLEND_ALPHA + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_STENCILREFMASK + 0x00000000, // DB_STENCILREFMASK_BF + 0x00000000, // SX_ALPHA_REF + 0x00000000, // PA_CL_VPORT_XSCALE + 0x00000000, // PA_CL_VPORT_XOFFSET + 0x00000000, // PA_CL_VPORT_YSCALE + 0x00000000, // PA_CL_VPORT_YOFFSET + 0x00000000, // PA_CL_VPORT_ZSCALE + 0x00000000, // PA_CL_VPORT_ZOFFSET + 0x00000000, // PA_CL_VPORT_XSCALE_1 + 0x00000000, // PA_CL_VPORT_XOFFSET_1 + 0x00000000, // PA_CL_VPORT_YSCALE_1 + 0x00000000, // PA_CL_VPORT_YOFFSET_1 + 0x00000000, // PA_CL_VPORT_ZSCALE_1 + 0x00000000, // PA_CL_VPORT_ZOFFSET_1 + 0x00000000, // PA_CL_VPORT_XSCALE_2 + 0x00000000, // PA_CL_VPORT_XOFFSET_2 + 0x00000000, // PA_CL_VPORT_YSCALE_2 + 0x00000000, // PA_CL_VPORT_YOFFSET_2 + 0x00000000, // PA_CL_VPORT_ZSCALE_2 + 0x00000000, // PA_CL_VPORT_ZOFFSET_2 + 0x00000000, // PA_CL_VPORT_XSCALE_3 + 0x00000000, // PA_CL_VPORT_XOFFSET_3 + 0x00000000, // PA_CL_VPORT_YSCALE_3 + 0x00000000, // PA_CL_VPORT_YOFFSET_3 + 0x00000000, // PA_CL_VPORT_ZSCALE_3 + 0x00000000, // PA_CL_VPORT_ZOFFSET_3 + 0x00000000, // PA_CL_VPORT_XSCALE_4 + 0x00000000, // PA_CL_VPORT_XOFFSET_4 + 0x00000000, // PA_CL_VPORT_YSCALE_4 + 0x00000000, // PA_CL_VPORT_YOFFSET_4 + 0x00000000, // PA_CL_VPORT_ZSCALE_4 + 0x00000000, // PA_CL_VPORT_ZOFFSET_4 + 0x00000000, // PA_CL_VPORT_XSCALE_5 + 0x00000000, // PA_CL_VPORT_XOFFSET_5 + 0x00000000, // PA_CL_VPORT_YSCALE_5 + 0x00000000, // PA_CL_VPORT_YOFFSET_5 + 0x00000000, // PA_CL_VPORT_ZSCALE_5 + 0x00000000, // PA_CL_VPORT_ZOFFSET_5 + 0x00000000, // PA_CL_VPORT_XSCALE_6 + 0x00000000, // PA_CL_VPORT_XOFFSET_6 + 0x00000000, // PA_CL_VPORT_YSCALE_6 + 0x00000000, // PA_CL_VPORT_YOFFSET_6 + 0x00000000, // PA_CL_VPORT_ZSCALE_6 + 0x00000000, // PA_CL_VPORT_ZOFFSET_6 + 0x00000000, // PA_CL_VPORT_XSCALE_7 + 0x00000000, // PA_CL_VPORT_XOFFSET_7 + 0x00000000, // PA_CL_VPORT_YSCALE_7 + 0x00000000, // PA_CL_VPORT_YOFFSET_7 + 0x00000000, // PA_CL_VPORT_ZSCALE_7 + 0x00000000, // PA_CL_VPORT_ZOFFSET_7 + 0x00000000, // PA_CL_VPORT_XSCALE_8 + 0x00000000, // PA_CL_VPORT_XOFFSET_8 + 0x00000000, // PA_CL_VPORT_YSCALE_8 + 0x00000000, // PA_CL_VPORT_YOFFSET_8 + 0x00000000, // PA_CL_VPORT_ZSCALE_8 + 0x00000000, // PA_CL_VPORT_ZOFFSET_8 + 0x00000000, // PA_CL_VPORT_XSCALE_9 + 0x00000000, // PA_CL_VPORT_XOFFSET_9 + 0x00000000, // PA_CL_VPORT_YSCALE_9 + 0x00000000, // PA_CL_VPORT_YOFFSET_9 + 0x00000000, // PA_CL_VPORT_ZSCALE_9 + 0x00000000, // PA_CL_VPORT_ZOFFSET_9 + 0x00000000, // PA_CL_VPORT_XSCALE_10 + 0x00000000, // PA_CL_VPORT_XOFFSET_10 + 0x00000000, // PA_CL_VPORT_YSCALE_10 + 0x00000000, // PA_CL_VPORT_YOFFSET_10 + 0x00000000, // PA_CL_VPORT_ZSCALE_10 + 0x00000000, // PA_CL_VPORT_ZOFFSET_10 + 0x00000000, // PA_CL_VPORT_XSCALE_11 + 0x00000000, // PA_CL_VPORT_XOFFSET_11 + 0x00000000, // PA_CL_VPORT_YSCALE_11 + 0x00000000, // PA_CL_VPORT_YOFFSET_11 + 0x00000000, // PA_CL_VPORT_ZSCALE_11 + 0x00000000, // PA_CL_VPORT_ZOFFSET_11 + 0x00000000, // PA_CL_VPORT_XSCALE_12 + 0x00000000, // PA_CL_VPORT_XOFFSET_12 + 0x00000000, // PA_CL_VPORT_YSCALE_12 + 0x00000000, // PA_CL_VPORT_YOFFSET_12 + 0x00000000, // PA_CL_VPORT_ZSCALE_12 + 0x00000000, // PA_CL_VPORT_ZOFFSET_12 + 0x00000000, // PA_CL_VPORT_XSCALE_13 + 0x00000000, // PA_CL_VPORT_XOFFSET_13 + 0x00000000, // PA_CL_VPORT_YSCALE_13 + 0x00000000, // PA_CL_VPORT_YOFFSET_13 + 0x00000000, // PA_CL_VPORT_ZSCALE_13 + 0x00000000, // PA_CL_VPORT_ZOFFSET_13 + 0x00000000, // PA_CL_VPORT_XSCALE_14 + 0x00000000, // PA_CL_VPORT_XOFFSET_14 + 0x00000000, // PA_CL_VPORT_YSCALE_14 + 0x00000000, // PA_CL_VPORT_YOFFSET_14 + 0x00000000, // PA_CL_VPORT_ZSCALE_14 + 0x00000000, // PA_CL_VPORT_ZOFFSET_14 + 0x00000000, // PA_CL_VPORT_XSCALE_15 + 0x00000000, // PA_CL_VPORT_XOFFSET_15 + 0x00000000, // PA_CL_VPORT_YSCALE_15 + 0x00000000, // PA_CL_VPORT_YOFFSET_15 + 0x00000000, // PA_CL_VPORT_ZSCALE_15 + 0x00000000, // PA_CL_VPORT_ZOFFSET_15 + 0x00000000, // PA_CL_UCP_0_X + 0x00000000, // PA_CL_UCP_0_Y + 0x00000000, // PA_CL_UCP_0_Z + 0x00000000, // PA_CL_UCP_0_W + 0x00000000, // PA_CL_UCP_1_X + 0x00000000, // PA_CL_UCP_1_Y + 0x00000000, // PA_CL_UCP_1_Z + 0x00000000, // PA_CL_UCP_1_W + 0x00000000, // PA_CL_UCP_2_X + 0x00000000, // PA_CL_UCP_2_Y + 0x00000000, // PA_CL_UCP_2_Z + 0x00000000, // PA_CL_UCP_2_W + 0x00000000, // PA_CL_UCP_3_X + 0x00000000, // PA_CL_UCP_3_Y + 0x00000000, // PA_CL_UCP_3_Z + 0x00000000, // PA_CL_UCP_3_W + 0x00000000, // PA_CL_UCP_4_X + 0x00000000, // PA_CL_UCP_4_Y + 0x00000000, // PA_CL_UCP_4_Z + 0x00000000, // PA_CL_UCP_4_W + 0x00000000, // PA_CL_UCP_5_X + 0x00000000, // PA_CL_UCP_5_Y + 0x00000000, // PA_CL_UCP_5_Z + 0x00000000, // PA_CL_UCP_5_W + 0x00000000, // SPI_VS_OUT_ID_0 + 0x00000000, // SPI_VS_OUT_ID_1 + 0x00000000, // SPI_VS_OUT_ID_2 + 0x00000000, // SPI_VS_OUT_ID_3 + 0x00000000, // SPI_VS_OUT_ID_4 + 0x00000000, // SPI_VS_OUT_ID_5 + 0x00000000, // SPI_VS_OUT_ID_6 + 0x00000000, // SPI_VS_OUT_ID_7 + 0x00000000, // SPI_VS_OUT_ID_8 + 0x00000000, // SPI_VS_OUT_ID_9 + 0x00000000, // SPI_PS_INPUT_CNTL_0 + 0x00000000, // SPI_PS_INPUT_CNTL_1 + 0x00000000, // SPI_PS_INPUT_CNTL_2 + 0x00000000, // SPI_PS_INPUT_CNTL_3 + 0x00000000, // SPI_PS_INPUT_CNTL_4 + 0x00000000, // SPI_PS_INPUT_CNTL_5 + 0x00000000, // SPI_PS_INPUT_CNTL_6 + 0x00000000, // SPI_PS_INPUT_CNTL_7 + 0x00000000, // SPI_PS_INPUT_CNTL_8 + 0x00000000, // SPI_PS_INPUT_CNTL_9 + 0x00000000, // SPI_PS_INPUT_CNTL_10 + 0x00000000, // SPI_PS_INPUT_CNTL_11 + 0x00000000, // SPI_PS_INPUT_CNTL_12 + 0x00000000, // SPI_PS_INPUT_CNTL_13 + 0x00000000, // SPI_PS_INPUT_CNTL_14 + 0x00000000, // SPI_PS_INPUT_CNTL_15 + 0x00000000, // SPI_PS_INPUT_CNTL_16 + 0x00000000, // SPI_PS_INPUT_CNTL_17 + 0x00000000, // SPI_PS_INPUT_CNTL_18 + 0x00000000, // SPI_PS_INPUT_CNTL_19 + 0x00000000, // SPI_PS_INPUT_CNTL_20 + 0x00000000, // SPI_PS_INPUT_CNTL_21 + 0x00000000, // SPI_PS_INPUT_CNTL_22 + 0x00000000, // SPI_PS_INPUT_CNTL_23 + 0x00000000, // SPI_PS_INPUT_CNTL_24 + 0x00000000, // SPI_PS_INPUT_CNTL_25 + 0x00000000, // SPI_PS_INPUT_CNTL_26 + 0x00000000, // SPI_PS_INPUT_CNTL_27 + 0x00000000, // SPI_PS_INPUT_CNTL_28 + 0x00000000, // SPI_PS_INPUT_CNTL_29 + 0x00000000, // SPI_PS_INPUT_CNTL_30 + 0x00000000, // SPI_PS_INPUT_CNTL_31 + 0x00000000, // SPI_VS_OUT_CONFIG + 0x00000001, // SPI_THREAD_GROUPING + 0x00000002, // SPI_PS_IN_CONTROL_0 + 0x00000000, // SPI_PS_IN_CONTROL_1 + 0x00000000, // SPI_INTERP_CONTROL_0 + 0x00000000, // SPI_INPUT_Z + 0x00000000, // SPI_FOG_CNTL + 0x00000000, // SPI_BARYC_CNTL + 0x00000000, // SPI_PS_IN_CONTROL_2 + 0x00000000, // SPI_COMPUTE_INPUT_CNTL + 0x00000000, // SPI_COMPUTE_NUM_THREAD_X + 0x00000000, // SPI_COMPUTE_NUM_THREAD_Y + 0x00000000, // SPI_COMPUTE_NUM_THREAD_Z + 0x00000000, // SPI_GPR_MGMT + 0x00000000, // SPI_LDS_MGMT + 0x00000000, // SPI_STACK_MGMT + 0x00000000, // SPI_WAVE_MGMT_1 + 0x00000000, // SPI_WAVE_MGMT_2 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // GDS_ADDR_BASE + 0x00003fff, // GDS_ADDR_SIZE + 0, // HOLE + 0, // HOLE + 0x00000000, // GDS_ORDERED_COUNT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // GDS_APPEND_CONSUME_UAV0 + 0x00000000, // GDS_APPEND_CONSUME_UAV1 + 0x00000000, // GDS_APPEND_CONSUME_UAV2 + 0x00000000, // GDS_APPEND_CONSUME_UAV3 + 0x00000000, // GDS_APPEND_CONSUME_UAV4 + 0x00000000, // GDS_APPEND_CONSUME_UAV5 + 0x00000000, // GDS_APPEND_CONSUME_UAV6 + 0x00000000, // GDS_APPEND_CONSUME_UAV7 + 0x00000000, // GDS_APPEND_CONSUME_UAV8 + 0x00000000, // GDS_APPEND_CONSUME_UAV9 + 0x00000000, // GDS_APPEND_CONSUME_UAV10 + 0x00000000, // GDS_APPEND_CONSUME_UAV11 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_BLEND0_CONTROL + 0x00000000, // CB_BLEND1_CONTROL + 0x00000000, // CB_BLEND2_CONTROL + 0x00000000, // CB_BLEND3_CONTROL + 0x00000000, // CB_BLEND4_CONTROL + 0x00000000, // CB_BLEND5_CONTROL + 0x00000000, // CB_BLEND6_CONTROL + 0x00000000, // CB_BLEND7_CONTROL +}; +static const u32 SECT_CONTEXT_def_2[] = +{ + 0x00000000, // PA_CL_POINT_X_RAD + 0x00000000, // PA_CL_POINT_Y_RAD + 0x00000000, // PA_CL_POINT_SIZE + 0x00000000, // PA_CL_POINT_CULL_RAD + 0x00000000, // VGT_DMA_BASE_HI + 0x00000000, // VGT_DMA_BASE +}; +static const u32 SECT_CONTEXT_def_3[] = +{ + 0x00000000, // DB_DEPTH_CONTROL + 0x00000000, // DB_EQAA + 0x00000000, // CB_COLOR_CONTROL + 0x00000200, // DB_SHADER_CONTROL + 0x00000000, // PA_CL_CLIP_CNTL + 0x00000000, // PA_SU_SC_MODE_CNTL + 0x00000000, // PA_CL_VTE_CNTL + 0x00000000, // PA_CL_VS_OUT_CNTL + 0x00000000, // PA_CL_NANINF_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_SCALE + 0x00000000, // PA_SU_PRIM_FILTER_CNTL + 0x00000000, // SQ_LSTMP_RING_ITEMSIZE + 0x00000000, // SQ_HSTMP_RING_ITEMSIZE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_PS + 0x00000000, // SQ_PGM_RESOURCES_PS + 0x00000000, // SQ_PGM_RESOURCES_2_PS + 0x00000000, // SQ_PGM_EXPORTS_PS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_VS + 0x00000000, // SQ_PGM_RESOURCES_VS + 0x00000000, // SQ_PGM_RESOURCES_2_VS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_GS + 0x00000000, // SQ_PGM_RESOURCES_GS + 0x00000000, // SQ_PGM_RESOURCES_2_GS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_ES + 0x00000000, // SQ_PGM_RESOURCES_ES + 0x00000000, // SQ_PGM_RESOURCES_2_ES + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_FS + 0x00000000, // SQ_PGM_RESOURCES_FS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_HS + 0x00000000, // SQ_PGM_RESOURCES_HS + 0x00000000, // SQ_PGM_RESOURCES_2_HS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_LS + 0x00000000, // SQ_PGM_RESOURCES_LS + 0x00000000, // SQ_PGM_RESOURCES_2_LS +}; +static const u32 SECT_CONTEXT_def_4[] = +{ + 0x00000000, // SQ_LDS_ALLOC + 0x00000000, // SQ_LDS_ALLOC_PS + 0x00000000, // SQ_VTX_SEMANTIC_CLEAR + 0, // HOLE + 0x00000000, // SQ_THREAD_TRACE_CTRL + 0, // HOLE + 0x00000000, // SQ_ESGS_RING_ITEMSIZE + 0x00000000, // SQ_GSVS_RING_ITEMSIZE + 0x00000000, // SQ_ESTMP_RING_ITEMSIZE + 0x00000000, // SQ_GSTMP_RING_ITEMSIZE + 0x00000000, // SQ_VSTMP_RING_ITEMSIZE + 0x00000000, // SQ_PSTMP_RING_ITEMSIZE + 0, // HOLE + 0x00000000, // SQ_GS_VERT_ITEMSIZE + 0x00000000, // SQ_GS_VERT_ITEMSIZE_1 + 0x00000000, // SQ_GS_VERT_ITEMSIZE_2 + 0x00000000, // SQ_GS_VERT_ITEMSIZE_3 + 0x00000000, // SQ_GSVS_RING_OFFSET_1 + 0x00000000, // SQ_GSVS_RING_OFFSET_2 + 0x00000000, // SQ_GSVS_RING_OFFSET_3 + 0x00000000, // SQ_GWS_RING_OFFSET + 0, // HOLE + 0x00000000, // SQ_ALU_CONST_CACHE_PS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_15 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_15 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_15 + 0x00000000, // PA_SU_POINT_SIZE + 0x00000000, // PA_SU_POINT_MINMAX + 0x00000000, // PA_SU_LINE_CNTL + 0x00000000, // PA_SC_LINE_STIPPLE + 0x00000000, // VGT_OUTPUT_PATH_CNTL + 0x00000000, // VGT_HOS_CNTL + 0x00000000, // VGT_HOS_MAX_TESS_LEVEL + 0x00000000, // VGT_HOS_MIN_TESS_LEVEL + 0x00000000, // VGT_HOS_REUSE_DEPTH + 0x00000000, // VGT_GROUP_PRIM_TYPE + 0x00000000, // VGT_GROUP_FIRST_DECR + 0x00000000, // VGT_GROUP_DECR + 0x00000000, // VGT_GROUP_VECT_0_CNTL + 0x00000000, // VGT_GROUP_VECT_1_CNTL + 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL + 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL + 0x00000000, // VGT_GS_MODE + 0, // HOLE + 0x00000000, // PA_SC_MODE_CNTL_0 + 0x00000000, // PA_SC_MODE_CNTL_1 + 0x00000000, // VGT_ENHANCE + 0x00000100, // VGT_GS_PER_ES + 0x00000080, // VGT_ES_PER_GS + 0x00000002, // VGT_GS_PER_VS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_GS_OUT_PRIM_TYPE + 0x00000000, // IA_ENHANCE +}; +static const u32 SECT_CONTEXT_def_5[] = +{ + 0x00000000, // VGT_DMA_MAX_SIZE + 0x00000000, // VGT_DMA_INDEX_TYPE + 0, // HOLE + 0x00000000, // VGT_PRIMITIVEID_EN + 0x00000000, // VGT_DMA_NUM_INSTANCES +}; +static const u32 SECT_CONTEXT_def_6[] = +{ + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_INSTANCE_STEP_RATE_0 + 0x00000000, // VGT_INSTANCE_STEP_RATE_1 + 0x000000ff, // IA_MULTI_VGT_PARAM + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_REUSE_OFF + 0x00000000, // VGT_VTX_CNT_EN + 0x00000000, // DB_HTILE_SURFACE + 0x00000000, // DB_SRESULTS_COMPARE_STATE0 + 0x00000000, // DB_SRESULTS_COMPARE_STATE1 + 0x00000000, // DB_PRELOAD_CONTROL + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0 + 0x00000000, // VGT_STRMOUT_BUFFER_BASE_0 + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1 + 0x00000000, // VGT_STRMOUT_BUFFER_BASE_1 + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2 + 0x00000000, // VGT_STRMOUT_BUFFER_BASE_2 + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3 + 0x00000000, // VGT_STRMOUT_BUFFER_BASE_3 + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE + 0, // HOLE + 0x00000000, // VGT_GS_MAX_VERT_OUT + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_0 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_1 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_2 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_3 + 0x00000000, // VGT_SHADER_STAGES_EN + 0x00000000, // VGT_LS_HS_CONFIG + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_TF_PARAM + 0x00000000, // DB_ALPHA_TO_MASK +}; +static const u32 SECT_CONTEXT_def_7[] = +{ + 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL + 0x00000000, // PA_SU_POLY_OFFSET_CLAMP + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET + 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET + 0x00000000, // VGT_GS_INSTANCE_CNT + 0x00000000, // VGT_STRMOUT_CONFIG + 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG + 0x00000000, // CB_IMMED0_BASE + 0x00000000, // CB_IMMED1_BASE + 0x00000000, // CB_IMMED2_BASE + 0x00000000, // CB_IMMED3_BASE + 0x00000000, // CB_IMMED4_BASE + 0x00000000, // CB_IMMED5_BASE + 0x00000000, // CB_IMMED6_BASE + 0x00000000, // CB_IMMED7_BASE + 0x00000000, // CB_IMMED8_BASE + 0x00000000, // CB_IMMED9_BASE + 0x00000000, // CB_IMMED10_BASE + 0x00000000, // CB_IMMED11_BASE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SC_CENTROID_PRIORITY_0 + 0x00000000, // PA_SC_CENTROID_PRIORITY_1 + 0x00001000, // PA_SC_LINE_CNTL + 0x00000000, // PA_SC_AA_CONFIG + 0x00000005, // PA_SU_VTX_CNTL + 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ + 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ + 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ + 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 + 0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0 + 0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1 + 0x00000000, // CB_CLRCMP_CONTROL + 0x00000000, // CB_CLRCMP_SRC + 0x00000000, // CB_CLRCMP_DST + 0x00000000, // CB_CLRCMP_MSK + 0, // HOLE + 0, // HOLE + 0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL + 0x00000010, // VGT_OUT_DEALLOC_CNTL + 0x00000000, // CB_COLOR0_BASE + 0x00000000, // CB_COLOR0_PITCH + 0x00000000, // CB_COLOR0_SLICE + 0x00000000, // CB_COLOR0_VIEW + 0x00000000, // CB_COLOR0_INFO + 0x00000000, // CB_COLOR0_ATTRIB + 0x00000000, // CB_COLOR0_DIM + 0x00000000, // CB_COLOR0_CMASK + 0x00000000, // CB_COLOR0_CMASK_SLICE + 0x00000000, // CB_COLOR0_FMASK + 0x00000000, // CB_COLOR0_FMASK_SLICE + 0x00000000, // CB_COLOR0_CLEAR_WORD0 + 0x00000000, // CB_COLOR0_CLEAR_WORD1 + 0x00000000, // CB_COLOR0_CLEAR_WORD2 + 0x00000000, // CB_COLOR0_CLEAR_WORD3 + 0x00000000, // CB_COLOR1_BASE + 0x00000000, // CB_COLOR1_PITCH + 0x00000000, // CB_COLOR1_SLICE + 0x00000000, // CB_COLOR1_VIEW + 0x00000000, // CB_COLOR1_INFO + 0x00000000, // CB_COLOR1_ATTRIB + 0x00000000, // CB_COLOR1_DIM + 0x00000000, // CB_COLOR1_CMASK + 0x00000000, // CB_COLOR1_CMASK_SLICE + 0x00000000, // CB_COLOR1_FMASK + 0x00000000, // CB_COLOR1_FMASK_SLICE + 0x00000000, // CB_COLOR1_CLEAR_WORD0 + 0x00000000, // CB_COLOR1_CLEAR_WORD1 + 0x00000000, // CB_COLOR1_CLEAR_WORD2 + 0x00000000, // CB_COLOR1_CLEAR_WORD3 + 0x00000000, // CB_COLOR2_BASE + 0x00000000, // CB_COLOR2_PITCH + 0x00000000, // CB_COLOR2_SLICE + 0x00000000, // CB_COLOR2_VIEW + 0x00000000, // CB_COLOR2_INFO + 0x00000000, // CB_COLOR2_ATTRIB + 0x00000000, // CB_COLOR2_DIM + 0x00000000, // CB_COLOR2_CMASK + 0x00000000, // CB_COLOR2_CMASK_SLICE + 0x00000000, // CB_COLOR2_FMASK + 0x00000000, // CB_COLOR2_FMASK_SLICE + 0x00000000, // CB_COLOR2_CLEAR_WORD0 + 0x00000000, // CB_COLOR2_CLEAR_WORD1 + 0x00000000, // CB_COLOR2_CLEAR_WORD2 + 0x00000000, // CB_COLOR2_CLEAR_WORD3 + 0x00000000, // CB_COLOR3_BASE + 0x00000000, // CB_COLOR3_PITCH + 0x00000000, // CB_COLOR3_SLICE + 0x00000000, // CB_COLOR3_VIEW + 0x00000000, // CB_COLOR3_INFO + 0x00000000, // CB_COLOR3_ATTRIB + 0x00000000, // CB_COLOR3_DIM + 0x00000000, // CB_COLOR3_CMASK + 0x00000000, // CB_COLOR3_CMASK_SLICE + 0x00000000, // CB_COLOR3_FMASK + 0x00000000, // CB_COLOR3_FMASK_SLICE + 0x00000000, // CB_COLOR3_CLEAR_WORD0 + 0x00000000, // CB_COLOR3_CLEAR_WORD1 + 0x00000000, // CB_COLOR3_CLEAR_WORD2 + 0x00000000, // CB_COLOR3_CLEAR_WORD3 + 0x00000000, // CB_COLOR4_BASE + 0x00000000, // CB_COLOR4_PITCH + 0x00000000, // CB_COLOR4_SLICE + 0x00000000, // CB_COLOR4_VIEW + 0x00000000, // CB_COLOR4_INFO + 0x00000000, // CB_COLOR4_ATTRIB + 0x00000000, // CB_COLOR4_DIM + 0x00000000, // CB_COLOR4_CMASK + 0x00000000, // CB_COLOR4_CMASK_SLICE + 0x00000000, // CB_COLOR4_FMASK + 0x00000000, // CB_COLOR4_FMASK_SLICE + 0x00000000, // CB_COLOR4_CLEAR_WORD0 + 0x00000000, // CB_COLOR4_CLEAR_WORD1 + 0x00000000, // CB_COLOR4_CLEAR_WORD2 + 0x00000000, // CB_COLOR4_CLEAR_WORD3 + 0x00000000, // CB_COLOR5_BASE + 0x00000000, // CB_COLOR5_PITCH + 0x00000000, // CB_COLOR5_SLICE + 0x00000000, // CB_COLOR5_VIEW + 0x00000000, // CB_COLOR5_INFO + 0x00000000, // CB_COLOR5_ATTRIB + 0x00000000, // CB_COLOR5_DIM + 0x00000000, // CB_COLOR5_CMASK + 0x00000000, // CB_COLOR5_CMASK_SLICE + 0x00000000, // CB_COLOR5_FMASK + 0x00000000, // CB_COLOR5_FMASK_SLICE + 0x00000000, // CB_COLOR5_CLEAR_WORD0 + 0x00000000, // CB_COLOR5_CLEAR_WORD1 + 0x00000000, // CB_COLOR5_CLEAR_WORD2 + 0x00000000, // CB_COLOR5_CLEAR_WORD3 + 0x00000000, // CB_COLOR6_BASE + 0x00000000, // CB_COLOR6_PITCH + 0x00000000, // CB_COLOR6_SLICE + 0x00000000, // CB_COLOR6_VIEW + 0x00000000, // CB_COLOR6_INFO + 0x00000000, // CB_COLOR6_ATTRIB + 0x00000000, // CB_COLOR6_DIM + 0x00000000, // CB_COLOR6_CMASK + 0x00000000, // CB_COLOR6_CMASK_SLICE + 0x00000000, // CB_COLOR6_FMASK + 0x00000000, // CB_COLOR6_FMASK_SLICE + 0x00000000, // CB_COLOR6_CLEAR_WORD0 + 0x00000000, // CB_COLOR6_CLEAR_WORD1 + 0x00000000, // CB_COLOR6_CLEAR_WORD2 + 0x00000000, // CB_COLOR6_CLEAR_WORD3 + 0x00000000, // CB_COLOR7_BASE + 0x00000000, // CB_COLOR7_PITCH + 0x00000000, // CB_COLOR7_SLICE + 0x00000000, // CB_COLOR7_VIEW + 0x00000000, // CB_COLOR7_INFO + 0x00000000, // CB_COLOR7_ATTRIB + 0x00000000, // CB_COLOR7_DIM + 0x00000000, // CB_COLOR7_CMASK + 0x00000000, // CB_COLOR7_CMASK_SLICE + 0x00000000, // CB_COLOR7_FMASK + 0x00000000, // CB_COLOR7_FMASK_SLICE + 0x00000000, // CB_COLOR7_CLEAR_WORD0 + 0x00000000, // CB_COLOR7_CLEAR_WORD1 + 0x00000000, // CB_COLOR7_CLEAR_WORD2 + 0x00000000, // CB_COLOR7_CLEAR_WORD3 + 0x00000000, // CB_COLOR8_BASE + 0x00000000, // CB_COLOR8_PITCH + 0x00000000, // CB_COLOR8_SLICE + 0x00000000, // CB_COLOR8_VIEW + 0x00000000, // CB_COLOR8_INFO + 0x00000000, // CB_COLOR8_ATTRIB + 0x00000000, // CB_COLOR8_DIM + 0x00000000, // CB_COLOR9_BASE + 0x00000000, // CB_COLOR9_PITCH + 0x00000000, // CB_COLOR9_SLICE + 0x00000000, // CB_COLOR9_VIEW + 0x00000000, // CB_COLOR9_INFO + 0x00000000, // CB_COLOR9_ATTRIB + 0x00000000, // CB_COLOR9_DIM + 0x00000000, // CB_COLOR10_BASE + 0x00000000, // CB_COLOR10_PITCH + 0x00000000, // CB_COLOR10_SLICE + 0x00000000, // CB_COLOR10_VIEW + 0x00000000, // CB_COLOR10_INFO + 0x00000000, // CB_COLOR10_ATTRIB + 0x00000000, // CB_COLOR10_DIM + 0x00000000, // CB_COLOR11_BASE + 0x00000000, // CB_COLOR11_PITCH + 0x00000000, // CB_COLOR11_SLICE + 0x00000000, // CB_COLOR11_VIEW + 0x00000000, // CB_COLOR11_INFO + 0x00000000, // CB_COLOR11_ATTRIB + 0x00000000, // CB_COLOR11_DIM + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_ALU_CONST_CACHE_HS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_15 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_15 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_15 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_15 +}; +static const struct cs_extent_def SECT_CONTEXT_defs[] = +{ + {SECT_CONTEXT_def_1, 0x0000a000, 488 }, + {SECT_CONTEXT_def_2, 0x0000a1f5, 6 }, + {SECT_CONTEXT_def_3, 0x0000a200, 55 }, + {SECT_CONTEXT_def_4, 0x0000a23a, 99 }, + {SECT_CONTEXT_def_5, 0x0000a29e, 5 }, + {SECT_CONTEXT_def_6, 0x0000a2a5, 56 }, + {SECT_CONTEXT_def_7, 0x0000a2de, 290 }, + { NULL, 0, 0 } +}; +static const u32 SECT_CLEAR_def_1[] = +{ + 0xffffffff, // SQ_TEX_SAMPLER_CLEAR + 0xffffffff, // SQ_TEX_RESOURCE_CLEAR + 0xffffffff, // SQ_LOOP_BOOL_CLEAR +}; +static const struct cs_extent_def SECT_CLEAR_defs[] = +{ + {SECT_CLEAR_def_1, 0x0000ffc0, 3 }, + { NULL, 0, 0 } +}; +static const u32 SECT_CTRLCONST_def_1[] = +{ + 0x00000000, // SQ_VTX_BASE_VTX_LOC + 0x00000000, // SQ_VTX_START_INST_LOC +}; +static const struct cs_extent_def SECT_CTRLCONST_defs[] = +{ + {SECT_CTRLCONST_def_1, 0x0000f3fc, 2 }, + { NULL, 0, 0 } +}; +static const struct cs_section_def cayman_cs_data[] = { + { SECT_CONTEXT_defs, SECT_CONTEXT }, + { SECT_CLEAR_defs, SECT_CLEAR }, + { SECT_CTRLCONST_defs, SECT_CTRLCONST }, + { NULL, SECT_NONE } +}; diff --git a/sys/dev/pci/drm/radeon/clearstate_ci.h b/sys/dev/pci/drm/radeon/clearstate_ci.h new file mode 100644 index 00000000000..f55d06664e3 --- /dev/null +++ b/sys/dev/pci/drm/radeon/clearstate_ci.h @@ -0,0 +1,944 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +static const unsigned int ci_SECT_CONTEXT_def_1[] = +{ + 0x00000000, // DB_RENDER_CONTROL + 0x00000000, // DB_COUNT_CONTROL + 0x00000000, // DB_DEPTH_VIEW + 0x00000000, // DB_RENDER_OVERRIDE + 0x00000000, // DB_RENDER_OVERRIDE2 + 0x00000000, // DB_HTILE_DATA_BASE + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_DEPTH_BOUNDS_MIN + 0x00000000, // DB_DEPTH_BOUNDS_MAX + 0x00000000, // DB_STENCIL_CLEAR + 0x00000000, // DB_DEPTH_CLEAR + 0x00000000, // PA_SC_SCREEN_SCISSOR_TL + 0x40004000, // PA_SC_SCREEN_SCISSOR_BR + 0, // HOLE + 0x00000000, // DB_DEPTH_INFO + 0x00000000, // DB_Z_INFO + 0x00000000, // DB_STENCIL_INFO + 0x00000000, // DB_Z_READ_BASE + 0x00000000, // DB_STENCIL_READ_BASE + 0x00000000, // DB_Z_WRITE_BASE + 0x00000000, // DB_STENCIL_WRITE_BASE + 0x00000000, // DB_DEPTH_SIZE + 0x00000000, // DB_DEPTH_SLICE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // TA_BC_BASE_ADDR + 0x00000000, // TA_BC_BASE_ADDR_HI + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // COHER_DEST_BASE_HI_0 + 0x00000000, // COHER_DEST_BASE_HI_1 + 0x00000000, // COHER_DEST_BASE_HI_2 + 0x00000000, // COHER_DEST_BASE_HI_3 + 0x00000000, // COHER_DEST_BASE_2 + 0x00000000, // COHER_DEST_BASE_3 + 0x00000000, // PA_SC_WINDOW_OFFSET + 0x80000000, // PA_SC_WINDOW_SCISSOR_TL + 0x40004000, // PA_SC_WINDOW_SCISSOR_BR + 0x0000ffff, // PA_SC_CLIPRECT_RULE + 0x00000000, // PA_SC_CLIPRECT_0_TL + 0x40004000, // PA_SC_CLIPRECT_0_BR + 0x00000000, // PA_SC_CLIPRECT_1_TL + 0x40004000, // PA_SC_CLIPRECT_1_BR + 0x00000000, // PA_SC_CLIPRECT_2_TL + 0x40004000, // PA_SC_CLIPRECT_2_BR + 0x00000000, // PA_SC_CLIPRECT_3_TL + 0x40004000, // PA_SC_CLIPRECT_3_BR + 0xaa99aaaa, // PA_SC_EDGERULE + 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET + 0xffffffff, // CB_TARGET_MASK + 0xffffffff, // CB_SHADER_MASK + 0x80000000, // PA_SC_GENERIC_SCISSOR_TL + 0x40004000, // PA_SC_GENERIC_SCISSOR_BR + 0x00000000, // COHER_DEST_BASE_0 + 0x00000000, // COHER_DEST_BASE_1 + 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR + 0x00000000, // PA_SC_VPORT_ZMIN_0 + 0x3f800000, // PA_SC_VPORT_ZMAX_0 + 0x00000000, // PA_SC_VPORT_ZMIN_1 + 0x3f800000, // PA_SC_VPORT_ZMAX_1 + 0x00000000, // PA_SC_VPORT_ZMIN_2 + 0x3f800000, // PA_SC_VPORT_ZMAX_2 + 0x00000000, // PA_SC_VPORT_ZMIN_3 + 0x3f800000, // PA_SC_VPORT_ZMAX_3 + 0x00000000, // PA_SC_VPORT_ZMIN_4 + 0x3f800000, // PA_SC_VPORT_ZMAX_4 + 0x00000000, // PA_SC_VPORT_ZMIN_5 + 0x3f800000, // PA_SC_VPORT_ZMAX_5 + 0x00000000, // PA_SC_VPORT_ZMIN_6 + 0x3f800000, // PA_SC_VPORT_ZMAX_6 + 0x00000000, // PA_SC_VPORT_ZMIN_7 + 0x3f800000, // PA_SC_VPORT_ZMAX_7 + 0x00000000, // PA_SC_VPORT_ZMIN_8 + 0x3f800000, // PA_SC_VPORT_ZMAX_8 + 0x00000000, // PA_SC_VPORT_ZMIN_9 + 0x3f800000, // PA_SC_VPORT_ZMAX_9 + 0x00000000, // PA_SC_VPORT_ZMIN_10 + 0x3f800000, // PA_SC_VPORT_ZMAX_10 + 0x00000000, // PA_SC_VPORT_ZMIN_11 + 0x3f800000, // PA_SC_VPORT_ZMAX_11 + 0x00000000, // PA_SC_VPORT_ZMIN_12 + 0x3f800000, // PA_SC_VPORT_ZMAX_12 + 0x00000000, // PA_SC_VPORT_ZMIN_13 + 0x3f800000, // PA_SC_VPORT_ZMAX_13 + 0x00000000, // PA_SC_VPORT_ZMIN_14 + 0x3f800000, // PA_SC_VPORT_ZMAX_14 + 0x00000000, // PA_SC_VPORT_ZMIN_15 + 0x3f800000, // PA_SC_VPORT_ZMAX_15 +}; +static const unsigned int ci_SECT_CONTEXT_def_2[] = +{ + 0x00000000, // PA_SC_SCREEN_EXTENT_CONTROL + 0, // HOLE + 0x00000000, // CP_PERFMON_CNTX_CNTL + 0x00000000, // CP_RINGID + 0x00000000, // CP_VMID + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0xffffffff, // VGT_MAX_VTX_INDX + 0x00000000, // VGT_MIN_VTX_INDX + 0x00000000, // VGT_INDX_OFFSET + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX + 0, // HOLE + 0x00000000, // CB_BLEND_RED + 0x00000000, // CB_BLEND_GREEN + 0x00000000, // CB_BLEND_BLUE + 0x00000000, // CB_BLEND_ALPHA + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_STENCIL_CONTROL + 0x00000000, // DB_STENCILREFMASK + 0x00000000, // DB_STENCILREFMASK_BF + 0, // HOLE + 0x00000000, // PA_CL_VPORT_XSCALE + 0x00000000, // PA_CL_VPORT_XOFFSET + 0x00000000, // PA_CL_VPORT_YSCALE + 0x00000000, // PA_CL_VPORT_YOFFSET + 0x00000000, // PA_CL_VPORT_ZSCALE + 0x00000000, // PA_CL_VPORT_ZOFFSET + 0x00000000, // PA_CL_VPORT_XSCALE_1 + 0x00000000, // PA_CL_VPORT_XOFFSET_1 + 0x00000000, // PA_CL_VPORT_YSCALE_1 + 0x00000000, // PA_CL_VPORT_YOFFSET_1 + 0x00000000, // PA_CL_VPORT_ZSCALE_1 + 0x00000000, // PA_CL_VPORT_ZOFFSET_1 + 0x00000000, // PA_CL_VPORT_XSCALE_2 + 0x00000000, // PA_CL_VPORT_XOFFSET_2 + 0x00000000, // PA_CL_VPORT_YSCALE_2 + 0x00000000, // PA_CL_VPORT_YOFFSET_2 + 0x00000000, // PA_CL_VPORT_ZSCALE_2 + 0x00000000, // PA_CL_VPORT_ZOFFSET_2 + 0x00000000, // PA_CL_VPORT_XSCALE_3 + 0x00000000, // PA_CL_VPORT_XOFFSET_3 + 0x00000000, // PA_CL_VPORT_YSCALE_3 + 0x00000000, // PA_CL_VPORT_YOFFSET_3 + 0x00000000, // PA_CL_VPORT_ZSCALE_3 + 0x00000000, // PA_CL_VPORT_ZOFFSET_3 + 0x00000000, // PA_CL_VPORT_XSCALE_4 + 0x00000000, // PA_CL_VPORT_XOFFSET_4 + 0x00000000, // PA_CL_VPORT_YSCALE_4 + 0x00000000, // PA_CL_VPORT_YOFFSET_4 + 0x00000000, // PA_CL_VPORT_ZSCALE_4 + 0x00000000, // PA_CL_VPORT_ZOFFSET_4 + 0x00000000, // PA_CL_VPORT_XSCALE_5 + 0x00000000, // PA_CL_VPORT_XOFFSET_5 + 0x00000000, // PA_CL_VPORT_YSCALE_5 + 0x00000000, // PA_CL_VPORT_YOFFSET_5 + 0x00000000, // PA_CL_VPORT_ZSCALE_5 + 0x00000000, // PA_CL_VPORT_ZOFFSET_5 + 0x00000000, // PA_CL_VPORT_XSCALE_6 + 0x00000000, // PA_CL_VPORT_XOFFSET_6 + 0x00000000, // PA_CL_VPORT_YSCALE_6 + 0x00000000, // PA_CL_VPORT_YOFFSET_6 + 0x00000000, // PA_CL_VPORT_ZSCALE_6 + 0x00000000, // PA_CL_VPORT_ZOFFSET_6 + 0x00000000, // PA_CL_VPORT_XSCALE_7 + 0x00000000, // PA_CL_VPORT_XOFFSET_7 + 0x00000000, // PA_CL_VPORT_YSCALE_7 + 0x00000000, // PA_CL_VPORT_YOFFSET_7 + 0x00000000, // PA_CL_VPORT_ZSCALE_7 + 0x00000000, // PA_CL_VPORT_ZOFFSET_7 + 0x00000000, // PA_CL_VPORT_XSCALE_8 + 0x00000000, // PA_CL_VPORT_XOFFSET_8 + 0x00000000, // PA_CL_VPORT_YSCALE_8 + 0x00000000, // PA_CL_VPORT_YOFFSET_8 + 0x00000000, // PA_CL_VPORT_ZSCALE_8 + 0x00000000, // PA_CL_VPORT_ZOFFSET_8 + 0x00000000, // PA_CL_VPORT_XSCALE_9 + 0x00000000, // PA_CL_VPORT_XOFFSET_9 + 0x00000000, // PA_CL_VPORT_YSCALE_9 + 0x00000000, // PA_CL_VPORT_YOFFSET_9 + 0x00000000, // PA_CL_VPORT_ZSCALE_9 + 0x00000000, // PA_CL_VPORT_ZOFFSET_9 + 0x00000000, // PA_CL_VPORT_XSCALE_10 + 0x00000000, // PA_CL_VPORT_XOFFSET_10 + 0x00000000, // PA_CL_VPORT_YSCALE_10 + 0x00000000, // PA_CL_VPORT_YOFFSET_10 + 0x00000000, // PA_CL_VPORT_ZSCALE_10 + 0x00000000, // PA_CL_VPORT_ZOFFSET_10 + 0x00000000, // PA_CL_VPORT_XSCALE_11 + 0x00000000, // PA_CL_VPORT_XOFFSET_11 + 0x00000000, // PA_CL_VPORT_YSCALE_11 + 0x00000000, // PA_CL_VPORT_YOFFSET_11 + 0x00000000, // PA_CL_VPORT_ZSCALE_11 + 0x00000000, // PA_CL_VPORT_ZOFFSET_11 + 0x00000000, // PA_CL_VPORT_XSCALE_12 + 0x00000000, // PA_CL_VPORT_XOFFSET_12 + 0x00000000, // PA_CL_VPORT_YSCALE_12 + 0x00000000, // PA_CL_VPORT_YOFFSET_12 + 0x00000000, // PA_CL_VPORT_ZSCALE_12 + 0x00000000, // PA_CL_VPORT_ZOFFSET_12 + 0x00000000, // PA_CL_VPORT_XSCALE_13 + 0x00000000, // PA_CL_VPORT_XOFFSET_13 + 0x00000000, // PA_CL_VPORT_YSCALE_13 + 0x00000000, // PA_CL_VPORT_YOFFSET_13 + 0x00000000, // PA_CL_VPORT_ZSCALE_13 + 0x00000000, // PA_CL_VPORT_ZOFFSET_13 + 0x00000000, // PA_CL_VPORT_XSCALE_14 + 0x00000000, // PA_CL_VPORT_XOFFSET_14 + 0x00000000, // PA_CL_VPORT_YSCALE_14 + 0x00000000, // PA_CL_VPORT_YOFFSET_14 + 0x00000000, // PA_CL_VPORT_ZSCALE_14 + 0x00000000, // PA_CL_VPORT_ZOFFSET_14 + 0x00000000, // PA_CL_VPORT_XSCALE_15 + 0x00000000, // PA_CL_VPORT_XOFFSET_15 + 0x00000000, // PA_CL_VPORT_YSCALE_15 + 0x00000000, // PA_CL_VPORT_YOFFSET_15 + 0x00000000, // PA_CL_VPORT_ZSCALE_15 + 0x00000000, // PA_CL_VPORT_ZOFFSET_15 + 0x00000000, // PA_CL_UCP_0_X + 0x00000000, // PA_CL_UCP_0_Y + 0x00000000, // PA_CL_UCP_0_Z + 0x00000000, // PA_CL_UCP_0_W + 0x00000000, // PA_CL_UCP_1_X + 0x00000000, // PA_CL_UCP_1_Y + 0x00000000, // PA_CL_UCP_1_Z + 0x00000000, // PA_CL_UCP_1_W + 0x00000000, // PA_CL_UCP_2_X + 0x00000000, // PA_CL_UCP_2_Y + 0x00000000, // PA_CL_UCP_2_Z + 0x00000000, // PA_CL_UCP_2_W + 0x00000000, // PA_CL_UCP_3_X + 0x00000000, // PA_CL_UCP_3_Y + 0x00000000, // PA_CL_UCP_3_Z + 0x00000000, // PA_CL_UCP_3_W + 0x00000000, // PA_CL_UCP_4_X + 0x00000000, // PA_CL_UCP_4_Y + 0x00000000, // PA_CL_UCP_4_Z + 0x00000000, // PA_CL_UCP_4_W + 0x00000000, // PA_CL_UCP_5_X + 0x00000000, // PA_CL_UCP_5_Y + 0x00000000, // PA_CL_UCP_5_Z + 0x00000000, // PA_CL_UCP_5_W + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_CNTL_0 + 0x00000000, // SPI_PS_INPUT_CNTL_1 + 0x00000000, // SPI_PS_INPUT_CNTL_2 + 0x00000000, // SPI_PS_INPUT_CNTL_3 + 0x00000000, // SPI_PS_INPUT_CNTL_4 + 0x00000000, // SPI_PS_INPUT_CNTL_5 + 0x00000000, // SPI_PS_INPUT_CNTL_6 + 0x00000000, // SPI_PS_INPUT_CNTL_7 + 0x00000000, // SPI_PS_INPUT_CNTL_8 + 0x00000000, // SPI_PS_INPUT_CNTL_9 + 0x00000000, // SPI_PS_INPUT_CNTL_10 + 0x00000000, // SPI_PS_INPUT_CNTL_11 + 0x00000000, // SPI_PS_INPUT_CNTL_12 + 0x00000000, // SPI_PS_INPUT_CNTL_13 + 0x00000000, // SPI_PS_INPUT_CNTL_14 + 0x00000000, // SPI_PS_INPUT_CNTL_15 + 0x00000000, // SPI_PS_INPUT_CNTL_16 + 0x00000000, // SPI_PS_INPUT_CNTL_17 + 0x00000000, // SPI_PS_INPUT_CNTL_18 + 0x00000000, // SPI_PS_INPUT_CNTL_19 + 0x00000000, // SPI_PS_INPUT_CNTL_20 + 0x00000000, // SPI_PS_INPUT_CNTL_21 + 0x00000000, // SPI_PS_INPUT_CNTL_22 + 0x00000000, // SPI_PS_INPUT_CNTL_23 + 0x00000000, // SPI_PS_INPUT_CNTL_24 + 0x00000000, // SPI_PS_INPUT_CNTL_25 + 0x00000000, // SPI_PS_INPUT_CNTL_26 + 0x00000000, // SPI_PS_INPUT_CNTL_27 + 0x00000000, // SPI_PS_INPUT_CNTL_28 + 0x00000000, // SPI_PS_INPUT_CNTL_29 + 0x00000000, // SPI_PS_INPUT_CNTL_30 + 0x00000000, // SPI_PS_INPUT_CNTL_31 + 0x00000000, // SPI_VS_OUT_CONFIG + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_ENA + 0x00000000, // SPI_PS_INPUT_ADDR + 0x00000000, // SPI_INTERP_CONTROL_0 + 0x00000002, // SPI_PS_IN_CONTROL + 0, // HOLE + 0x00000000, // SPI_BARYC_CNTL + 0, // HOLE + 0x00000000, // SPI_TMPRING_SIZE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_SHADER_POS_FORMAT + 0x00000000, // SPI_SHADER_Z_FORMAT + 0x00000000, // SPI_SHADER_COL_FORMAT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_BLEND0_CONTROL + 0x00000000, // CB_BLEND1_CONTROL + 0x00000000, // CB_BLEND2_CONTROL + 0x00000000, // CB_BLEND3_CONTROL + 0x00000000, // CB_BLEND4_CONTROL + 0x00000000, // CB_BLEND5_CONTROL + 0x00000000, // CB_BLEND6_CONTROL + 0x00000000, // CB_BLEND7_CONTROL +}; +static const unsigned int ci_SECT_CONTEXT_def_3[] = +{ + 0x00000000, // PA_CL_POINT_X_RAD + 0x00000000, // PA_CL_POINT_Y_RAD + 0x00000000, // PA_CL_POINT_SIZE + 0x00000000, // PA_CL_POINT_CULL_RAD + 0x00000000, // VGT_DMA_BASE_HI + 0x00000000, // VGT_DMA_BASE +}; +static const unsigned int ci_SECT_CONTEXT_def_4[] = +{ + 0x00000000, // DB_DEPTH_CONTROL + 0x00000000, // DB_EQAA + 0x00000000, // CB_COLOR_CONTROL + 0x00000000, // DB_SHADER_CONTROL + 0x00090000, // PA_CL_CLIP_CNTL + 0x00000004, // PA_SU_SC_MODE_CNTL + 0x00000000, // PA_CL_VTE_CNTL + 0x00000000, // PA_CL_VS_OUT_CNTL + 0x00000000, // PA_CL_NANINF_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_SCALE + 0x00000000, // PA_SU_PRIM_FILTER_CNTL + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SU_POINT_SIZE + 0x00000000, // PA_SU_POINT_MINMAX + 0x00000000, // PA_SU_LINE_CNTL + 0x00000000, // PA_SC_LINE_STIPPLE + 0x00000000, // VGT_OUTPUT_PATH_CNTL + 0x00000000, // VGT_HOS_CNTL + 0x00000000, // VGT_HOS_MAX_TESS_LEVEL + 0x00000000, // VGT_HOS_MIN_TESS_LEVEL + 0x00000000, // VGT_HOS_REUSE_DEPTH + 0x00000000, // VGT_GROUP_PRIM_TYPE + 0x00000000, // VGT_GROUP_FIRST_DECR + 0x00000000, // VGT_GROUP_DECR + 0x00000000, // VGT_GROUP_VECT_0_CNTL + 0x00000000, // VGT_GROUP_VECT_1_CNTL + 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL + 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL + 0x00000000, // VGT_GS_MODE + 0x00000000, // VGT_GS_ONCHIP_CNTL + 0x00000000, // PA_SC_MODE_CNTL_0 + 0x00000000, // PA_SC_MODE_CNTL_1 + 0x00000000, // VGT_ENHANCE + 0x00000100, // VGT_GS_PER_ES + 0x00000080, // VGT_ES_PER_GS + 0x00000002, // VGT_GS_PER_VS + 0x00000000, // VGT_GSVS_RING_OFFSET_1 + 0x00000000, // VGT_GSVS_RING_OFFSET_2 + 0x00000000, // VGT_GSVS_RING_OFFSET_3 + 0x00000000, // VGT_GS_OUT_PRIM_TYPE + 0x00000000, // IA_ENHANCE +}; +static const unsigned int ci_SECT_CONTEXT_def_5[] = +{ + 0x00000000, // WD_ENHANCE + 0x00000000, // VGT_PRIMITIVEID_EN +}; +static const unsigned int ci_SECT_CONTEXT_def_6[] = +{ + 0x00000000, // VGT_PRIMITIVEID_RESET +}; +static const unsigned int ci_SECT_CONTEXT_def_7[] = +{ + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_INSTANCE_STEP_RATE_0 + 0x00000000, // VGT_INSTANCE_STEP_RATE_1 + 0x000000ff, // IA_MULTI_VGT_PARAM + 0x00000000, // VGT_ESGS_RING_ITEMSIZE + 0x00000000, // VGT_GSVS_RING_ITEMSIZE + 0x00000000, // VGT_REUSE_OFF + 0x00000000, // VGT_VTX_CNT_EN + 0x00000000, // DB_HTILE_SURFACE + 0x00000000, // DB_SRESULTS_COMPARE_STATE0 + 0x00000000, // DB_SRESULTS_COMPARE_STATE1 + 0x00000000, // DB_PRELOAD_CONTROL + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE + 0, // HOLE + 0x00000000, // VGT_GS_MAX_VERT_OUT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_SHADER_STAGES_EN + 0x00000000, // VGT_LS_HS_CONFIG + 0x00000000, // VGT_GS_VERT_ITEMSIZE + 0x00000000, // VGT_GS_VERT_ITEMSIZE_1 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_2 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_3 + 0x00000000, // VGT_TF_PARAM + 0x00000000, // DB_ALPHA_TO_MASK + 0, // HOLE + 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL + 0x00000000, // PA_SU_POLY_OFFSET_CLAMP + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET + 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET + 0x00000000, // VGT_GS_INSTANCE_CNT + 0x00000000, // VGT_STRMOUT_CONFIG + 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SC_CENTROID_PRIORITY_0 + 0x00000000, // PA_SC_CENTROID_PRIORITY_1 + 0x00001000, // PA_SC_LINE_CNTL + 0x00000000, // PA_SC_AA_CONFIG + 0x00000005, // PA_SU_VTX_CNTL + 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ + 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ + 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ + 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 + 0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0 + 0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL + 0x00000010, // VGT_OUT_DEALLOC_CNTL + 0x00000000, // CB_COLOR0_BASE + 0x00000000, // CB_COLOR0_PITCH + 0x00000000, // CB_COLOR0_SLICE + 0x00000000, // CB_COLOR0_VIEW + 0x00000000, // CB_COLOR0_INFO + 0x00000000, // CB_COLOR0_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR0_CMASK + 0x00000000, // CB_COLOR0_CMASK_SLICE + 0x00000000, // CB_COLOR0_FMASK + 0x00000000, // CB_COLOR0_FMASK_SLICE + 0x00000000, // CB_COLOR0_CLEAR_WORD0 + 0x00000000, // CB_COLOR0_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR1_BASE + 0x00000000, // CB_COLOR1_PITCH + 0x00000000, // CB_COLOR1_SLICE + 0x00000000, // CB_COLOR1_VIEW + 0x00000000, // CB_COLOR1_INFO + 0x00000000, // CB_COLOR1_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR1_CMASK + 0x00000000, // CB_COLOR1_CMASK_SLICE + 0x00000000, // CB_COLOR1_FMASK + 0x00000000, // CB_COLOR1_FMASK_SLICE + 0x00000000, // CB_COLOR1_CLEAR_WORD0 + 0x00000000, // CB_COLOR1_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR2_BASE + 0x00000000, // CB_COLOR2_PITCH + 0x00000000, // CB_COLOR2_SLICE + 0x00000000, // CB_COLOR2_VIEW + 0x00000000, // CB_COLOR2_INFO + 0x00000000, // CB_COLOR2_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR2_CMASK + 0x00000000, // CB_COLOR2_CMASK_SLICE + 0x00000000, // CB_COLOR2_FMASK + 0x00000000, // CB_COLOR2_FMASK_SLICE + 0x00000000, // CB_COLOR2_CLEAR_WORD0 + 0x00000000, // CB_COLOR2_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR3_BASE + 0x00000000, // CB_COLOR3_PITCH + 0x00000000, // CB_COLOR3_SLICE + 0x00000000, // CB_COLOR3_VIEW + 0x00000000, // CB_COLOR3_INFO + 0x00000000, // CB_COLOR3_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR3_CMASK + 0x00000000, // CB_COLOR3_CMASK_SLICE + 0x00000000, // CB_COLOR3_FMASK + 0x00000000, // CB_COLOR3_FMASK_SLICE + 0x00000000, // CB_COLOR3_CLEAR_WORD0 + 0x00000000, // CB_COLOR3_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR4_BASE + 0x00000000, // CB_COLOR4_PITCH + 0x00000000, // CB_COLOR4_SLICE + 0x00000000, // CB_COLOR4_VIEW + 0x00000000, // CB_COLOR4_INFO + 0x00000000, // CB_COLOR4_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR4_CMASK + 0x00000000, // CB_COLOR4_CMASK_SLICE + 0x00000000, // CB_COLOR4_FMASK + 0x00000000, // CB_COLOR4_FMASK_SLICE + 0x00000000, // CB_COLOR4_CLEAR_WORD0 + 0x00000000, // CB_COLOR4_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR5_BASE + 0x00000000, // CB_COLOR5_PITCH + 0x00000000, // CB_COLOR5_SLICE + 0x00000000, // CB_COLOR5_VIEW + 0x00000000, // CB_COLOR5_INFO + 0x00000000, // CB_COLOR5_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR5_CMASK + 0x00000000, // CB_COLOR5_CMASK_SLICE + 0x00000000, // CB_COLOR5_FMASK + 0x00000000, // CB_COLOR5_FMASK_SLICE + 0x00000000, // CB_COLOR5_CLEAR_WORD0 + 0x00000000, // CB_COLOR5_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR6_BASE + 0x00000000, // CB_COLOR6_PITCH + 0x00000000, // CB_COLOR6_SLICE + 0x00000000, // CB_COLOR6_VIEW + 0x00000000, // CB_COLOR6_INFO + 0x00000000, // CB_COLOR6_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR6_CMASK + 0x00000000, // CB_COLOR6_CMASK_SLICE + 0x00000000, // CB_COLOR6_FMASK + 0x00000000, // CB_COLOR6_FMASK_SLICE + 0x00000000, // CB_COLOR6_CLEAR_WORD0 + 0x00000000, // CB_COLOR6_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR7_BASE + 0x00000000, // CB_COLOR7_PITCH + 0x00000000, // CB_COLOR7_SLICE + 0x00000000, // CB_COLOR7_VIEW + 0x00000000, // CB_COLOR7_INFO + 0x00000000, // CB_COLOR7_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR7_CMASK + 0x00000000, // CB_COLOR7_CMASK_SLICE + 0x00000000, // CB_COLOR7_FMASK + 0x00000000, // CB_COLOR7_FMASK_SLICE + 0x00000000, // CB_COLOR7_CLEAR_WORD0 + 0x00000000, // CB_COLOR7_CLEAR_WORD1 +}; +static const struct cs_extent_def ci_SECT_CONTEXT_defs[] = +{ + {ci_SECT_CONTEXT_def_1, 0x0000a000, 212 }, + {ci_SECT_CONTEXT_def_2, 0x0000a0d6, 274 }, + {ci_SECT_CONTEXT_def_3, 0x0000a1f5, 6 }, + {ci_SECT_CONTEXT_def_4, 0x0000a200, 157 }, + {ci_SECT_CONTEXT_def_5, 0x0000a2a0, 2 }, + {ci_SECT_CONTEXT_def_6, 0x0000a2a3, 1 }, + {ci_SECT_CONTEXT_def_7, 0x0000a2a5, 233 }, + { NULL, 0, 0 } +}; +static const struct cs_section_def ci_cs_data[] = { + { ci_SECT_CONTEXT_defs, SECT_CONTEXT }, + { NULL, SECT_NONE } +}; diff --git a/sys/dev/pci/drm/radeon/clearstate_defs.h b/sys/dev/pci/drm/radeon/clearstate_defs.h new file mode 100644 index 00000000000..3eda707d738 --- /dev/null +++ b/sys/dev/pci/drm/radeon/clearstate_defs.h @@ -0,0 +1,44 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef CLEARSTATE_DEFS_H +#define CLEARSTATE_DEFS_H + +enum section_id { + SECT_NONE, + SECT_CONTEXT, + SECT_CLEAR, + SECT_CTRLCONST +}; + +struct cs_extent_def { + const unsigned int *extent; + const unsigned int reg_index; + const unsigned int reg_count; +}; + +struct cs_section_def { + const struct cs_extent_def *section; + const enum section_id id; +}; + +#endif diff --git a/sys/dev/pci/drm/radeon/clearstate_evergreen.h b/sys/dev/pci/drm/radeon/clearstate_evergreen.h new file mode 100644 index 00000000000..63a1ffbb3ce --- /dev/null +++ b/sys/dev/pci/drm/radeon/clearstate_evergreen.h @@ -0,0 +1,1080 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +static const u32 SECT_CONTEXT_def_1[] = +{ + 0x00000000, // DB_RENDER_CONTROL + 0x00000000, // DB_COUNT_CONTROL + 0x00000000, // DB_DEPTH_VIEW + 0x00000000, // DB_RENDER_OVERRIDE + 0x00000000, // DB_RENDER_OVERRIDE2 + 0x00000000, // DB_HTILE_DATA_BASE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_STENCIL_CLEAR + 0x00000000, // DB_DEPTH_CLEAR + 0x00000000, // PA_SC_SCREEN_SCISSOR_TL + 0x40004000, // PA_SC_SCREEN_SCISSOR_BR + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_Z_INFO + 0x00000000, // DB_STENCIL_INFO + 0x00000000, // DB_Z_READ_BASE + 0x00000000, // DB_STENCIL_READ_BASE + 0x00000000, // DB_Z_WRITE_BASE + 0x00000000, // DB_STENCIL_WRITE_BASE + 0x00000000, // DB_DEPTH_SIZE + 0x00000000, // DB_DEPTH_SLICE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_15 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_15 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_15 + 0x00000000, // PA_SC_WINDOW_OFFSET + 0x80000000, // PA_SC_WINDOW_SCISSOR_TL + 0x40004000, // PA_SC_WINDOW_SCISSOR_BR + 0x0000ffff, // PA_SC_CLIPRECT_RULE + 0x00000000, // PA_SC_CLIPRECT_0_TL + 0x40004000, // PA_SC_CLIPRECT_0_BR + 0x00000000, // PA_SC_CLIPRECT_1_TL + 0x40004000, // PA_SC_CLIPRECT_1_BR + 0x00000000, // PA_SC_CLIPRECT_2_TL + 0x40004000, // PA_SC_CLIPRECT_2_BR + 0x00000000, // PA_SC_CLIPRECT_3_TL + 0x40004000, // PA_SC_CLIPRECT_3_BR + 0xaa99aaaa, // PA_SC_EDGERULE + 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET + 0xffffffff, // CB_TARGET_MASK + 0xffffffff, // CB_SHADER_MASK + 0x80000000, // PA_SC_GENERIC_SCISSOR_TL + 0x40004000, // PA_SC_GENERIC_SCISSOR_BR + 0x00000000, // COHER_DEST_BASE_0 + 0x00000000, // COHER_DEST_BASE_1 + 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR + 0x00000000, // PA_SC_VPORT_ZMIN_0 + 0x3f800000, // PA_SC_VPORT_ZMAX_0 + 0x00000000, // PA_SC_VPORT_ZMIN_1 + 0x3f800000, // PA_SC_VPORT_ZMAX_1 + 0x00000000, // PA_SC_VPORT_ZMIN_2 + 0x3f800000, // PA_SC_VPORT_ZMAX_2 + 0x00000000, // PA_SC_VPORT_ZMIN_3 + 0x3f800000, // PA_SC_VPORT_ZMAX_3 + 0x00000000, // PA_SC_VPORT_ZMIN_4 + 0x3f800000, // PA_SC_VPORT_ZMAX_4 + 0x00000000, // PA_SC_VPORT_ZMIN_5 + 0x3f800000, // PA_SC_VPORT_ZMAX_5 + 0x00000000, // PA_SC_VPORT_ZMIN_6 + 0x3f800000, // PA_SC_VPORT_ZMAX_6 + 0x00000000, // PA_SC_VPORT_ZMIN_7 + 0x3f800000, // PA_SC_VPORT_ZMAX_7 + 0x00000000, // PA_SC_VPORT_ZMIN_8 + 0x3f800000, // PA_SC_VPORT_ZMAX_8 + 0x00000000, // PA_SC_VPORT_ZMIN_9 + 0x3f800000, // PA_SC_VPORT_ZMAX_9 + 0x00000000, // PA_SC_VPORT_ZMIN_10 + 0x3f800000, // PA_SC_VPORT_ZMAX_10 + 0x00000000, // PA_SC_VPORT_ZMIN_11 + 0x3f800000, // PA_SC_VPORT_ZMAX_11 + 0x00000000, // PA_SC_VPORT_ZMIN_12 + 0x3f800000, // PA_SC_VPORT_ZMAX_12 + 0x00000000, // PA_SC_VPORT_ZMIN_13 + 0x3f800000, // PA_SC_VPORT_ZMAX_13 + 0x00000000, // PA_SC_VPORT_ZMIN_14 + 0x3f800000, // PA_SC_VPORT_ZMAX_14 + 0x00000000, // PA_SC_VPORT_ZMIN_15 + 0x3f800000, // PA_SC_VPORT_ZMAX_15 + 0x00000000, // SX_MISC + 0x00000000, // SX_SURFACE_SYNC + 0x00000000, // CP_PERFMON_CNTX_CNTL + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_VTX_SEMANTIC_0 + 0x00000000, // SQ_VTX_SEMANTIC_1 + 0x00000000, // SQ_VTX_SEMANTIC_2 + 0x00000000, // SQ_VTX_SEMANTIC_3 + 0x00000000, // SQ_VTX_SEMANTIC_4 + 0x00000000, // SQ_VTX_SEMANTIC_5 + 0x00000000, // SQ_VTX_SEMANTIC_6 + 0x00000000, // SQ_VTX_SEMANTIC_7 + 0x00000000, // SQ_VTX_SEMANTIC_8 + 0x00000000, // SQ_VTX_SEMANTIC_9 + 0x00000000, // SQ_VTX_SEMANTIC_10 + 0x00000000, // SQ_VTX_SEMANTIC_11 + 0x00000000, // SQ_VTX_SEMANTIC_12 + 0x00000000, // SQ_VTX_SEMANTIC_13 + 0x00000000, // SQ_VTX_SEMANTIC_14 + 0x00000000, // SQ_VTX_SEMANTIC_15 + 0x00000000, // SQ_VTX_SEMANTIC_16 + 0x00000000, // SQ_VTX_SEMANTIC_17 + 0x00000000, // SQ_VTX_SEMANTIC_18 + 0x00000000, // SQ_VTX_SEMANTIC_19 + 0x00000000, // SQ_VTX_SEMANTIC_20 + 0x00000000, // SQ_VTX_SEMANTIC_21 + 0x00000000, // SQ_VTX_SEMANTIC_22 + 0x00000000, // SQ_VTX_SEMANTIC_23 + 0x00000000, // SQ_VTX_SEMANTIC_24 + 0x00000000, // SQ_VTX_SEMANTIC_25 + 0x00000000, // SQ_VTX_SEMANTIC_26 + 0x00000000, // SQ_VTX_SEMANTIC_27 + 0x00000000, // SQ_VTX_SEMANTIC_28 + 0x00000000, // SQ_VTX_SEMANTIC_29 + 0x00000000, // SQ_VTX_SEMANTIC_30 + 0x00000000, // SQ_VTX_SEMANTIC_31 + 0xffffffff, // VGT_MAX_VTX_INDX + 0x00000000, // VGT_MIN_VTX_INDX + 0x00000000, // VGT_INDX_OFFSET + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX + 0x00000000, // SX_ALPHA_TEST_CONTROL + 0x00000000, // CB_BLEND_RED + 0x00000000, // CB_BLEND_GREEN + 0x00000000, // CB_BLEND_BLUE + 0x00000000, // CB_BLEND_ALPHA + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_STENCILREFMASK + 0x00000000, // DB_STENCILREFMASK_BF + 0x00000000, // SX_ALPHA_REF + 0x00000000, // PA_CL_VPORT_XSCALE + 0x00000000, // PA_CL_VPORT_XOFFSET + 0x00000000, // PA_CL_VPORT_YSCALE + 0x00000000, // PA_CL_VPORT_YOFFSET + 0x00000000, // PA_CL_VPORT_ZSCALE + 0x00000000, // PA_CL_VPORT_ZOFFSET + 0x00000000, // PA_CL_VPORT_XSCALE_1 + 0x00000000, // PA_CL_VPORT_XOFFSET_1 + 0x00000000, // PA_CL_VPORT_YSCALE_1 + 0x00000000, // PA_CL_VPORT_YOFFSET_1 + 0x00000000, // PA_CL_VPORT_ZSCALE_1 + 0x00000000, // PA_CL_VPORT_ZOFFSET_1 + 0x00000000, // PA_CL_VPORT_XSCALE_2 + 0x00000000, // PA_CL_VPORT_XOFFSET_2 + 0x00000000, // PA_CL_VPORT_YSCALE_2 + 0x00000000, // PA_CL_VPORT_YOFFSET_2 + 0x00000000, // PA_CL_VPORT_ZSCALE_2 + 0x00000000, // PA_CL_VPORT_ZOFFSET_2 + 0x00000000, // PA_CL_VPORT_XSCALE_3 + 0x00000000, // PA_CL_VPORT_XOFFSET_3 + 0x00000000, // PA_CL_VPORT_YSCALE_3 + 0x00000000, // PA_CL_VPORT_YOFFSET_3 + 0x00000000, // PA_CL_VPORT_ZSCALE_3 + 0x00000000, // PA_CL_VPORT_ZOFFSET_3 + 0x00000000, // PA_CL_VPORT_XSCALE_4 + 0x00000000, // PA_CL_VPORT_XOFFSET_4 + 0x00000000, // PA_CL_VPORT_YSCALE_4 + 0x00000000, // PA_CL_VPORT_YOFFSET_4 + 0x00000000, // PA_CL_VPORT_ZSCALE_4 + 0x00000000, // PA_CL_VPORT_ZOFFSET_4 + 0x00000000, // PA_CL_VPORT_XSCALE_5 + 0x00000000, // PA_CL_VPORT_XOFFSET_5 + 0x00000000, // PA_CL_VPORT_YSCALE_5 + 0x00000000, // PA_CL_VPORT_YOFFSET_5 + 0x00000000, // PA_CL_VPORT_ZSCALE_5 + 0x00000000, // PA_CL_VPORT_ZOFFSET_5 + 0x00000000, // PA_CL_VPORT_XSCALE_6 + 0x00000000, // PA_CL_VPORT_XOFFSET_6 + 0x00000000, // PA_CL_VPORT_YSCALE_6 + 0x00000000, // PA_CL_VPORT_YOFFSET_6 + 0x00000000, // PA_CL_VPORT_ZSCALE_6 + 0x00000000, // PA_CL_VPORT_ZOFFSET_6 + 0x00000000, // PA_CL_VPORT_XSCALE_7 + 0x00000000, // PA_CL_VPORT_XOFFSET_7 + 0x00000000, // PA_CL_VPORT_YSCALE_7 + 0x00000000, // PA_CL_VPORT_YOFFSET_7 + 0x00000000, // PA_CL_VPORT_ZSCALE_7 + 0x00000000, // PA_CL_VPORT_ZOFFSET_7 + 0x00000000, // PA_CL_VPORT_XSCALE_8 + 0x00000000, // PA_CL_VPORT_XOFFSET_8 + 0x00000000, // PA_CL_VPORT_YSCALE_8 + 0x00000000, // PA_CL_VPORT_YOFFSET_8 + 0x00000000, // PA_CL_VPORT_ZSCALE_8 + 0x00000000, // PA_CL_VPORT_ZOFFSET_8 + 0x00000000, // PA_CL_VPORT_XSCALE_9 + 0x00000000, // PA_CL_VPORT_XOFFSET_9 + 0x00000000, // PA_CL_VPORT_YSCALE_9 + 0x00000000, // PA_CL_VPORT_YOFFSET_9 + 0x00000000, // PA_CL_VPORT_ZSCALE_9 + 0x00000000, // PA_CL_VPORT_ZOFFSET_9 + 0x00000000, // PA_CL_VPORT_XSCALE_10 + 0x00000000, // PA_CL_VPORT_XOFFSET_10 + 0x00000000, // PA_CL_VPORT_YSCALE_10 + 0x00000000, // PA_CL_VPORT_YOFFSET_10 + 0x00000000, // PA_CL_VPORT_ZSCALE_10 + 0x00000000, // PA_CL_VPORT_ZOFFSET_10 + 0x00000000, // PA_CL_VPORT_XSCALE_11 + 0x00000000, // PA_CL_VPORT_XOFFSET_11 + 0x00000000, // PA_CL_VPORT_YSCALE_11 + 0x00000000, // PA_CL_VPORT_YOFFSET_11 + 0x00000000, // PA_CL_VPORT_ZSCALE_11 + 0x00000000, // PA_CL_VPORT_ZOFFSET_11 + 0x00000000, // PA_CL_VPORT_XSCALE_12 + 0x00000000, // PA_CL_VPORT_XOFFSET_12 + 0x00000000, // PA_CL_VPORT_YSCALE_12 + 0x00000000, // PA_CL_VPORT_YOFFSET_12 + 0x00000000, // PA_CL_VPORT_ZSCALE_12 + 0x00000000, // PA_CL_VPORT_ZOFFSET_12 + 0x00000000, // PA_CL_VPORT_XSCALE_13 + 0x00000000, // PA_CL_VPORT_XOFFSET_13 + 0x00000000, // PA_CL_VPORT_YSCALE_13 + 0x00000000, // PA_CL_VPORT_YOFFSET_13 + 0x00000000, // PA_CL_VPORT_ZSCALE_13 + 0x00000000, // PA_CL_VPORT_ZOFFSET_13 + 0x00000000, // PA_CL_VPORT_XSCALE_14 + 0x00000000, // PA_CL_VPORT_XOFFSET_14 + 0x00000000, // PA_CL_VPORT_YSCALE_14 + 0x00000000, // PA_CL_VPORT_YOFFSET_14 + 0x00000000, // PA_CL_VPORT_ZSCALE_14 + 0x00000000, // PA_CL_VPORT_ZOFFSET_14 + 0x00000000, // PA_CL_VPORT_XSCALE_15 + 0x00000000, // PA_CL_VPORT_XOFFSET_15 + 0x00000000, // PA_CL_VPORT_YSCALE_15 + 0x00000000, // PA_CL_VPORT_YOFFSET_15 + 0x00000000, // PA_CL_VPORT_ZSCALE_15 + 0x00000000, // PA_CL_VPORT_ZOFFSET_15 + 0x00000000, // PA_CL_UCP_0_X + 0x00000000, // PA_CL_UCP_0_Y + 0x00000000, // PA_CL_UCP_0_Z + 0x00000000, // PA_CL_UCP_0_W + 0x00000000, // PA_CL_UCP_1_X + 0x00000000, // PA_CL_UCP_1_Y + 0x00000000, // PA_CL_UCP_1_Z + 0x00000000, // PA_CL_UCP_1_W + 0x00000000, // PA_CL_UCP_2_X + 0x00000000, // PA_CL_UCP_2_Y + 0x00000000, // PA_CL_UCP_2_Z + 0x00000000, // PA_CL_UCP_2_W + 0x00000000, // PA_CL_UCP_3_X + 0x00000000, // PA_CL_UCP_3_Y + 0x00000000, // PA_CL_UCP_3_Z + 0x00000000, // PA_CL_UCP_3_W + 0x00000000, // PA_CL_UCP_4_X + 0x00000000, // PA_CL_UCP_4_Y + 0x00000000, // PA_CL_UCP_4_Z + 0x00000000, // PA_CL_UCP_4_W + 0x00000000, // PA_CL_UCP_5_X + 0x00000000, // PA_CL_UCP_5_Y + 0x00000000, // PA_CL_UCP_5_Z + 0x00000000, // PA_CL_UCP_5_W + 0x00000000, // SPI_VS_OUT_ID_0 + 0x00000000, // SPI_VS_OUT_ID_1 + 0x00000000, // SPI_VS_OUT_ID_2 + 0x00000000, // SPI_VS_OUT_ID_3 + 0x00000000, // SPI_VS_OUT_ID_4 + 0x00000000, // SPI_VS_OUT_ID_5 + 0x00000000, // SPI_VS_OUT_ID_6 + 0x00000000, // SPI_VS_OUT_ID_7 + 0x00000000, // SPI_VS_OUT_ID_8 + 0x00000000, // SPI_VS_OUT_ID_9 + 0x00000000, // SPI_PS_INPUT_CNTL_0 + 0x00000000, // SPI_PS_INPUT_CNTL_1 + 0x00000000, // SPI_PS_INPUT_CNTL_2 + 0x00000000, // SPI_PS_INPUT_CNTL_3 + 0x00000000, // SPI_PS_INPUT_CNTL_4 + 0x00000000, // SPI_PS_INPUT_CNTL_5 + 0x00000000, // SPI_PS_INPUT_CNTL_6 + 0x00000000, // SPI_PS_INPUT_CNTL_7 + 0x00000000, // SPI_PS_INPUT_CNTL_8 + 0x00000000, // SPI_PS_INPUT_CNTL_9 + 0x00000000, // SPI_PS_INPUT_CNTL_10 + 0x00000000, // SPI_PS_INPUT_CNTL_11 + 0x00000000, // SPI_PS_INPUT_CNTL_12 + 0x00000000, // SPI_PS_INPUT_CNTL_13 + 0x00000000, // SPI_PS_INPUT_CNTL_14 + 0x00000000, // SPI_PS_INPUT_CNTL_15 + 0x00000000, // SPI_PS_INPUT_CNTL_16 + 0x00000000, // SPI_PS_INPUT_CNTL_17 + 0x00000000, // SPI_PS_INPUT_CNTL_18 + 0x00000000, // SPI_PS_INPUT_CNTL_19 + 0x00000000, // SPI_PS_INPUT_CNTL_20 + 0x00000000, // SPI_PS_INPUT_CNTL_21 + 0x00000000, // SPI_PS_INPUT_CNTL_22 + 0x00000000, // SPI_PS_INPUT_CNTL_23 + 0x00000000, // SPI_PS_INPUT_CNTL_24 + 0x00000000, // SPI_PS_INPUT_CNTL_25 + 0x00000000, // SPI_PS_INPUT_CNTL_26 + 0x00000000, // SPI_PS_INPUT_CNTL_27 + 0x00000000, // SPI_PS_INPUT_CNTL_28 + 0x00000000, // SPI_PS_INPUT_CNTL_29 + 0x00000000, // SPI_PS_INPUT_CNTL_30 + 0x00000000, // SPI_PS_INPUT_CNTL_31 + 0x00000000, // SPI_VS_OUT_CONFIG + 0x00000001, // SPI_THREAD_GROUPING + 0x00000000, // SPI_PS_IN_CONTROL_0 + 0x00000000, // SPI_PS_IN_CONTROL_1 + 0x00000000, // SPI_INTERP_CONTROL_0 + 0x00000000, // SPI_INPUT_Z + 0x00000000, // SPI_FOG_CNTL + 0x00000000, // SPI_BARYC_CNTL + 0x00000000, // SPI_PS_IN_CONTROL_2 + 0x00000000, // SPI_COMPUTE_INPUT_CNTL + 0x00000000, // SPI_COMPUTE_NUM_THREAD_X + 0x00000000, // SPI_COMPUTE_NUM_THREAD_Y + 0x00000000, // SPI_COMPUTE_NUM_THREAD_Z + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // GDS_ADDR_BASE + 0x00003fff, // GDS_ADDR_SIZE + 0x00000001, // GDS_ORDERED_WAVE_PER_SE + 0x00000000, // GDS_APPEND_CONSUME_UAV0 + 0x00000000, // GDS_APPEND_CONSUME_UAV1 + 0x00000000, // GDS_APPEND_CONSUME_UAV2 + 0x00000000, // GDS_APPEND_CONSUME_UAV3 + 0x00000000, // GDS_APPEND_CONSUME_UAV4 + 0x00000000, // GDS_APPEND_CONSUME_UAV5 + 0x00000000, // GDS_APPEND_CONSUME_UAV6 + 0x00000000, // GDS_APPEND_CONSUME_UAV7 + 0x00000000, // GDS_APPEND_CONSUME_UAV8 + 0x00000000, // GDS_APPEND_CONSUME_UAV9 + 0x00000000, // GDS_APPEND_CONSUME_UAV10 + 0x00000000, // GDS_APPEND_CONSUME_UAV11 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_BLEND0_CONTROL + 0x00000000, // CB_BLEND1_CONTROL + 0x00000000, // CB_BLEND2_CONTROL + 0x00000000, // CB_BLEND3_CONTROL + 0x00000000, // CB_BLEND4_CONTROL + 0x00000000, // CB_BLEND5_CONTROL + 0x00000000, // CB_BLEND6_CONTROL + 0x00000000, // CB_BLEND7_CONTROL +}; +static const u32 SECT_CONTEXT_def_2[] = +{ + 0x00000000, // PA_CL_POINT_X_RAD + 0x00000000, // PA_CL_POINT_Y_RAD + 0x00000000, // PA_CL_POINT_SIZE + 0x00000000, // PA_CL_POINT_CULL_RAD + 0x00000000, // VGT_DMA_BASE_HI + 0x00000000, // VGT_DMA_BASE +}; +static const u32 SECT_CONTEXT_def_3[] = +{ + 0x00000000, // DB_DEPTH_CONTROL + 0, // HOLE + 0x00000000, // CB_COLOR_CONTROL + 0x00000200, // DB_SHADER_CONTROL + 0x00000000, // PA_CL_CLIP_CNTL + 0x00000000, // PA_SU_SC_MODE_CNTL + 0x00000000, // PA_CL_VTE_CNTL + 0x00000000, // PA_CL_VS_OUT_CNTL + 0x00000000, // PA_CL_NANINF_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_SCALE + 0x00000000, // PA_SU_PRIM_FILTER_CNTL + 0x00000000, // SQ_LSTMP_RING_ITEMSIZE + 0x00000000, // SQ_HSTMP_RING_ITEMSIZE + 0x00000000, // SQ_DYN_GPR_RESOURCE_LIMIT_1 + 0, // HOLE + 0x00000000, // SQ_PGM_START_PS + 0x00000000, // SQ_PGM_RESOURCES_PS + 0x00000000, // SQ_PGM_RESOURCES_2_PS + 0x00000000, // SQ_PGM_EXPORTS_PS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_VS + 0x00000000, // SQ_PGM_RESOURCES_VS + 0x00000000, // SQ_PGM_RESOURCES_2_VS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_GS + 0x00000000, // SQ_PGM_RESOURCES_GS + 0x00000000, // SQ_PGM_RESOURCES_2_GS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_ES + 0x00000000, // SQ_PGM_RESOURCES_ES + 0x00000000, // SQ_PGM_RESOURCES_2_ES + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_FS + 0x00000000, // SQ_PGM_RESOURCES_FS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_HS + 0x00000000, // SQ_PGM_RESOURCES_HS + 0x00000000, // SQ_PGM_RESOURCES_2_HS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_PGM_START_LS + 0x00000000, // SQ_PGM_RESOURCES_LS + 0x00000000, // SQ_PGM_RESOURCES_2_LS +}; +static const u32 SECT_CONTEXT_def_4[] = +{ + 0x00000000, // SQ_LDS_ALLOC + 0x00000000, // SQ_LDS_ALLOC_PS + 0x00000000, // SQ_VTX_SEMANTIC_CLEAR + 0, // HOLE + 0x00000000, // SQ_THREAD_TRACE_CTRL + 0, // HOLE + 0x00000000, // SQ_ESGS_RING_ITEMSIZE + 0x00000000, // SQ_GSVS_RING_ITEMSIZE + 0x00000000, // SQ_ESTMP_RING_ITEMSIZE + 0x00000000, // SQ_GSTMP_RING_ITEMSIZE + 0x00000000, // SQ_VSTMP_RING_ITEMSIZE + 0x00000000, // SQ_PSTMP_RING_ITEMSIZE + 0, // HOLE + 0x00000000, // SQ_GS_VERT_ITEMSIZE + 0x00000000, // SQ_GS_VERT_ITEMSIZE_1 + 0x00000000, // SQ_GS_VERT_ITEMSIZE_2 + 0x00000000, // SQ_GS_VERT_ITEMSIZE_3 + 0x00000000, // SQ_GSVS_RING_OFFSET_1 + 0x00000000, // SQ_GSVS_RING_OFFSET_2 + 0x00000000, // SQ_GSVS_RING_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_ALU_CONST_CACHE_PS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_PS_15 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_VS_15 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_GS_15 + 0x00000000, // PA_SU_POINT_SIZE + 0x00000000, // PA_SU_POINT_MINMAX + 0x00000000, // PA_SU_LINE_CNTL + 0x00000000, // PA_SC_LINE_STIPPLE + 0x00000000, // VGT_OUTPUT_PATH_CNTL + 0x00000000, // VGT_HOS_CNTL + 0x00000000, // VGT_HOS_MAX_TESS_LEVEL + 0x00000000, // VGT_HOS_MIN_TESS_LEVEL + 0x00000000, // VGT_HOS_REUSE_DEPTH + 0x00000000, // VGT_GROUP_PRIM_TYPE + 0x00000000, // VGT_GROUP_FIRST_DECR + 0x00000000, // VGT_GROUP_DECR + 0x00000000, // VGT_GROUP_VECT_0_CNTL + 0x00000000, // VGT_GROUP_VECT_1_CNTL + 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL + 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL + 0x00000000, // VGT_GS_MODE + 0, // HOLE + 0x00000000, // PA_SC_MODE_CNTL_0 + 0x00000000, // PA_SC_MODE_CNTL_1 + 0x00000000, // VGT_ENHANCE + 0x00000000, // VGT_GS_PER_ES + 0x00000000, // VGT_ES_PER_GS + 0x00000000, // VGT_GS_PER_VS + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_GS_OUT_PRIM_TYPE +}; +static const u32 SECT_CONTEXT_def_5[] = +{ + 0x00000000, // VGT_DMA_MAX_SIZE + 0x00000000, // VGT_DMA_INDEX_TYPE + 0, // HOLE + 0x00000000, // VGT_PRIMITIVEID_EN + 0x00000000, // VGT_DMA_NUM_INSTANCES +}; +static const u32 SECT_CONTEXT_def_6[] = +{ + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_INSTANCE_STEP_RATE_0 + 0x00000000, // VGT_INSTANCE_STEP_RATE_1 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_REUSE_OFF + 0x00000000, // VGT_VTX_CNT_EN + 0x00000000, // DB_HTILE_SURFACE + 0x00000000, // DB_SRESULTS_COMPARE_STATE0 + 0x00000000, // DB_SRESULTS_COMPARE_STATE1 + 0x00000000, // DB_PRELOAD_CONTROL + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0 + 0x00000000, // VGT_STRMOUT_BUFFER_BASE_0 + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1 + 0x00000000, // VGT_STRMOUT_BUFFER_BASE_1 + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2 + 0x00000000, // VGT_STRMOUT_BUFFER_BASE_2 + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3 + 0x00000000, // VGT_STRMOUT_BUFFER_BASE_3 + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE + 0, // HOLE + 0x00000000, // VGT_GS_MAX_VERT_OUT + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_0 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_1 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_2 + 0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_3 + 0x00000000, // VGT_SHADER_STAGES_EN + 0x00000000, // VGT_LS_HS_CONFIG + 0x00000000, // VGT_LS_SIZE + 0x00000000, // VGT_HS_SIZE + 0x00000000, // VGT_LS_HS_ALLOC + 0x00000000, // VGT_HS_PATCH_CONST + 0x00000000, // VGT_TF_PARAM + 0x00000000, // DB_ALPHA_TO_MASK +}; +static const u32 SECT_CONTEXT_def_7[] = +{ + 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL + 0x00000000, // PA_SU_POLY_OFFSET_CLAMP + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET + 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET + 0x00000000, // VGT_GS_INSTANCE_CNT + 0x00000000, // VGT_STRMOUT_CONFIG + 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG + 0x00000000, // CB_IMMED0_BASE + 0x00000000, // CB_IMMED1_BASE + 0x00000000, // CB_IMMED2_BASE + 0x00000000, // CB_IMMED3_BASE + 0x00000000, // CB_IMMED4_BASE + 0x00000000, // CB_IMMED5_BASE + 0x00000000, // CB_IMMED6_BASE + 0x00000000, // CB_IMMED7_BASE + 0x00000000, // CB_IMMED8_BASE + 0x00000000, // CB_IMMED9_BASE + 0x00000000, // CB_IMMED10_BASE + 0x00000000, // CB_IMMED11_BASE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00001000, // PA_SC_LINE_CNTL + 0x00000000, // PA_SC_AA_CONFIG + 0x00000005, // PA_SU_VTX_CNTL + 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ + 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ + 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ + 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_4 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_5 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_6 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_7 + 0xffffffff, // PA_SC_AA_MASK + 0x00000000, // CB_CLRCMP_CONTROL + 0x00000000, // CB_CLRCMP_SRC + 0x00000000, // CB_CLRCMP_DST + 0x00000000, // CB_CLRCMP_MSK + 0, // HOLE + 0, // HOLE + 0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL + 0x00000010, // VGT_OUT_DEALLOC_CNTL + 0x00000000, // CB_COLOR0_BASE + 0x00000000, // CB_COLOR0_PITCH + 0x00000000, // CB_COLOR0_SLICE + 0x00000000, // CB_COLOR0_VIEW + 0x00000000, // CB_COLOR0_INFO + 0x00000000, // CB_COLOR0_ATTRIB + 0x00000000, // CB_COLOR0_DIM + 0x00000000, // CB_COLOR0_CMASK + 0x00000000, // CB_COLOR0_CMASK_SLICE + 0x00000000, // CB_COLOR0_FMASK + 0x00000000, // CB_COLOR0_FMASK_SLICE + 0x00000000, // CB_COLOR0_CLEAR_WORD0 + 0x00000000, // CB_COLOR0_CLEAR_WORD1 + 0x00000000, // CB_COLOR0_CLEAR_WORD2 + 0x00000000, // CB_COLOR0_CLEAR_WORD3 + 0x00000000, // CB_COLOR1_BASE + 0x00000000, // CB_COLOR1_PITCH + 0x00000000, // CB_COLOR1_SLICE + 0x00000000, // CB_COLOR1_VIEW + 0x00000000, // CB_COLOR1_INFO + 0x00000000, // CB_COLOR1_ATTRIB + 0x00000000, // CB_COLOR1_DIM + 0x00000000, // CB_COLOR1_CMASK + 0x00000000, // CB_COLOR1_CMASK_SLICE + 0x00000000, // CB_COLOR1_FMASK + 0x00000000, // CB_COLOR1_FMASK_SLICE + 0x00000000, // CB_COLOR1_CLEAR_WORD0 + 0x00000000, // CB_COLOR1_CLEAR_WORD1 + 0x00000000, // CB_COLOR1_CLEAR_WORD2 + 0x00000000, // CB_COLOR1_CLEAR_WORD3 + 0x00000000, // CB_COLOR2_BASE + 0x00000000, // CB_COLOR2_PITCH + 0x00000000, // CB_COLOR2_SLICE + 0x00000000, // CB_COLOR2_VIEW + 0x00000000, // CB_COLOR2_INFO + 0x00000000, // CB_COLOR2_ATTRIB + 0x00000000, // CB_COLOR2_DIM + 0x00000000, // CB_COLOR2_CMASK + 0x00000000, // CB_COLOR2_CMASK_SLICE + 0x00000000, // CB_COLOR2_FMASK + 0x00000000, // CB_COLOR2_FMASK_SLICE + 0x00000000, // CB_COLOR2_CLEAR_WORD0 + 0x00000000, // CB_COLOR2_CLEAR_WORD1 + 0x00000000, // CB_COLOR2_CLEAR_WORD2 + 0x00000000, // CB_COLOR2_CLEAR_WORD3 + 0x00000000, // CB_COLOR3_BASE + 0x00000000, // CB_COLOR3_PITCH + 0x00000000, // CB_COLOR3_SLICE + 0x00000000, // CB_COLOR3_VIEW + 0x00000000, // CB_COLOR3_INFO + 0x00000000, // CB_COLOR3_ATTRIB + 0x00000000, // CB_COLOR3_DIM + 0x00000000, // CB_COLOR3_CMASK + 0x00000000, // CB_COLOR3_CMASK_SLICE + 0x00000000, // CB_COLOR3_FMASK + 0x00000000, // CB_COLOR3_FMASK_SLICE + 0x00000000, // CB_COLOR3_CLEAR_WORD0 + 0x00000000, // CB_COLOR3_CLEAR_WORD1 + 0x00000000, // CB_COLOR3_CLEAR_WORD2 + 0x00000000, // CB_COLOR3_CLEAR_WORD3 + 0x00000000, // CB_COLOR4_BASE + 0x00000000, // CB_COLOR4_PITCH + 0x00000000, // CB_COLOR4_SLICE + 0x00000000, // CB_COLOR4_VIEW + 0x00000000, // CB_COLOR4_INFO + 0x00000000, // CB_COLOR4_ATTRIB + 0x00000000, // CB_COLOR4_DIM + 0x00000000, // CB_COLOR4_CMASK + 0x00000000, // CB_COLOR4_CMASK_SLICE + 0x00000000, // CB_COLOR4_FMASK + 0x00000000, // CB_COLOR4_FMASK_SLICE + 0x00000000, // CB_COLOR4_CLEAR_WORD0 + 0x00000000, // CB_COLOR4_CLEAR_WORD1 + 0x00000000, // CB_COLOR4_CLEAR_WORD2 + 0x00000000, // CB_COLOR4_CLEAR_WORD3 + 0x00000000, // CB_COLOR5_BASE + 0x00000000, // CB_COLOR5_PITCH + 0x00000000, // CB_COLOR5_SLICE + 0x00000000, // CB_COLOR5_VIEW + 0x00000000, // CB_COLOR5_INFO + 0x00000000, // CB_COLOR5_ATTRIB + 0x00000000, // CB_COLOR5_DIM + 0x00000000, // CB_COLOR5_CMASK + 0x00000000, // CB_COLOR5_CMASK_SLICE + 0x00000000, // CB_COLOR5_FMASK + 0x00000000, // CB_COLOR5_FMASK_SLICE + 0x00000000, // CB_COLOR5_CLEAR_WORD0 + 0x00000000, // CB_COLOR5_CLEAR_WORD1 + 0x00000000, // CB_COLOR5_CLEAR_WORD2 + 0x00000000, // CB_COLOR5_CLEAR_WORD3 + 0x00000000, // CB_COLOR6_BASE + 0x00000000, // CB_COLOR6_PITCH + 0x00000000, // CB_COLOR6_SLICE + 0x00000000, // CB_COLOR6_VIEW + 0x00000000, // CB_COLOR6_INFO + 0x00000000, // CB_COLOR6_ATTRIB + 0x00000000, // CB_COLOR6_DIM + 0x00000000, // CB_COLOR6_CMASK + 0x00000000, // CB_COLOR6_CMASK_SLICE + 0x00000000, // CB_COLOR6_FMASK + 0x00000000, // CB_COLOR6_FMASK_SLICE + 0x00000000, // CB_COLOR6_CLEAR_WORD0 + 0x00000000, // CB_COLOR6_CLEAR_WORD1 + 0x00000000, // CB_COLOR6_CLEAR_WORD2 + 0x00000000, // CB_COLOR6_CLEAR_WORD3 + 0x00000000, // CB_COLOR7_BASE + 0x00000000, // CB_COLOR7_PITCH + 0x00000000, // CB_COLOR7_SLICE + 0x00000000, // CB_COLOR7_VIEW + 0x00000000, // CB_COLOR7_INFO + 0x00000000, // CB_COLOR7_ATTRIB + 0x00000000, // CB_COLOR7_DIM + 0x00000000, // CB_COLOR7_CMASK + 0x00000000, // CB_COLOR7_CMASK_SLICE + 0x00000000, // CB_COLOR7_FMASK + 0x00000000, // CB_COLOR7_FMASK_SLICE + 0x00000000, // CB_COLOR7_CLEAR_WORD0 + 0x00000000, // CB_COLOR7_CLEAR_WORD1 + 0x00000000, // CB_COLOR7_CLEAR_WORD2 + 0x00000000, // CB_COLOR7_CLEAR_WORD3 + 0x00000000, // CB_COLOR8_BASE + 0x00000000, // CB_COLOR8_PITCH + 0x00000000, // CB_COLOR8_SLICE + 0x00000000, // CB_COLOR8_VIEW + 0x00000000, // CB_COLOR8_INFO + 0x00000000, // CB_COLOR8_ATTRIB + 0x00000000, // CB_COLOR8_DIM + 0x00000000, // CB_COLOR9_BASE + 0x00000000, // CB_COLOR9_PITCH + 0x00000000, // CB_COLOR9_SLICE + 0x00000000, // CB_COLOR9_VIEW + 0x00000000, // CB_COLOR9_INFO + 0x00000000, // CB_COLOR9_ATTRIB + 0x00000000, // CB_COLOR9_DIM + 0x00000000, // CB_COLOR10_BASE + 0x00000000, // CB_COLOR10_PITCH + 0x00000000, // CB_COLOR10_SLICE + 0x00000000, // CB_COLOR10_VIEW + 0x00000000, // CB_COLOR10_INFO + 0x00000000, // CB_COLOR10_ATTRIB + 0x00000000, // CB_COLOR10_DIM + 0x00000000, // CB_COLOR11_BASE + 0x00000000, // CB_COLOR11_PITCH + 0x00000000, // CB_COLOR11_SLICE + 0x00000000, // CB_COLOR11_VIEW + 0x00000000, // CB_COLOR11_INFO + 0x00000000, // CB_COLOR11_ATTRIB + 0x00000000, // CB_COLOR11_DIM + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SQ_ALU_CONST_CACHE_HS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_HS_15 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_0 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_1 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_2 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_3 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_4 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_5 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_6 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_7 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_8 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_9 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_10 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_11 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_12 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_13 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_14 + 0x00000000, // SQ_ALU_CONST_CACHE_LS_15 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_15 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_0 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_1 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_2 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_3 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_4 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_5 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_6 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_7 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_8 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_9 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_10 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_11 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_12 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_13 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_14 + 0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_15 +}; +static const struct cs_extent_def SECT_CONTEXT_defs[] = +{ + {SECT_CONTEXT_def_1, 0x0000a000, 488 }, + {SECT_CONTEXT_def_2, 0x0000a1f5, 6 }, + {SECT_CONTEXT_def_3, 0x0000a200, 55 }, + {SECT_CONTEXT_def_4, 0x0000a23a, 98 }, + {SECT_CONTEXT_def_5, 0x0000a29e, 5 }, + {SECT_CONTEXT_def_6, 0x0000a2a5, 56 }, + {SECT_CONTEXT_def_7, 0x0000a2de, 290 }, + { 0, 0, 0 } +}; +static const u32 SECT_CLEAR_def_1[] = +{ + 0xffffffff, // SQ_TEX_SAMPLER_CLEAR + 0xffffffff, // SQ_TEX_RESOURCE_CLEAR + 0xffffffff, // SQ_LOOP_BOOL_CLEAR +}; +static const struct cs_extent_def SECT_CLEAR_defs[] = +{ + {SECT_CLEAR_def_1, 0x0000ffc0, 3 }, + { 0, 0, 0 } +}; +static const u32 SECT_CTRLCONST_def_1[] = +{ + 0x00000000, // SQ_VTX_BASE_VTX_LOC + 0x00000000, // SQ_VTX_START_INST_LOC +}; +static const struct cs_extent_def SECT_CTRLCONST_defs[] = +{ + {SECT_CTRLCONST_def_1, 0x0000f3fc, 2 }, + { 0, 0, 0 } +}; +static const struct cs_section_def evergreen_cs_data[] = { + { SECT_CONTEXT_defs, SECT_CONTEXT }, + { SECT_CLEAR_defs, SECT_CLEAR }, + { SECT_CTRLCONST_defs, SECT_CTRLCONST }, + { 0, SECT_NONE } +}; diff --git a/sys/dev/pci/drm/radeon/clearstate_si.h b/sys/dev/pci/drm/radeon/clearstate_si.h new file mode 100644 index 00000000000..66e39cdb5cb --- /dev/null +++ b/sys/dev/pci/drm/radeon/clearstate_si.h @@ -0,0 +1,941 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +static const u32 si_SECT_CONTEXT_def_1[] = +{ + 0x00000000, // DB_RENDER_CONTROL + 0x00000000, // DB_COUNT_CONTROL + 0x00000000, // DB_DEPTH_VIEW + 0x00000000, // DB_RENDER_OVERRIDE + 0x00000000, // DB_RENDER_OVERRIDE2 + 0x00000000, // DB_HTILE_DATA_BASE + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_DEPTH_BOUNDS_MIN + 0x00000000, // DB_DEPTH_BOUNDS_MAX + 0x00000000, // DB_STENCIL_CLEAR + 0x00000000, // DB_DEPTH_CLEAR + 0x00000000, // PA_SC_SCREEN_SCISSOR_TL + 0x40004000, // PA_SC_SCREEN_SCISSOR_BR + 0, // HOLE + 0x00000000, // DB_DEPTH_INFO + 0x00000000, // DB_Z_INFO + 0x00000000, // DB_STENCIL_INFO + 0x00000000, // DB_Z_READ_BASE + 0x00000000, // DB_STENCIL_READ_BASE + 0x00000000, // DB_Z_WRITE_BASE + 0x00000000, // DB_STENCIL_WRITE_BASE + 0x00000000, // DB_DEPTH_SIZE + 0x00000000, // DB_DEPTH_SLICE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // TA_BC_BASE_ADDR + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // COHER_DEST_BASE_2 + 0x00000000, // COHER_DEST_BASE_3 + 0x00000000, // PA_SC_WINDOW_OFFSET + 0x80000000, // PA_SC_WINDOW_SCISSOR_TL + 0x40004000, // PA_SC_WINDOW_SCISSOR_BR + 0x0000ffff, // PA_SC_CLIPRECT_RULE + 0x00000000, // PA_SC_CLIPRECT_0_TL + 0x40004000, // PA_SC_CLIPRECT_0_BR + 0x00000000, // PA_SC_CLIPRECT_1_TL + 0x40004000, // PA_SC_CLIPRECT_1_BR + 0x00000000, // PA_SC_CLIPRECT_2_TL + 0x40004000, // PA_SC_CLIPRECT_2_BR + 0x00000000, // PA_SC_CLIPRECT_3_TL + 0x40004000, // PA_SC_CLIPRECT_3_BR + 0xaa99aaaa, // PA_SC_EDGERULE + 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET + 0xffffffff, // CB_TARGET_MASK + 0xffffffff, // CB_SHADER_MASK + 0x80000000, // PA_SC_GENERIC_SCISSOR_TL + 0x40004000, // PA_SC_GENERIC_SCISSOR_BR + 0x00000000, // COHER_DEST_BASE_0 + 0x00000000, // COHER_DEST_BASE_1 + 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR + 0x00000000, // PA_SC_VPORT_ZMIN_0 + 0x3f800000, // PA_SC_VPORT_ZMAX_0 + 0x00000000, // PA_SC_VPORT_ZMIN_1 + 0x3f800000, // PA_SC_VPORT_ZMAX_1 + 0x00000000, // PA_SC_VPORT_ZMIN_2 + 0x3f800000, // PA_SC_VPORT_ZMAX_2 + 0x00000000, // PA_SC_VPORT_ZMIN_3 + 0x3f800000, // PA_SC_VPORT_ZMAX_3 + 0x00000000, // PA_SC_VPORT_ZMIN_4 + 0x3f800000, // PA_SC_VPORT_ZMAX_4 + 0x00000000, // PA_SC_VPORT_ZMIN_5 + 0x3f800000, // PA_SC_VPORT_ZMAX_5 + 0x00000000, // PA_SC_VPORT_ZMIN_6 + 0x3f800000, // PA_SC_VPORT_ZMAX_6 + 0x00000000, // PA_SC_VPORT_ZMIN_7 + 0x3f800000, // PA_SC_VPORT_ZMAX_7 + 0x00000000, // PA_SC_VPORT_ZMIN_8 + 0x3f800000, // PA_SC_VPORT_ZMAX_8 + 0x00000000, // PA_SC_VPORT_ZMIN_9 + 0x3f800000, // PA_SC_VPORT_ZMAX_9 + 0x00000000, // PA_SC_VPORT_ZMIN_10 + 0x3f800000, // PA_SC_VPORT_ZMAX_10 + 0x00000000, // PA_SC_VPORT_ZMIN_11 + 0x3f800000, // PA_SC_VPORT_ZMAX_11 + 0x00000000, // PA_SC_VPORT_ZMIN_12 + 0x3f800000, // PA_SC_VPORT_ZMAX_12 + 0x00000000, // PA_SC_VPORT_ZMIN_13 + 0x3f800000, // PA_SC_VPORT_ZMAX_13 + 0x00000000, // PA_SC_VPORT_ZMIN_14 + 0x3f800000, // PA_SC_VPORT_ZMAX_14 + 0x00000000, // PA_SC_VPORT_ZMIN_15 + 0x3f800000, // PA_SC_VPORT_ZMAX_15 +}; +static const u32 si_SECT_CONTEXT_def_2[] = +{ + 0x00000000, // CP_PERFMON_CNTX_CNTL + 0x00000000, // CP_RINGID + 0x00000000, // CP_VMID + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0xffffffff, // VGT_MAX_VTX_INDX + 0x00000000, // VGT_MIN_VTX_INDX + 0x00000000, // VGT_INDX_OFFSET + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX + 0, // HOLE + 0x00000000, // CB_BLEND_RED + 0x00000000, // CB_BLEND_GREEN + 0x00000000, // CB_BLEND_BLUE + 0x00000000, // CB_BLEND_ALPHA + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_STENCIL_CONTROL + 0x00000000, // DB_STENCILREFMASK + 0x00000000, // DB_STENCILREFMASK_BF + 0, // HOLE + 0x00000000, // PA_CL_VPORT_XSCALE + 0x00000000, // PA_CL_VPORT_XOFFSET + 0x00000000, // PA_CL_VPORT_YSCALE + 0x00000000, // PA_CL_VPORT_YOFFSET + 0x00000000, // PA_CL_VPORT_ZSCALE + 0x00000000, // PA_CL_VPORT_ZOFFSET + 0x00000000, // PA_CL_VPORT_XSCALE_1 + 0x00000000, // PA_CL_VPORT_XOFFSET_1 + 0x00000000, // PA_CL_VPORT_YSCALE_1 + 0x00000000, // PA_CL_VPORT_YOFFSET_1 + 0x00000000, // PA_CL_VPORT_ZSCALE_1 + 0x00000000, // PA_CL_VPORT_ZOFFSET_1 + 0x00000000, // PA_CL_VPORT_XSCALE_2 + 0x00000000, // PA_CL_VPORT_XOFFSET_2 + 0x00000000, // PA_CL_VPORT_YSCALE_2 + 0x00000000, // PA_CL_VPORT_YOFFSET_2 + 0x00000000, // PA_CL_VPORT_ZSCALE_2 + 0x00000000, // PA_CL_VPORT_ZOFFSET_2 + 0x00000000, // PA_CL_VPORT_XSCALE_3 + 0x00000000, // PA_CL_VPORT_XOFFSET_3 + 0x00000000, // PA_CL_VPORT_YSCALE_3 + 0x00000000, // PA_CL_VPORT_YOFFSET_3 + 0x00000000, // PA_CL_VPORT_ZSCALE_3 + 0x00000000, // PA_CL_VPORT_ZOFFSET_3 + 0x00000000, // PA_CL_VPORT_XSCALE_4 + 0x00000000, // PA_CL_VPORT_XOFFSET_4 + 0x00000000, // PA_CL_VPORT_YSCALE_4 + 0x00000000, // PA_CL_VPORT_YOFFSET_4 + 0x00000000, // PA_CL_VPORT_ZSCALE_4 + 0x00000000, // PA_CL_VPORT_ZOFFSET_4 + 0x00000000, // PA_CL_VPORT_XSCALE_5 + 0x00000000, // PA_CL_VPORT_XOFFSET_5 + 0x00000000, // PA_CL_VPORT_YSCALE_5 + 0x00000000, // PA_CL_VPORT_YOFFSET_5 + 0x00000000, // PA_CL_VPORT_ZSCALE_5 + 0x00000000, // PA_CL_VPORT_ZOFFSET_5 + 0x00000000, // PA_CL_VPORT_XSCALE_6 + 0x00000000, // PA_CL_VPORT_XOFFSET_6 + 0x00000000, // PA_CL_VPORT_YSCALE_6 + 0x00000000, // PA_CL_VPORT_YOFFSET_6 + 0x00000000, // PA_CL_VPORT_ZSCALE_6 + 0x00000000, // PA_CL_VPORT_ZOFFSET_6 + 0x00000000, // PA_CL_VPORT_XSCALE_7 + 0x00000000, // PA_CL_VPORT_XOFFSET_7 + 0x00000000, // PA_CL_VPORT_YSCALE_7 + 0x00000000, // PA_CL_VPORT_YOFFSET_7 + 0x00000000, // PA_CL_VPORT_ZSCALE_7 + 0x00000000, // PA_CL_VPORT_ZOFFSET_7 + 0x00000000, // PA_CL_VPORT_XSCALE_8 + 0x00000000, // PA_CL_VPORT_XOFFSET_8 + 0x00000000, // PA_CL_VPORT_YSCALE_8 + 0x00000000, // PA_CL_VPORT_YOFFSET_8 + 0x00000000, // PA_CL_VPORT_ZSCALE_8 + 0x00000000, // PA_CL_VPORT_ZOFFSET_8 + 0x00000000, // PA_CL_VPORT_XSCALE_9 + 0x00000000, // PA_CL_VPORT_XOFFSET_9 + 0x00000000, // PA_CL_VPORT_YSCALE_9 + 0x00000000, // PA_CL_VPORT_YOFFSET_9 + 0x00000000, // PA_CL_VPORT_ZSCALE_9 + 0x00000000, // PA_CL_VPORT_ZOFFSET_9 + 0x00000000, // PA_CL_VPORT_XSCALE_10 + 0x00000000, // PA_CL_VPORT_XOFFSET_10 + 0x00000000, // PA_CL_VPORT_YSCALE_10 + 0x00000000, // PA_CL_VPORT_YOFFSET_10 + 0x00000000, // PA_CL_VPORT_ZSCALE_10 + 0x00000000, // PA_CL_VPORT_ZOFFSET_10 + 0x00000000, // PA_CL_VPORT_XSCALE_11 + 0x00000000, // PA_CL_VPORT_XOFFSET_11 + 0x00000000, // PA_CL_VPORT_YSCALE_11 + 0x00000000, // PA_CL_VPORT_YOFFSET_11 + 0x00000000, // PA_CL_VPORT_ZSCALE_11 + 0x00000000, // PA_CL_VPORT_ZOFFSET_11 + 0x00000000, // PA_CL_VPORT_XSCALE_12 + 0x00000000, // PA_CL_VPORT_XOFFSET_12 + 0x00000000, // PA_CL_VPORT_YSCALE_12 + 0x00000000, // PA_CL_VPORT_YOFFSET_12 + 0x00000000, // PA_CL_VPORT_ZSCALE_12 + 0x00000000, // PA_CL_VPORT_ZOFFSET_12 + 0x00000000, // PA_CL_VPORT_XSCALE_13 + 0x00000000, // PA_CL_VPORT_XOFFSET_13 + 0x00000000, // PA_CL_VPORT_YSCALE_13 + 0x00000000, // PA_CL_VPORT_YOFFSET_13 + 0x00000000, // PA_CL_VPORT_ZSCALE_13 + 0x00000000, // PA_CL_VPORT_ZOFFSET_13 + 0x00000000, // PA_CL_VPORT_XSCALE_14 + 0x00000000, // PA_CL_VPORT_XOFFSET_14 + 0x00000000, // PA_CL_VPORT_YSCALE_14 + 0x00000000, // PA_CL_VPORT_YOFFSET_14 + 0x00000000, // PA_CL_VPORT_ZSCALE_14 + 0x00000000, // PA_CL_VPORT_ZOFFSET_14 + 0x00000000, // PA_CL_VPORT_XSCALE_15 + 0x00000000, // PA_CL_VPORT_XOFFSET_15 + 0x00000000, // PA_CL_VPORT_YSCALE_15 + 0x00000000, // PA_CL_VPORT_YOFFSET_15 + 0x00000000, // PA_CL_VPORT_ZSCALE_15 + 0x00000000, // PA_CL_VPORT_ZOFFSET_15 + 0x00000000, // PA_CL_UCP_0_X + 0x00000000, // PA_CL_UCP_0_Y + 0x00000000, // PA_CL_UCP_0_Z + 0x00000000, // PA_CL_UCP_0_W + 0x00000000, // PA_CL_UCP_1_X + 0x00000000, // PA_CL_UCP_1_Y + 0x00000000, // PA_CL_UCP_1_Z + 0x00000000, // PA_CL_UCP_1_W + 0x00000000, // PA_CL_UCP_2_X + 0x00000000, // PA_CL_UCP_2_Y + 0x00000000, // PA_CL_UCP_2_Z + 0x00000000, // PA_CL_UCP_2_W + 0x00000000, // PA_CL_UCP_3_X + 0x00000000, // PA_CL_UCP_3_Y + 0x00000000, // PA_CL_UCP_3_Z + 0x00000000, // PA_CL_UCP_3_W + 0x00000000, // PA_CL_UCP_4_X + 0x00000000, // PA_CL_UCP_4_Y + 0x00000000, // PA_CL_UCP_4_Z + 0x00000000, // PA_CL_UCP_4_W + 0x00000000, // PA_CL_UCP_5_X + 0x00000000, // PA_CL_UCP_5_Y + 0x00000000, // PA_CL_UCP_5_Z + 0x00000000, // PA_CL_UCP_5_W + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_CNTL_0 + 0x00000000, // SPI_PS_INPUT_CNTL_1 + 0x00000000, // SPI_PS_INPUT_CNTL_2 + 0x00000000, // SPI_PS_INPUT_CNTL_3 + 0x00000000, // SPI_PS_INPUT_CNTL_4 + 0x00000000, // SPI_PS_INPUT_CNTL_5 + 0x00000000, // SPI_PS_INPUT_CNTL_6 + 0x00000000, // SPI_PS_INPUT_CNTL_7 + 0x00000000, // SPI_PS_INPUT_CNTL_8 + 0x00000000, // SPI_PS_INPUT_CNTL_9 + 0x00000000, // SPI_PS_INPUT_CNTL_10 + 0x00000000, // SPI_PS_INPUT_CNTL_11 + 0x00000000, // SPI_PS_INPUT_CNTL_12 + 0x00000000, // SPI_PS_INPUT_CNTL_13 + 0x00000000, // SPI_PS_INPUT_CNTL_14 + 0x00000000, // SPI_PS_INPUT_CNTL_15 + 0x00000000, // SPI_PS_INPUT_CNTL_16 + 0x00000000, // SPI_PS_INPUT_CNTL_17 + 0x00000000, // SPI_PS_INPUT_CNTL_18 + 0x00000000, // SPI_PS_INPUT_CNTL_19 + 0x00000000, // SPI_PS_INPUT_CNTL_20 + 0x00000000, // SPI_PS_INPUT_CNTL_21 + 0x00000000, // SPI_PS_INPUT_CNTL_22 + 0x00000000, // SPI_PS_INPUT_CNTL_23 + 0x00000000, // SPI_PS_INPUT_CNTL_24 + 0x00000000, // SPI_PS_INPUT_CNTL_25 + 0x00000000, // SPI_PS_INPUT_CNTL_26 + 0x00000000, // SPI_PS_INPUT_CNTL_27 + 0x00000000, // SPI_PS_INPUT_CNTL_28 + 0x00000000, // SPI_PS_INPUT_CNTL_29 + 0x00000000, // SPI_PS_INPUT_CNTL_30 + 0x00000000, // SPI_PS_INPUT_CNTL_31 + 0x00000000, // SPI_VS_OUT_CONFIG + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_ENA + 0x00000000, // SPI_PS_INPUT_ADDR + 0x00000000, // SPI_INTERP_CONTROL_0 + 0x00000002, // SPI_PS_IN_CONTROL + 0, // HOLE + 0x00000000, // SPI_BARYC_CNTL + 0, // HOLE + 0x00000000, // SPI_TMPRING_SIZE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_WAVE_MGMT_1 + 0x00000000, // SPI_WAVE_MGMT_2 + 0x00000000, // SPI_SHADER_POS_FORMAT + 0x00000000, // SPI_SHADER_Z_FORMAT + 0x00000000, // SPI_SHADER_COL_FORMAT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_BLEND0_CONTROL + 0x00000000, // CB_BLEND1_CONTROL + 0x00000000, // CB_BLEND2_CONTROL + 0x00000000, // CB_BLEND3_CONTROL + 0x00000000, // CB_BLEND4_CONTROL + 0x00000000, // CB_BLEND5_CONTROL + 0x00000000, // CB_BLEND6_CONTROL + 0x00000000, // CB_BLEND7_CONTROL +}; +static const u32 si_SECT_CONTEXT_def_3[] = +{ + 0x00000000, // PA_CL_POINT_X_RAD + 0x00000000, // PA_CL_POINT_Y_RAD + 0x00000000, // PA_CL_POINT_SIZE + 0x00000000, // PA_CL_POINT_CULL_RAD + 0x00000000, // VGT_DMA_BASE_HI + 0x00000000, // VGT_DMA_BASE +}; +static const u32 si_SECT_CONTEXT_def_4[] = +{ + 0x00000000, // DB_DEPTH_CONTROL + 0x00000000, // DB_EQAA + 0x00000000, // CB_COLOR_CONTROL + 0x00000000, // DB_SHADER_CONTROL + 0x00090000, // PA_CL_CLIP_CNTL + 0x00000004, // PA_SU_SC_MODE_CNTL + 0x00000000, // PA_CL_VTE_CNTL + 0x00000000, // PA_CL_VS_OUT_CNTL + 0x00000000, // PA_CL_NANINF_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_SCALE + 0x00000000, // PA_SU_PRIM_FILTER_CNTL + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SU_POINT_SIZE + 0x00000000, // PA_SU_POINT_MINMAX + 0x00000000, // PA_SU_LINE_CNTL + 0x00000000, // PA_SC_LINE_STIPPLE + 0x00000000, // VGT_OUTPUT_PATH_CNTL + 0x00000000, // VGT_HOS_CNTL + 0x00000000, // VGT_HOS_MAX_TESS_LEVEL + 0x00000000, // VGT_HOS_MIN_TESS_LEVEL + 0x00000000, // VGT_HOS_REUSE_DEPTH + 0x00000000, // VGT_GROUP_PRIM_TYPE + 0x00000000, // VGT_GROUP_FIRST_DECR + 0x00000000, // VGT_GROUP_DECR + 0x00000000, // VGT_GROUP_VECT_0_CNTL + 0x00000000, // VGT_GROUP_VECT_1_CNTL + 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL + 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL + 0x00000000, // VGT_GS_MODE + 0, // HOLE + 0x00000000, // PA_SC_MODE_CNTL_0 + 0x00000000, // PA_SC_MODE_CNTL_1 + 0x00000000, // VGT_ENHANCE + 0x00000100, // VGT_GS_PER_ES + 0x00000080, // VGT_ES_PER_GS + 0x00000002, // VGT_GS_PER_VS + 0x00000000, // VGT_GSVS_RING_OFFSET_1 + 0x00000000, // VGT_GSVS_RING_OFFSET_2 + 0x00000000, // VGT_GSVS_RING_OFFSET_3 + 0x00000000, // VGT_GS_OUT_PRIM_TYPE + 0x00000000, // IA_ENHANCE +}; +static const u32 si_SECT_CONTEXT_def_5[] = +{ + 0x00000000, // VGT_PRIMITIVEID_EN +}; +static const u32 si_SECT_CONTEXT_def_6[] = +{ + 0x00000000, // VGT_PRIMITIVEID_RESET +}; +static const u32 si_SECT_CONTEXT_def_7[] = +{ + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_INSTANCE_STEP_RATE_0 + 0x00000000, // VGT_INSTANCE_STEP_RATE_1 + 0x000000ff, // IA_MULTI_VGT_PARAM + 0x00000000, // VGT_ESGS_RING_ITEMSIZE + 0x00000000, // VGT_GSVS_RING_ITEMSIZE + 0x00000000, // VGT_REUSE_OFF + 0x00000000, // VGT_VTX_CNT_EN + 0x00000000, // DB_HTILE_SURFACE + 0x00000000, // DB_SRESULTS_COMPARE_STATE0 + 0x00000000, // DB_SRESULTS_COMPARE_STATE1 + 0x00000000, // DB_PRELOAD_CONTROL + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE + 0, // HOLE + 0x00000000, // VGT_GS_MAX_VERT_OUT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_SHADER_STAGES_EN + 0x00000000, // VGT_LS_HS_CONFIG + 0x00000000, // VGT_GS_VERT_ITEMSIZE + 0x00000000, // VGT_GS_VERT_ITEMSIZE_1 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_2 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_3 + 0x00000000, // VGT_TF_PARAM + 0x00000000, // DB_ALPHA_TO_MASK + 0, // HOLE + 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL + 0x00000000, // PA_SU_POLY_OFFSET_CLAMP + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET + 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET + 0x00000000, // VGT_GS_INSTANCE_CNT + 0x00000000, // VGT_STRMOUT_CONFIG + 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SC_CENTROID_PRIORITY_0 + 0x00000000, // PA_SC_CENTROID_PRIORITY_1 + 0x00001000, // PA_SC_LINE_CNTL + 0x00000000, // PA_SC_AA_CONFIG + 0x00000005, // PA_SU_VTX_CNTL + 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ + 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ + 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ + 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 + 0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0 + 0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL + 0x00000010, // VGT_OUT_DEALLOC_CNTL + 0x00000000, // CB_COLOR0_BASE + 0x00000000, // CB_COLOR0_PITCH + 0x00000000, // CB_COLOR0_SLICE + 0x00000000, // CB_COLOR0_VIEW + 0x00000000, // CB_COLOR0_INFO + 0x00000000, // CB_COLOR0_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR0_CMASK + 0x00000000, // CB_COLOR0_CMASK_SLICE + 0x00000000, // CB_COLOR0_FMASK + 0x00000000, // CB_COLOR0_FMASK_SLICE + 0x00000000, // CB_COLOR0_CLEAR_WORD0 + 0x00000000, // CB_COLOR0_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR1_BASE + 0x00000000, // CB_COLOR1_PITCH + 0x00000000, // CB_COLOR1_SLICE + 0x00000000, // CB_COLOR1_VIEW + 0x00000000, // CB_COLOR1_INFO + 0x00000000, // CB_COLOR1_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR1_CMASK + 0x00000000, // CB_COLOR1_CMASK_SLICE + 0x00000000, // CB_COLOR1_FMASK + 0x00000000, // CB_COLOR1_FMASK_SLICE + 0x00000000, // CB_COLOR1_CLEAR_WORD0 + 0x00000000, // CB_COLOR1_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR2_BASE + 0x00000000, // CB_COLOR2_PITCH + 0x00000000, // CB_COLOR2_SLICE + 0x00000000, // CB_COLOR2_VIEW + 0x00000000, // CB_COLOR2_INFO + 0x00000000, // CB_COLOR2_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR2_CMASK + 0x00000000, // CB_COLOR2_CMASK_SLICE + 0x00000000, // CB_COLOR2_FMASK + 0x00000000, // CB_COLOR2_FMASK_SLICE + 0x00000000, // CB_COLOR2_CLEAR_WORD0 + 0x00000000, // CB_COLOR2_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR3_BASE + 0x00000000, // CB_COLOR3_PITCH + 0x00000000, // CB_COLOR3_SLICE + 0x00000000, // CB_COLOR3_VIEW + 0x00000000, // CB_COLOR3_INFO + 0x00000000, // CB_COLOR3_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR3_CMASK + 0x00000000, // CB_COLOR3_CMASK_SLICE + 0x00000000, // CB_COLOR3_FMASK + 0x00000000, // CB_COLOR3_FMASK_SLICE + 0x00000000, // CB_COLOR3_CLEAR_WORD0 + 0x00000000, // CB_COLOR3_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR4_BASE + 0x00000000, // CB_COLOR4_PITCH + 0x00000000, // CB_COLOR4_SLICE + 0x00000000, // CB_COLOR4_VIEW + 0x00000000, // CB_COLOR4_INFO + 0x00000000, // CB_COLOR4_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR4_CMASK + 0x00000000, // CB_COLOR4_CMASK_SLICE + 0x00000000, // CB_COLOR4_FMASK + 0x00000000, // CB_COLOR4_FMASK_SLICE + 0x00000000, // CB_COLOR4_CLEAR_WORD0 + 0x00000000, // CB_COLOR4_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR5_BASE + 0x00000000, // CB_COLOR5_PITCH + 0x00000000, // CB_COLOR5_SLICE + 0x00000000, // CB_COLOR5_VIEW + 0x00000000, // CB_COLOR5_INFO + 0x00000000, // CB_COLOR5_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR5_CMASK + 0x00000000, // CB_COLOR5_CMASK_SLICE + 0x00000000, // CB_COLOR5_FMASK + 0x00000000, // CB_COLOR5_FMASK_SLICE + 0x00000000, // CB_COLOR5_CLEAR_WORD0 + 0x00000000, // CB_COLOR5_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR6_BASE + 0x00000000, // CB_COLOR6_PITCH + 0x00000000, // CB_COLOR6_SLICE + 0x00000000, // CB_COLOR6_VIEW + 0x00000000, // CB_COLOR6_INFO + 0x00000000, // CB_COLOR6_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR6_CMASK + 0x00000000, // CB_COLOR6_CMASK_SLICE + 0x00000000, // CB_COLOR6_FMASK + 0x00000000, // CB_COLOR6_FMASK_SLICE + 0x00000000, // CB_COLOR6_CLEAR_WORD0 + 0x00000000, // CB_COLOR6_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR7_BASE + 0x00000000, // CB_COLOR7_PITCH + 0x00000000, // CB_COLOR7_SLICE + 0x00000000, // CB_COLOR7_VIEW + 0x00000000, // CB_COLOR7_INFO + 0x00000000, // CB_COLOR7_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR7_CMASK + 0x00000000, // CB_COLOR7_CMASK_SLICE + 0x00000000, // CB_COLOR7_FMASK + 0x00000000, // CB_COLOR7_FMASK_SLICE + 0x00000000, // CB_COLOR7_CLEAR_WORD0 + 0x00000000, // CB_COLOR7_CLEAR_WORD1 +}; +static const struct cs_extent_def si_SECT_CONTEXT_defs[] = +{ + {si_SECT_CONTEXT_def_1, 0x0000a000, 212 }, + {si_SECT_CONTEXT_def_2, 0x0000a0d8, 272 }, + {si_SECT_CONTEXT_def_3, 0x0000a1f5, 6 }, + {si_SECT_CONTEXT_def_4, 0x0000a200, 157 }, + {si_SECT_CONTEXT_def_5, 0x0000a2a1, 1 }, + {si_SECT_CONTEXT_def_6, 0x0000a2a3, 1 }, + {si_SECT_CONTEXT_def_7, 0x0000a2a5, 233 }, + { NULL, 0, 0 } +}; +static const struct cs_section_def si_cs_data[] = { + { si_SECT_CONTEXT_defs, SECT_CONTEXT }, + { NULL, SECT_NONE } +}; diff --git a/sys/dev/pci/drm/radeon/cypress_dpm.c b/sys/dev/pci/drm/radeon/cypress_dpm.c new file mode 100644 index 00000000000..6a18da7c757 --- /dev/null +++ b/sys/dev/pci/drm/radeon/cypress_dpm.c @@ -0,0 +1,2165 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "evergreend.h" +#include "r600_dpm.h" +#include "cypress_dpm.h" +#include "atom.h" + +#define SMC_RAM_END 0x8000 + +#define MC_CG_ARB_FREQ_F0 0x0a +#define MC_CG_ARB_FREQ_F1 0x0b +#define MC_CG_ARB_FREQ_F2 0x0c +#define MC_CG_ARB_FREQ_F3 0x0d + +#define MC_CG_SEQ_DRAMCONF_S0 0x05 +#define MC_CG_SEQ_DRAMCONF_S1 0x06 +#define MC_CG_SEQ_YCLK_SUSPEND 0x04 +#define MC_CG_SEQ_YCLK_RESUME 0x0a + +struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps); +struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev); +struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev); + +static void cypress_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 tmp, bif; + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + if (enable) { + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) && + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) { + if (!pi->boot_in_gen2) { + bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK; + bif |= CG_CLIENT_REQ(0xd); + WREG32(CG_BIF_REQ_AND_RSP, bif); + + tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK; + tmp |= LC_HW_VOLTAGE_IF_CONTROL(1); + tmp |= LC_GEN2_EN_STRAP; + + tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + udelay(10); + tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + } + } + } else { + if (!pi->boot_in_gen2) { + tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK; + tmp &= ~LC_GEN2_EN_STRAP; + } + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) || + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + } +} + +static void cypress_enable_dynamic_pcie_gen2(struct radeon_device *rdev, + bool enable) +{ + cypress_enable_bif_dynamic_pcie_gen2(rdev, enable); + + if (enable) + WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE); + else + WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE); +} + +#if 0 +static int cypress_enter_ulp_state(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (pi->gfx_clock_gating) { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN); + WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON); + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON); + + RREG32(GB_ADDR_CONFIG); + } + + WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower), + ~HOST_SMC_MSG_MASK); + + udelay(7000); + + return 0; +} +#endif + +static void cypress_gfx_clock_gating_enable(struct radeon_device *rdev, + bool enable) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (enable) { + if (eg_pi->light_sleep) { + WREG32(GRBM_GFX_INDEX, 0xC0000000); + + WREG32_CG(CG_CGLS_TILE_0, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_1, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_2, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_3, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_4, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_5, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_6, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_7, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_8, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_9, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_10, 0xFFFFFFFF); + WREG32_CG(CG_CGLS_TILE_11, 0xFFFFFFFF); + + WREG32_P(SCLK_PWRMGT_CNTL, DYN_LIGHT_SLEEP_EN, ~DYN_LIGHT_SLEEP_EN); + } + WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN); + } else { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN); + WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON); + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON); + RREG32(GB_ADDR_CONFIG); + + if (eg_pi->light_sleep) { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_LIGHT_SLEEP_EN); + + WREG32(GRBM_GFX_INDEX, 0xC0000000); + + WREG32_CG(CG_CGLS_TILE_0, 0); + WREG32_CG(CG_CGLS_TILE_1, 0); + WREG32_CG(CG_CGLS_TILE_2, 0); + WREG32_CG(CG_CGLS_TILE_3, 0); + WREG32_CG(CG_CGLS_TILE_4, 0); + WREG32_CG(CG_CGLS_TILE_5, 0); + WREG32_CG(CG_CGLS_TILE_6, 0); + WREG32_CG(CG_CGLS_TILE_7, 0); + WREG32_CG(CG_CGLS_TILE_8, 0); + WREG32_CG(CG_CGLS_TILE_9, 0); + WREG32_CG(CG_CGLS_TILE_10, 0); + WREG32_CG(CG_CGLS_TILE_11, 0); + } + } +} + +static void cypress_mg_clock_gating_enable(struct radeon_device *rdev, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (enable) { + u32 cgts_sm_ctrl_reg; + + if (rdev->family == CHIP_CEDAR) + cgts_sm_ctrl_reg = CEDAR_MGCGCGTSSMCTRL_DFLT; + else if (rdev->family == CHIP_REDWOOD) + cgts_sm_ctrl_reg = REDWOOD_MGCGCGTSSMCTRL_DFLT; + else + cgts_sm_ctrl_reg = CYPRESS_MGCGCGTSSMCTRL_DFLT; + + WREG32(GRBM_GFX_INDEX, 0xC0000000); + + WREG32_CG(CG_CGTT_LOCAL_0, CYPRESS_MGCGTTLOCAL0_DFLT); + WREG32_CG(CG_CGTT_LOCAL_1, CYPRESS_MGCGTTLOCAL1_DFLT & 0xFFFFCFFF); + WREG32_CG(CG_CGTT_LOCAL_2, CYPRESS_MGCGTTLOCAL2_DFLT); + WREG32_CG(CG_CGTT_LOCAL_3, CYPRESS_MGCGTTLOCAL3_DFLT); + + if (pi->mgcgtssm) + WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg); + + if (eg_pi->mcls) { + WREG32_P(MC_CITF_MISC_RD_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE); + WREG32_P(MC_CITF_MISC_WR_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE); + WREG32_P(MC_CITF_MISC_VM_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE); + WREG32_P(MC_HUB_MISC_HUB_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE); + WREG32_P(MC_HUB_MISC_VM_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE); + WREG32_P(MC_HUB_MISC_SIP_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE); + WREG32_P(MC_XPB_CLK_GAT, MEM_LS_ENABLE, ~MEM_LS_ENABLE); + WREG32_P(VM_L2_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE); + } + } else { + WREG32(GRBM_GFX_INDEX, 0xC0000000); + + WREG32_CG(CG_CGTT_LOCAL_0, 0xFFFFFFFF); + WREG32_CG(CG_CGTT_LOCAL_1, 0xFFFFFFFF); + WREG32_CG(CG_CGTT_LOCAL_2, 0xFFFFFFFF); + WREG32_CG(CG_CGTT_LOCAL_3, 0xFFFFFFFF); + + if (pi->mgcgtssm) + WREG32(CGTS_SM_CTRL_REG, 0x81f44bc0); + } +} + +void cypress_enable_spread_spectrum(struct radeon_device *rdev, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (enable) { + if (pi->sclk_ss) + WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN); + + if (pi->mclk_ss) + WREG32_P(MPLL_CNTL_MODE, SS_SSEN, ~SS_SSEN); + } else { + WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN); + WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN); + WREG32_P(MPLL_CNTL_MODE, 0, ~SS_SSEN); + WREG32_P(MPLL_CNTL_MODE, 0, ~SS_DSMODE_EN); + } +} + +void cypress_start_dpm(struct radeon_device *rdev) +{ + WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN); +} + +void cypress_enable_sclk_control(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF); + else + WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF); +} + +void cypress_enable_mclk_control(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF); + else + WREG32_P(MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF); +} + +int cypress_notify_smc_display_change(struct radeon_device *rdev, + bool has_display) +{ + PPSMC_Msg msg = has_display ? + (PPSMC_Msg)PPSMC_MSG_HasDisplay : (PPSMC_Msg)PPSMC_MSG_NoDisplay; + + if (rv770_send_msg_to_smc(rdev, msg) != PPSMC_Result_OK) + return -EINVAL; + + return 0; +} + +void cypress_program_response_times(struct radeon_device *rdev) +{ + u32 reference_clock; + u32 mclk_switch_limit; + + reference_clock = radeon_get_xclk(rdev); + mclk_switch_limit = (460 * reference_clock) / 100; + + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_mclk_switch_lim, + mclk_switch_limit); + + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_mvdd_chg_time, 1); + + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_mc_block_delay, 0xAA); + + rv770_program_response_times(rdev); + + if (ASIC_IS_LOMBOK(rdev)) + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_is_asic_lombok, 1); + +} + +static int cypress_pcie_performance_request(struct radeon_device *rdev, + u8 perf_req, bool advertise) +{ +#if defined(CONFIG_ACPI) + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); +#endif + u32 tmp; + + udelay(10); + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) && (tmp & LC_CURRENT_DATA_RATE)) + return 0; + +#if defined(CONFIG_ACPI) + if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) || + (perf_req == PCIE_PERF_REQ_PECI_GEN2)) { + eg_pi->pcie_performance_request_registered = true; + return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise); + } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) && + eg_pi->pcie_performance_request_registered) { + eg_pi->pcie_performance_request_registered = false; + return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise); + } +#endif + + return 0; +} + +void cypress_advertise_gen2_capability(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 tmp; + +#if defined(CONFIG_ACPI) + radeon_acpi_pcie_notify_device_ready(rdev); +#endif + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) && + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) + pi->pcie_gen2 = true; + else + pi->pcie_gen2 = false; + + if (!pi->pcie_gen2) + cypress_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true); + +} + +static enum radeon_pcie_gen cypress_get_maximum_link_speed(struct radeon_ps *radeon_state) +{ + struct rv7xx_ps *state = rv770_get_ps(radeon_state); + + if (state->high.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) + return 1; + return 0; +} + +void cypress_notify_link_speed_change_after_state_change(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + enum radeon_pcie_gen pcie_link_speed_target = + cypress_get_maximum_link_speed(radeon_new_state); + enum radeon_pcie_gen pcie_link_speed_current = + cypress_get_maximum_link_speed(radeon_current_state); + u8 request; + + if (pcie_link_speed_target < pcie_link_speed_current) { + if (pcie_link_speed_target == RADEON_PCIE_GEN1) + request = PCIE_PERF_REQ_PECI_GEN1; + else if (pcie_link_speed_target == RADEON_PCIE_GEN2) + request = PCIE_PERF_REQ_PECI_GEN2; + else + request = PCIE_PERF_REQ_PECI_GEN3; + + cypress_pcie_performance_request(rdev, request, false); + } +} + +void cypress_notify_link_speed_change_before_state_change(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + enum radeon_pcie_gen pcie_link_speed_target = + cypress_get_maximum_link_speed(radeon_new_state); + enum radeon_pcie_gen pcie_link_speed_current = + cypress_get_maximum_link_speed(radeon_current_state); + u8 request; + + if (pcie_link_speed_target > pcie_link_speed_current) { + if (pcie_link_speed_target == RADEON_PCIE_GEN1) + request = PCIE_PERF_REQ_PECI_GEN1; + else if (pcie_link_speed_target == RADEON_PCIE_GEN2) + request = PCIE_PERF_REQ_PECI_GEN2; + else + request = PCIE_PERF_REQ_PECI_GEN3; + + cypress_pcie_performance_request(rdev, request, false); + } +} + +static int cypress_populate_voltage_value(struct radeon_device *rdev, + struct atom_voltage_table *table, + u16 value, RV770_SMC_VOLTAGE_VALUE *voltage) +{ + unsigned int i; + + for (i = 0; i < table->count; i++) { + if (value <= table->entries[i].value) { + voltage->index = (u8)i; + voltage->value = cpu_to_be16(table->entries[i].value); + break; + } + } + + if (i == table->count) + return -EINVAL; + + return 0; +} + +u8 cypress_get_strobe_mode_settings(struct radeon_device *rdev, u32 mclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u8 result = 0; + bool strobe_mode = false; + + if (pi->mem_gddr5) { + if (mclk <= pi->mclk_strobe_mode_threshold) + strobe_mode = true; + result = cypress_get_mclk_frequency_ratio(rdev, mclk, strobe_mode); + + if (strobe_mode) + result |= SMC_STROBE_ENABLE; + } + + return result; +} + +u32 cypress_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf) +{ + u32 ref_clk = rdev->clock.mpll.reference_freq; + u32 vco = clkf * ref_clk; + + /* 100 Mhz ref clk */ + if (ref_clk == 10000) { + if (vco > 500000) + return 0xC6; + if (vco > 400000) + return 0x9D; + if (vco > 330000) + return 0x6C; + if (vco > 250000) + return 0x2B; + if (vco > 160000) + return 0x5B; + if (vco > 120000) + return 0x0A; + return 0x4B; + } + + /* 27 Mhz ref clk */ + if (vco > 250000) + return 0x8B; + if (vco > 200000) + return 0xCC; + if (vco > 150000) + return 0x9B; + return 0x6B; +} + +static int cypress_populate_mclk_value(struct radeon_device *rdev, + u32 engine_clock, u32 memory_clock, + RV7XX_SMC_MCLK_VALUE *mclk, + bool strobe_mode, bool dll_state_on) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + u32 mpll_ad_func_cntl = + pi->clk_regs.rv770.mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2 = + pi->clk_regs.rv770.mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl = + pi->clk_regs.rv770.mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2 = + pi->clk_regs.rv770.mpll_dq_func_cntl_2; + u32 mclk_pwrmgt_cntl = + pi->clk_regs.rv770.mclk_pwrmgt_cntl; + u32 dll_cntl = + pi->clk_regs.rv770.dll_cntl; + u32 mpll_ss1 = pi->clk_regs.rv770.mpll_ss1; + u32 mpll_ss2 = pi->clk_regs.rv770.mpll_ss2; + struct atom_clock_dividers dividers; + u32 ibias; + u32 dll_speed; + int ret; + u32 mc_seq_misc7; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, + memory_clock, strobe_mode, ÷rs); + if (ret) + return ret; + + if (!strobe_mode) { + mc_seq_misc7 = RREG32(MC_SEQ_MISC7); + + if(mc_seq_misc7 & 0x8000000) + dividers.post_div = 1; + } + + ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div); + + mpll_ad_func_cntl &= ~(CLKR_MASK | + YCLK_POST_DIV_MASK | + CLKF_MASK | + CLKFRAC_MASK | + IBIAS_MASK); + mpll_ad_func_cntl |= CLKR(dividers.ref_div); + mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div); + mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div); + mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div); + mpll_ad_func_cntl |= IBIAS(ibias); + + if (dividers.vco_mode) + mpll_ad_func_cntl_2 |= VCO_MODE; + else + mpll_ad_func_cntl_2 &= ~VCO_MODE; + + if (pi->mem_gddr5) { + mpll_dq_func_cntl &= ~(CLKR_MASK | + YCLK_POST_DIV_MASK | + CLKF_MASK | + CLKFRAC_MASK | + IBIAS_MASK); + mpll_dq_func_cntl |= CLKR(dividers.ref_div); + mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div); + mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div); + mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div); + mpll_dq_func_cntl |= IBIAS(ibias); + + if (strobe_mode) + mpll_dq_func_cntl &= ~PDNB; + else + mpll_dq_func_cntl |= PDNB; + + if (dividers.vco_mode) + mpll_dq_func_cntl_2 |= VCO_MODE; + else + mpll_dq_func_cntl_2 &= ~VCO_MODE; + } + + if (pi->mclk_ss) { + struct radeon_atom_ss ss; + u32 vco_freq = memory_clock * dividers.post_div; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_MEMORY_SS, vco_freq)) { + u32 reference_clock = rdev->clock.mpll.reference_freq; + u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div); + u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate); + u32 clk_v = ss.percentage * + (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625); + + mpll_ss1 &= ~CLKV_MASK; + mpll_ss1 |= CLKV(clk_v); + + mpll_ss2 &= ~CLKS_MASK; + mpll_ss2 |= CLKS(clk_s); + } + } + + dll_speed = rv740_get_dll_speed(pi->mem_gddr5, + memory_clock); + + mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK; + mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed); + if (dll_state_on) + mclk_pwrmgt_cntl |= (MRDCKA0_PDNB | + MRDCKA1_PDNB | + MRDCKB0_PDNB | + MRDCKB1_PDNB | + MRDCKC0_PDNB | + MRDCKC1_PDNB | + MRDCKD0_PDNB | + MRDCKD1_PDNB); + else + mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB | + MRDCKA1_PDNB | + MRDCKB0_PDNB | + MRDCKB1_PDNB | + MRDCKC0_PDNB | + MRDCKC1_PDNB | + MRDCKD0_PDNB | + MRDCKD1_PDNB); + + mclk->mclk770.mclk_value = cpu_to_be32(memory_clock); + mclk->mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl); + mclk->mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2); + mclk->mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl); + mclk->mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2); + mclk->mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + mclk->mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl); + mclk->mclk770.vMPLL_SS = cpu_to_be32(mpll_ss1); + mclk->mclk770.vMPLL_SS2 = cpu_to_be32(mpll_ss2); + + return 0; +} + +u8 cypress_get_mclk_frequency_ratio(struct radeon_device *rdev, + u32 memory_clock, bool strobe_mode) +{ + u8 mc_para_index; + + if (rdev->family >= CHIP_BARTS) { + if (strobe_mode) { + if (memory_clock < 10000) + mc_para_index = 0x00; + else if (memory_clock > 47500) + mc_para_index = 0x0f; + else + mc_para_index = (u8)((memory_clock - 10000) / 2500); + } else { + if (memory_clock < 65000) + mc_para_index = 0x00; + else if (memory_clock > 135000) + mc_para_index = 0x0f; + else + mc_para_index = (u8)((memory_clock - 60000) / 5000); + } + } else { + if (strobe_mode) { + if (memory_clock < 10000) + mc_para_index = 0x00; + else if (memory_clock > 47500) + mc_para_index = 0x0f; + else + mc_para_index = (u8)((memory_clock - 10000) / 2500); + } else { + if (memory_clock < 40000) + mc_para_index = 0x00; + else if (memory_clock > 115000) + mc_para_index = 0x0f; + else + mc_para_index = (u8)((memory_clock - 40000) / 5000); + } + } + return mc_para_index; +} + +static int cypress_populate_mvdd_value(struct radeon_device *rdev, + u32 mclk, + RV770_SMC_VOLTAGE_VALUE *voltage) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (!pi->mvdd_control) { + voltage->index = eg_pi->mvdd_high_index; + voltage->value = cpu_to_be16(MVDD_HIGH_VALUE); + return 0; + } + + if (mclk <= pi->mvdd_split_frequency) { + voltage->index = eg_pi->mvdd_low_index; + voltage->value = cpu_to_be16(MVDD_LOW_VALUE); + } else { + voltage->index = eg_pi->mvdd_high_index; + voltage->value = cpu_to_be16(MVDD_HIGH_VALUE); + } + + return 0; +} + +int cypress_convert_power_level_to_smc(struct radeon_device *rdev, + struct rv7xx_pl *pl, + RV770_SMC_HW_PERFORMANCE_LEVEL *level, + u8 watermark_level) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + int ret; + bool dll_state_on; + + level->gen2PCIE = pi->pcie_gen2 ? + ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0; + level->gen2XSP = (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0; + level->backbias = (pl->flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? 1 : 0; + level->displayWatermark = watermark_level; + + ret = rv740_populate_sclk_value(rdev, pl->sclk, &level->sclk); + if (ret) + return ret; + + level->mcFlags = 0; + if (pi->mclk_stutter_mode_threshold && + (pl->mclk <= pi->mclk_stutter_mode_threshold) && + !eg_pi->uvd_enabled) { + level->mcFlags |= SMC_MC_STUTTER_EN; + if (eg_pi->sclk_deep_sleep) + level->stateFlags |= PPSMC_STATEFLAG_AUTO_PULSE_SKIP; + else + level->stateFlags &= ~PPSMC_STATEFLAG_AUTO_PULSE_SKIP; + } + + if (pi->mem_gddr5) { + if (pl->mclk > pi->mclk_edc_enable_threshold) + level->mcFlags |= SMC_MC_EDC_RD_FLAG; + + if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold) + level->mcFlags |= SMC_MC_EDC_WR_FLAG; + + level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk); + + if (level->strobeMode & SMC_STROBE_ENABLE) { + if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >= + ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf)) + dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false; + else + dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false; + } else + dll_state_on = eg_pi->dll_default_on; + + ret = cypress_populate_mclk_value(rdev, + pl->sclk, + pl->mclk, + &level->mclk, + (level->strobeMode & SMC_STROBE_ENABLE) != 0, + dll_state_on); + } else { + ret = cypress_populate_mclk_value(rdev, + pl->sclk, + pl->mclk, + &level->mclk, + true, + true); + } + if (ret) + return ret; + + ret = cypress_populate_voltage_value(rdev, + &eg_pi->vddc_voltage_table, + pl->vddc, + &level->vddc); + if (ret) + return ret; + + if (eg_pi->vddci_control) { + ret = cypress_populate_voltage_value(rdev, + &eg_pi->vddci_voltage_table, + pl->vddci, + &level->vddci); + if (ret) + return ret; + } + + ret = cypress_populate_mvdd_value(rdev, pl->mclk, &level->mvdd); + + return ret; +} + +static int cypress_convert_power_state_to_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + RV770_SMC_SWSTATE *smc_state) +{ + struct rv7xx_ps *state = rv770_get_ps(radeon_state); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + int ret; + + if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC)) + smc_state->flags |= PPSMC_SWSTATE_FLAG_DC; + + ret = cypress_convert_power_level_to_smc(rdev, + &state->low, + &smc_state->levels[0], + PPSMC_DISPLAY_WATERMARK_LOW); + if (ret) + return ret; + + ret = cypress_convert_power_level_to_smc(rdev, + &state->medium, + &smc_state->levels[1], + PPSMC_DISPLAY_WATERMARK_LOW); + if (ret) + return ret; + + ret = cypress_convert_power_level_to_smc(rdev, + &state->high, + &smc_state->levels[2], + PPSMC_DISPLAY_WATERMARK_HIGH); + if (ret) + return ret; + + smc_state->levels[0].arbValue = MC_CG_ARB_FREQ_F1; + smc_state->levels[1].arbValue = MC_CG_ARB_FREQ_F2; + smc_state->levels[2].arbValue = MC_CG_ARB_FREQ_F3; + + if (eg_pi->dynamic_ac_timing) { + smc_state->levels[0].ACIndex = 2; + smc_state->levels[1].ACIndex = 3; + smc_state->levels[2].ACIndex = 4; + } else { + smc_state->levels[0].ACIndex = 0; + smc_state->levels[1].ACIndex = 0; + smc_state->levels[2].ACIndex = 0; + } + + rv770_populate_smc_sp(rdev, radeon_state, smc_state); + + return rv770_populate_smc_t(rdev, radeon_state, smc_state); +} + +static void cypress_convert_mc_registers(struct evergreen_mc_reg_entry *entry, + SMC_Evergreen_MCRegisterSet *data, + u32 num_entries, u32 valid_flag) +{ + u32 i, j; + + for (i = 0, j = 0; j < num_entries; j++) { + if (valid_flag & (1 << j)) { + data->value[i] = cpu_to_be32(entry->mc_data[j]); + i++; + } + } +} + +static void cypress_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev, + struct rv7xx_pl *pl, + SMC_Evergreen_MCRegisterSet *mc_reg_table_data) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 i = 0; + + for (i = 0; i < eg_pi->mc_reg_table.num_entries; i++) { + if (pl->mclk <= + eg_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max) + break; + } + + if ((i == eg_pi->mc_reg_table.num_entries) && (i > 0)) + --i; + + cypress_convert_mc_registers(&eg_pi->mc_reg_table.mc_reg_table_entry[i], + mc_reg_table_data, + eg_pi->mc_reg_table.last, + eg_pi->mc_reg_table.valid_flag); +} + +static void cypress_convert_mc_reg_table_to_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + SMC_Evergreen_MCRegisters *mc_reg_table) +{ + struct rv7xx_ps *state = rv770_get_ps(radeon_state); + + cypress_convert_mc_reg_table_entry_to_smc(rdev, + &state->low, + &mc_reg_table->data[2]); + cypress_convert_mc_reg_table_entry_to_smc(rdev, + &state->medium, + &mc_reg_table->data[3]); + cypress_convert_mc_reg_table_entry_to_smc(rdev, + &state->high, + &mc_reg_table->data[4]); +} + +int cypress_upload_sw_state(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u16 address = pi->state_table_start + + offsetof(RV770_SMC_STATETABLE, driverState); + RV770_SMC_SWSTATE state = { 0 }; + int ret; + + ret = cypress_convert_power_state_to_smc(rdev, radeon_new_state, &state); + if (ret) + return ret; + + return rv770_copy_bytes_to_smc(rdev, address, (u8 *)&state, + sizeof(RV770_SMC_SWSTATE), + pi->sram_end); +} + +int cypress_upload_mc_reg_table(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + SMC_Evergreen_MCRegisters mc_reg_table = { 0 }; + u16 address; + + cypress_convert_mc_reg_table_to_smc(rdev, radeon_new_state, &mc_reg_table); + + address = eg_pi->mc_reg_table_start + + (u16)offsetof(SMC_Evergreen_MCRegisters, data[2]); + + return rv770_copy_bytes_to_smc(rdev, address, + (u8 *)&mc_reg_table.data[2], + sizeof(SMC_Evergreen_MCRegisterSet) * 3, + pi->sram_end); +} + +u32 cypress_calculate_burst_time(struct radeon_device *rdev, + u32 engine_clock, u32 memory_clock) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 multiplier = pi->mem_gddr5 ? 1 : 2; + u32 result = (4 * multiplier * engine_clock) / (memory_clock / 2); + u32 burst_time; + + if (result <= 4) + burst_time = 0; + else if (result < 8) + burst_time = result - 4; + else { + burst_time = result / 2 ; + if (burst_time > 18) + burst_time = 18; + } + + return burst_time; +} + +void cypress_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state); + u32 mc_arb_burst_time = RREG32(MC_ARB_BURST_TIME); + + mc_arb_burst_time &= ~(STATE1_MASK | STATE2_MASK | STATE3_MASK); + + mc_arb_burst_time |= STATE1(cypress_calculate_burst_time(rdev, + new_state->low.sclk, + new_state->low.mclk)); + mc_arb_burst_time |= STATE2(cypress_calculate_burst_time(rdev, + new_state->medium.sclk, + new_state->medium.mclk)); + mc_arb_burst_time |= STATE3(cypress_calculate_burst_time(rdev, + new_state->high.sclk, + new_state->high.mclk)); + + rv730_program_memory_timing_parameters(rdev, radeon_new_state); + + WREG32(MC_ARB_BURST_TIME, mc_arb_burst_time); +} + +static void cypress_populate_mc_reg_addresses(struct radeon_device *rdev, + SMC_Evergreen_MCRegisters *mc_reg_table) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 i, j; + + for (i = 0, j = 0; j < eg_pi->mc_reg_table.last; j++) { + if (eg_pi->mc_reg_table.valid_flag & (1 << j)) { + mc_reg_table->address[i].s0 = + cpu_to_be16(eg_pi->mc_reg_table.mc_reg_address[j].s0); + mc_reg_table->address[i].s1 = + cpu_to_be16(eg_pi->mc_reg_table.mc_reg_address[j].s1); + i++; + } + } + + mc_reg_table->last = (u8)i; +} + +static void cypress_set_mc_reg_address_table(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 i = 0; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_RAS_TIMING_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_RAS_TIMING >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_CAS_TIMING_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_CAS_TIMING >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_MISC_TIMING_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_MISC_TIMING >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_MISC_TIMING2_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_MISC_TIMING2 >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_RD_CTL_D0_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_RD_CTL_D0 >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_RD_CTL_D1_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_RD_CTL_D1 >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_WR_CTL_D0_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_WR_CTL_D0 >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_WR_CTL_D1_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_WR_CTL_D1 >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_PMG_CMD_EMRS >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_PMG_CMD_MRS >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_PMG_CMD_MRS1 >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_MISC1 >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_MISC1 >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_RESERVE_M >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_RESERVE_M >> 2; + i++; + + eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_MISC3 >> 2; + eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_MISC3 >> 2; + i++; + + eg_pi->mc_reg_table.last = (u8)i; +} + +static void cypress_retrieve_ac_timing_for_one_entry(struct radeon_device *rdev, + struct evergreen_mc_reg_entry *entry) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 i; + + for (i = 0; i < eg_pi->mc_reg_table.last; i++) + entry->mc_data[i] = + RREG32(eg_pi->mc_reg_table.mc_reg_address[i].s1 << 2); + +} + +static void cypress_retrieve_ac_timing_for_all_ranges(struct radeon_device *rdev, + struct atom_memory_clock_range_table *range_table) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 i, j; + + for (i = 0; i < range_table->num_entries; i++) { + eg_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max = + range_table->mclk[i]; + radeon_atom_set_ac_timing(rdev, range_table->mclk[i]); + cypress_retrieve_ac_timing_for_one_entry(rdev, + &eg_pi->mc_reg_table.mc_reg_table_entry[i]); + } + + eg_pi->mc_reg_table.num_entries = range_table->num_entries; + eg_pi->mc_reg_table.valid_flag = 0; + + for (i = 0; i < eg_pi->mc_reg_table.last; i++) { + for (j = 1; j < range_table->num_entries; j++) { + if (eg_pi->mc_reg_table.mc_reg_table_entry[j-1].mc_data[i] != + eg_pi->mc_reg_table.mc_reg_table_entry[j].mc_data[i]) { + eg_pi->mc_reg_table.valid_flag |= (1 << i); + break; + } + } + } +} + +static int cypress_initialize_mc_reg_table(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u8 module_index = rv770_get_memory_module_index(rdev); + struct atom_memory_clock_range_table range_table = { 0 }; + int ret; + + ret = radeon_atom_get_mclk_range_table(rdev, + pi->mem_gddr5, + module_index, &range_table); + if (ret) + return ret; + + cypress_retrieve_ac_timing_for_all_ranges(rdev, &range_table); + + return 0; +} + +static void cypress_wait_for_mc_sequencer(struct radeon_device *rdev, u8 value) +{ + u32 i, j; + u32 channels = 2; + + if ((rdev->family == CHIP_CYPRESS) || + (rdev->family == CHIP_HEMLOCK)) + channels = 4; + else if (rdev->family == CHIP_CEDAR) + channels = 1; + + for (i = 0; i < channels; i++) { + if ((rdev->family == CHIP_CYPRESS) || + (rdev->family == CHIP_HEMLOCK)) { + WREG32_P(MC_CONFIG_MCD, MC_RD_ENABLE_MCD(i), ~MC_RD_ENABLE_MCD_MASK); + WREG32_P(MC_CG_CONFIG_MCD, MC_RD_ENABLE_MCD(i), ~MC_RD_ENABLE_MCD_MASK); + } else { + WREG32_P(MC_CONFIG, MC_RD_ENABLE(i), ~MC_RD_ENABLE_MASK); + WREG32_P(MC_CG_CONFIG, MC_RD_ENABLE(i), ~MC_RD_ENABLE_MASK); + } + for (j = 0; j < rdev->usec_timeout; j++) { + if (((RREG32(MC_SEQ_CG) & CG_SEQ_RESP_MASK) >> CG_SEQ_RESP_SHIFT) == value) + break; + udelay(1); + } + } +} + +static void cypress_force_mc_use_s1(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state) +{ + struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state); + u32 strobe_mode; + u32 mc_seq_cg; + int i; + + if (RREG32(MC_SEQ_STATUS_M) & PMG_PWRSTATE) + return; + + radeon_atom_set_ac_timing(rdev, boot_state->low.mclk); + radeon_mc_wait_for_idle(rdev); + + if ((rdev->family == CHIP_CYPRESS) || + (rdev->family == CHIP_HEMLOCK)) { + WREG32(MC_CONFIG_MCD, 0xf); + WREG32(MC_CG_CONFIG_MCD, 0xf); + } else { + WREG32(MC_CONFIG, 0xf); + WREG32(MC_CG_CONFIG, 0xf); + } + + for (i = 0; i < rdev->num_crtc; i++) + radeon_wait_for_vblank(rdev, i); + + WREG32(MC_SEQ_CG, MC_CG_SEQ_YCLK_SUSPEND); + cypress_wait_for_mc_sequencer(rdev, MC_CG_SEQ_YCLK_SUSPEND); + + strobe_mode = cypress_get_strobe_mode_settings(rdev, + boot_state->low.mclk); + + mc_seq_cg = CG_SEQ_REQ(MC_CG_SEQ_DRAMCONF_S1); + mc_seq_cg |= SEQ_CG_RESP(strobe_mode); + WREG32(MC_SEQ_CG, mc_seq_cg); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(MC_SEQ_STATUS_M) & PMG_PWRSTATE) + break; + udelay(1); + } + + mc_seq_cg &= ~CG_SEQ_REQ_MASK; + mc_seq_cg |= CG_SEQ_REQ(MC_CG_SEQ_YCLK_RESUME); + WREG32(MC_SEQ_CG, mc_seq_cg); + + cypress_wait_for_mc_sequencer(rdev, MC_CG_SEQ_YCLK_RESUME); +} + +static void cypress_copy_ac_timing_from_s1_to_s0(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 value; + u32 i; + + for (i = 0; i < eg_pi->mc_reg_table.last; i++) { + value = RREG32(eg_pi->mc_reg_table.mc_reg_address[i].s1 << 2); + WREG32(eg_pi->mc_reg_table.mc_reg_address[i].s0 << 2, value); + } +} + +static void cypress_force_mc_use_s0(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state) +{ + struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state); + u32 strobe_mode; + u32 mc_seq_cg; + int i; + + cypress_copy_ac_timing_from_s1_to_s0(rdev); + radeon_mc_wait_for_idle(rdev); + + if ((rdev->family == CHIP_CYPRESS) || + (rdev->family == CHIP_HEMLOCK)) { + WREG32(MC_CONFIG_MCD, 0xf); + WREG32(MC_CG_CONFIG_MCD, 0xf); + } else { + WREG32(MC_CONFIG, 0xf); + WREG32(MC_CG_CONFIG, 0xf); + } + + for (i = 0; i < rdev->num_crtc; i++) + radeon_wait_for_vblank(rdev, i); + + WREG32(MC_SEQ_CG, MC_CG_SEQ_YCLK_SUSPEND); + cypress_wait_for_mc_sequencer(rdev, MC_CG_SEQ_YCLK_SUSPEND); + + strobe_mode = cypress_get_strobe_mode_settings(rdev, + boot_state->low.mclk); + + mc_seq_cg = CG_SEQ_REQ(MC_CG_SEQ_DRAMCONF_S0); + mc_seq_cg |= SEQ_CG_RESP(strobe_mode); + WREG32(MC_SEQ_CG, mc_seq_cg); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (!(RREG32(MC_SEQ_STATUS_M) & PMG_PWRSTATE)) + break; + udelay(1); + } + + mc_seq_cg &= ~CG_SEQ_REQ_MASK; + mc_seq_cg |= CG_SEQ_REQ(MC_CG_SEQ_YCLK_RESUME); + WREG32(MC_SEQ_CG, mc_seq_cg); + + cypress_wait_for_mc_sequencer(rdev, MC_CG_SEQ_YCLK_RESUME); +} + +static int cypress_populate_initial_mvdd_value(struct radeon_device *rdev, + RV770_SMC_VOLTAGE_VALUE *voltage) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + voltage->index = eg_pi->mvdd_high_index; + voltage->value = cpu_to_be16(MVDD_HIGH_VALUE); + + return 0; +} + +int cypress_populate_smc_initial_state(struct radeon_device *rdev, + struct radeon_ps *radeon_initial_state, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_ps *initial_state = rv770_get_ps(radeon_initial_state); + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 a_t; + + table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = + cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl); + table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = + cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl_2); + table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = + cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl); + table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = + cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl_2); + table->initialState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = + cpu_to_be32(pi->clk_regs.rv770.mclk_pwrmgt_cntl); + table->initialState.levels[0].mclk.mclk770.vDLL_CNTL = + cpu_to_be32(pi->clk_regs.rv770.dll_cntl); + + table->initialState.levels[0].mclk.mclk770.vMPLL_SS = + cpu_to_be32(pi->clk_regs.rv770.mpll_ss1); + table->initialState.levels[0].mclk.mclk770.vMPLL_SS2 = + cpu_to_be32(pi->clk_regs.rv770.mpll_ss2); + + table->initialState.levels[0].mclk.mclk770.mclk_value = + cpu_to_be32(initial_state->low.mclk); + + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_2); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_3); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum_2); + + table->initialState.levels[0].sclk.sclk_value = + cpu_to_be32(initial_state->low.sclk); + + table->initialState.levels[0].arbValue = MC_CG_ARB_FREQ_F0; + + table->initialState.levels[0].ACIndex = 0; + + cypress_populate_voltage_value(rdev, + &eg_pi->vddc_voltage_table, + initial_state->low.vddc, + &table->initialState.levels[0].vddc); + + if (eg_pi->vddci_control) + cypress_populate_voltage_value(rdev, + &eg_pi->vddci_voltage_table, + initial_state->low.vddci, + &table->initialState.levels[0].vddci); + + cypress_populate_initial_mvdd_value(rdev, + &table->initialState.levels[0].mvdd); + + a_t = CG_R(0xffff) | CG_L(0); + table->initialState.levels[0].aT = cpu_to_be32(a_t); + + table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp); + + + if (pi->boot_in_gen2) + table->initialState.levels[0].gen2PCIE = 1; + else + table->initialState.levels[0].gen2PCIE = 0; + if (initial_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) + table->initialState.levels[0].gen2XSP = 1; + else + table->initialState.levels[0].gen2XSP = 0; + + if (pi->mem_gddr5) { + table->initialState.levels[0].strobeMode = + cypress_get_strobe_mode_settings(rdev, + initial_state->low.mclk); + + if (initial_state->low.mclk > pi->mclk_edc_enable_threshold) + table->initialState.levels[0].mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG; + else + table->initialState.levels[0].mcFlags = 0; + } + + table->initialState.levels[1] = table->initialState.levels[0]; + table->initialState.levels[2] = table->initialState.levels[0]; + + table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC; + + return 0; +} + +int cypress_populate_smc_acpi_state(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 mpll_ad_func_cntl = + pi->clk_regs.rv770.mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2 = + pi->clk_regs.rv770.mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl = + pi->clk_regs.rv770.mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2 = + pi->clk_regs.rv770.mpll_dq_func_cntl_2; + u32 spll_func_cntl = + pi->clk_regs.rv770.cg_spll_func_cntl; + u32 spll_func_cntl_2 = + pi->clk_regs.rv770.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = + pi->clk_regs.rv770.cg_spll_func_cntl_3; + u32 mclk_pwrmgt_cntl = + pi->clk_regs.rv770.mclk_pwrmgt_cntl; + u32 dll_cntl = + pi->clk_regs.rv770.dll_cntl; + + table->ACPIState = table->initialState; + + table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC; + + if (pi->acpi_vddc) { + cypress_populate_voltage_value(rdev, + &eg_pi->vddc_voltage_table, + pi->acpi_vddc, + &table->ACPIState.levels[0].vddc); + if (pi->pcie_gen2) { + if (pi->acpi_pcie_gen2) + table->ACPIState.levels[0].gen2PCIE = 1; + else + table->ACPIState.levels[0].gen2PCIE = 0; + } else + table->ACPIState.levels[0].gen2PCIE = 0; + if (pi->acpi_pcie_gen2) + table->ACPIState.levels[0].gen2XSP = 1; + else + table->ACPIState.levels[0].gen2XSP = 0; + } else { + cypress_populate_voltage_value(rdev, + &eg_pi->vddc_voltage_table, + pi->min_vddc_in_table, + &table->ACPIState.levels[0].vddc); + table->ACPIState.levels[0].gen2PCIE = 0; + } + + if (eg_pi->acpi_vddci) { + if (eg_pi->vddci_control) { + cypress_populate_voltage_value(rdev, + &eg_pi->vddci_voltage_table, + eg_pi->acpi_vddci, + &table->ACPIState.levels[0].vddci); + } + } + + mpll_ad_func_cntl &= ~PDNB; + + mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN; + + if (pi->mem_gddr5) + mpll_dq_func_cntl &= ~PDNB; + mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS; + + mclk_pwrmgt_cntl |= (MRDCKA0_RESET | + MRDCKA1_RESET | + MRDCKB0_RESET | + MRDCKB1_RESET | + MRDCKC0_RESET | + MRDCKC1_RESET | + MRDCKD0_RESET | + MRDCKD1_RESET); + + mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB | + MRDCKA1_PDNB | + MRDCKB0_PDNB | + MRDCKB1_PDNB | + MRDCKC0_PDNB | + MRDCKC1_PDNB | + MRDCKD0_PDNB | + MRDCKD1_PDNB); + + dll_cntl |= (MRDCKA0_BYPASS | + MRDCKA1_BYPASS | + MRDCKB0_BYPASS | + MRDCKB1_BYPASS | + MRDCKC0_BYPASS | + MRDCKC1_BYPASS | + MRDCKD0_BYPASS | + MRDCKD1_BYPASS); + + /* evergreen only */ + if (rdev->family <= CHIP_HEMLOCK) + spll_func_cntl |= SPLL_RESET | SPLL_SLEEP | SPLL_BYPASS_EN; + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(4); + + table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = + cpu_to_be32(mpll_ad_func_cntl); + table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = + cpu_to_be32(mpll_ad_func_cntl_2); + table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = + cpu_to_be32(mpll_dq_func_cntl); + table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = + cpu_to_be32(mpll_dq_func_cntl_2); + table->ACPIState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = + cpu_to_be32(mclk_pwrmgt_cntl); + table->ACPIState.levels[0].mclk.mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl); + + table->ACPIState.levels[0].mclk.mclk770.mclk_value = 0; + + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = + cpu_to_be32(spll_func_cntl); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = + cpu_to_be32(spll_func_cntl_2); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = + cpu_to_be32(spll_func_cntl_3); + + table->ACPIState.levels[0].sclk.sclk_value = 0; + + cypress_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd); + + if (eg_pi->dynamic_ac_timing) + table->ACPIState.levels[0].ACIndex = 1; + + table->ACPIState.levels[1] = table->ACPIState.levels[0]; + table->ACPIState.levels[2] = table->ACPIState.levels[0]; + + return 0; +} + +static void cypress_trim_voltage_table_to_fit_state_table(struct radeon_device *rdev, + struct atom_voltage_table *voltage_table) +{ + unsigned int i, diff; + + if (voltage_table->count <= MAX_NO_VREG_STEPS) + return; + + diff = voltage_table->count - MAX_NO_VREG_STEPS; + + for (i= 0; i < MAX_NO_VREG_STEPS; i++) + voltage_table->entries[i] = voltage_table->entries[i + diff]; + + voltage_table->count = MAX_NO_VREG_STEPS; +} + +int cypress_construct_voltage_tables(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + int ret; + + ret = radeon_atom_get_voltage_table(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0, + &eg_pi->vddc_voltage_table); + if (ret) + return ret; + + if (eg_pi->vddc_voltage_table.count > MAX_NO_VREG_STEPS) + cypress_trim_voltage_table_to_fit_state_table(rdev, + &eg_pi->vddc_voltage_table); + + if (eg_pi->vddci_control) { + ret = radeon_atom_get_voltage_table(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0, + &eg_pi->vddci_voltage_table); + if (ret) + return ret; + + if (eg_pi->vddci_voltage_table.count > MAX_NO_VREG_STEPS) + cypress_trim_voltage_table_to_fit_state_table(rdev, + &eg_pi->vddci_voltage_table); + } + + return 0; +} + +static void cypress_populate_smc_voltage_table(struct radeon_device *rdev, + struct atom_voltage_table *voltage_table, + RV770_SMC_STATETABLE *table) +{ + unsigned int i; + + for (i = 0; i < voltage_table->count; i++) { + table->highSMIO[i] = 0; + table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low); + } +} + +int cypress_populate_smc_voltage_tables(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + unsigned char i; + + if (eg_pi->vddc_voltage_table.count) { + cypress_populate_smc_voltage_table(rdev, + &eg_pi->vddc_voltage_table, + table); + + table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_VDDC] = 0; + table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_VDDC] = + cpu_to_be32(eg_pi->vddc_voltage_table.mask_low); + + for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) { + if (pi->max_vddc_in_table <= + eg_pi->vddc_voltage_table.entries[i].value) { + table->maxVDDCIndexInPPTable = i; + break; + } + } + } + + if (eg_pi->vddci_voltage_table.count) { + cypress_populate_smc_voltage_table(rdev, + &eg_pi->vddci_voltage_table, + table); + + table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_VDDCI] = 0; + table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_VDDCI] = + cpu_to_be32(eg_pi->vddci_voltage_table.mask_low); + } + + return 0; +} + +static u32 cypress_get_mclk_split_point(struct atom_memory_info *memory_info) +{ + if ((memory_info->mem_type == MEM_TYPE_GDDR3) || + (memory_info->mem_type == MEM_TYPE_DDR3)) + return 30000; + + return 0; +} + +int cypress_get_mvdd_configuration(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u8 module_index; + struct atom_memory_info memory_info; + u32 tmp = RREG32(GENERAL_PWRMGT); + + if (!(tmp & BACKBIAS_PAD_EN)) { + eg_pi->mvdd_high_index = 0; + eg_pi->mvdd_low_index = 1; + pi->mvdd_control = false; + return 0; + } + + if (tmp & BACKBIAS_VALUE) + eg_pi->mvdd_high_index = 1; + else + eg_pi->mvdd_high_index = 0; + + eg_pi->mvdd_low_index = + (eg_pi->mvdd_high_index == 0) ? 1 : 0; + + module_index = rv770_get_memory_module_index(rdev); + + if (radeon_atom_get_memory_info(rdev, module_index, &memory_info)) { + pi->mvdd_control = false; + return 0; + } + + pi->mvdd_split_frequency = + cypress_get_mclk_split_point(&memory_info); + + if (pi->mvdd_split_frequency == 0) { + pi->mvdd_control = false; + return 0; + } + + return 0; +} + +static int cypress_init_smc_table(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + RV770_SMC_STATETABLE *table = &pi->smc_statetable; + int ret; + + memset(table, 0, sizeof(RV770_SMC_STATETABLE)); + + cypress_populate_smc_voltage_tables(rdev, table); + + switch (rdev->pm.int_thermal_type) { + case THERMAL_TYPE_EVERGREEN: + case THERMAL_TYPE_EMC2103_WITH_INTERNAL: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL; + break; + case THERMAL_TYPE_NONE: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE; + break; + default: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL; + break; + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) + table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT) + table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) + table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC; + + if (pi->mem_gddr5) + table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5; + + ret = cypress_populate_smc_initial_state(rdev, radeon_boot_state, table); + if (ret) + return ret; + + ret = cypress_populate_smc_acpi_state(rdev, table); + if (ret) + return ret; + + table->driverState = table->initialState; + + return rv770_copy_bytes_to_smc(rdev, + pi->state_table_start, + (u8 *)table, sizeof(RV770_SMC_STATETABLE), + pi->sram_end); +} + +int cypress_populate_mc_reg_table(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state); + SMC_Evergreen_MCRegisters mc_reg_table = { 0 }; + + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_seq_index, 1); + + cypress_populate_mc_reg_addresses(rdev, &mc_reg_table); + + cypress_convert_mc_reg_table_entry_to_smc(rdev, + &boot_state->low, + &mc_reg_table.data[0]); + + cypress_convert_mc_registers(&eg_pi->mc_reg_table.mc_reg_table_entry[0], + &mc_reg_table.data[1], eg_pi->mc_reg_table.last, + eg_pi->mc_reg_table.valid_flag); + + cypress_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, &mc_reg_table); + + return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start, + (u8 *)&mc_reg_table, sizeof(SMC_Evergreen_MCRegisters), + pi->sram_end); +} + +int cypress_get_table_locations(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 tmp; + int ret; + + ret = rv770_read_smc_sram_dword(rdev, + EVERGREEN_SMC_FIRMWARE_HEADER_LOCATION + + EVERGREEN_SMC_FIRMWARE_HEADER_stateTable, + &tmp, pi->sram_end); + if (ret) + return ret; + + pi->state_table_start = (u16)tmp; + + ret = rv770_read_smc_sram_dword(rdev, + EVERGREEN_SMC_FIRMWARE_HEADER_LOCATION + + EVERGREEN_SMC_FIRMWARE_HEADER_softRegisters, + &tmp, pi->sram_end); + if (ret) + return ret; + + pi->soft_regs_start = (u16)tmp; + + ret = rv770_read_smc_sram_dword(rdev, + EVERGREEN_SMC_FIRMWARE_HEADER_LOCATION + + EVERGREEN_SMC_FIRMWARE_HEADER_mcRegisterTable, + &tmp, pi->sram_end); + if (ret) + return ret; + + eg_pi->mc_reg_table_start = (u16)tmp; + + return 0; +} + +void cypress_enable_display_gap(struct radeon_device *rdev) +{ + u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL); + + tmp &= ~(DISP1_GAP_MASK | DISP2_GAP_MASK); + tmp |= (DISP1_GAP(R600_PM_DISPLAY_GAP_IGNORE) | + DISP2_GAP(R600_PM_DISPLAY_GAP_IGNORE)); + + tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK); + tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK) | + DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE)); + WREG32(CG_DISPLAY_GAP_CNTL, tmp); +} + +static void cypress_program_display_gap(struct radeon_device *rdev) +{ + u32 tmp, pipe; + int i; + + tmp = RREG32(CG_DISPLAY_GAP_CNTL) & ~(DISP1_GAP_MASK | DISP2_GAP_MASK); + if (rdev->pm.dpm.new_active_crtc_count > 0) + tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM); + else + tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_IGNORE); + + if (rdev->pm.dpm.new_active_crtc_count > 1) + tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM); + else + tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_IGNORE); + + WREG32(CG_DISPLAY_GAP_CNTL, tmp); + + tmp = RREG32(DCCG_DISP_SLOW_SELECT_REG); + pipe = (tmp & DCCG_DISP1_SLOW_SELECT_MASK) >> DCCG_DISP1_SLOW_SELECT_SHIFT; + + if ((rdev->pm.dpm.new_active_crtc_count > 0) && + (!(rdev->pm.dpm.new_active_crtcs & (1 << pipe)))) { + /* find the first active crtc */ + for (i = 0; i < rdev->num_crtc; i++) { + if (rdev->pm.dpm.new_active_crtcs & (1 << i)) + break; + } + if (i == rdev->num_crtc) + pipe = 0; + else + pipe = i; + + tmp &= ~DCCG_DISP1_SLOW_SELECT_MASK; + tmp |= DCCG_DISP1_SLOW_SELECT(pipe); + WREG32(DCCG_DISP_SLOW_SELECT_REG, tmp); + } + + cypress_notify_smc_display_change(rdev, rdev->pm.dpm.new_active_crtc_count > 0); +} + +void cypress_dpm_setup_asic(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + rv740_read_clock_registers(rdev); + rv770_read_voltage_smio_registers(rdev); + rv770_get_max_vddc(rdev); + rv770_get_memory_type(rdev); + + if (eg_pi->pcie_performance_request) + eg_pi->pcie_performance_request_registered = false; + + if (eg_pi->pcie_performance_request) + cypress_advertise_gen2_capability(rdev); + + rv770_get_pcie_gen2_status(rdev); + + rv770_enable_acpi_pm(rdev); +} + +int cypress_dpm_enable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + int ret; + + if (pi->gfx_clock_gating) + rv770_restore_cgcg(rdev); + + if (rv770_dpm_enabled(rdev)) + return -EINVAL; + + if (pi->voltage_control) { + rv770_enable_voltage_control(rdev, true); + ret = cypress_construct_voltage_tables(rdev); + if (ret) { + DRM_ERROR("cypress_construct_voltage_tables failed\n"); + return ret; + } + } + + if (pi->mvdd_control) { + ret = cypress_get_mvdd_configuration(rdev); + if (ret) { + DRM_ERROR("cypress_get_mvdd_configuration failed\n"); + return ret; + } + } + + if (eg_pi->dynamic_ac_timing) { + cypress_set_mc_reg_address_table(rdev); + cypress_force_mc_use_s0(rdev, boot_ps); + ret = cypress_initialize_mc_reg_table(rdev); + if (ret) + eg_pi->dynamic_ac_timing = false; + cypress_force_mc_use_s1(rdev, boot_ps); + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS) + rv770_enable_backbias(rdev, true); + + if (pi->dynamic_ss) + cypress_enable_spread_spectrum(rdev, true); + + if (pi->thermal_protection) + rv770_enable_thermal_protection(rdev, true); + + rv770_setup_bsp(rdev); + rv770_program_git(rdev); + rv770_program_tp(rdev); + rv770_program_tpp(rdev); + rv770_program_sstp(rdev); + rv770_program_engine_speed_parameters(rdev); + cypress_enable_display_gap(rdev); + rv770_program_vc(rdev); + + if (pi->dynamic_pcie_gen2) + cypress_enable_dynamic_pcie_gen2(rdev, true); + + ret = rv770_upload_firmware(rdev); + if (ret) { + DRM_ERROR("rv770_upload_firmware failed\n"); + return ret; + } + + ret = cypress_get_table_locations(rdev); + if (ret) { + DRM_ERROR("cypress_get_table_locations failed\n"); + return ret; + } + ret = cypress_init_smc_table(rdev, boot_ps); + if (ret) { + DRM_ERROR("cypress_init_smc_table failed\n"); + return ret; + } + if (eg_pi->dynamic_ac_timing) { + ret = cypress_populate_mc_reg_table(rdev, boot_ps); + if (ret) { + DRM_ERROR("cypress_populate_mc_reg_table failed\n"); + return ret; + } + } + + cypress_program_response_times(rdev); + + r7xx_start_smc(rdev); + + ret = cypress_notify_smc_display_change(rdev, false); + if (ret) { + DRM_ERROR("cypress_notify_smc_display_change failed\n"); + return ret; + } + cypress_enable_sclk_control(rdev, true); + + if (eg_pi->memory_transition) + cypress_enable_mclk_control(rdev, true); + + cypress_start_dpm(rdev); + + if (pi->gfx_clock_gating) + cypress_gfx_clock_gating_enable(rdev, true); + + if (pi->mg_clock_gating) + cypress_mg_clock_gating_enable(rdev, true); + + rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + + return 0; +} + +void cypress_dpm_disable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + + if (!rv770_dpm_enabled(rdev)) + return; + + rv770_clear_vc(rdev); + + if (pi->thermal_protection) + rv770_enable_thermal_protection(rdev, false); + + if (pi->dynamic_pcie_gen2) + cypress_enable_dynamic_pcie_gen2(rdev, false); + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + rdev->irq.dpm_thermal = false; + radeon_irq_set(rdev); + } + + if (pi->gfx_clock_gating) + cypress_gfx_clock_gating_enable(rdev, false); + + if (pi->mg_clock_gating) + cypress_mg_clock_gating_enable(rdev, false); + + rv770_stop_dpm(rdev); + r7xx_stop_smc(rdev); + + cypress_enable_spread_spectrum(rdev, false); + + if (eg_pi->dynamic_ac_timing) + cypress_force_mc_use_s1(rdev, boot_ps); + + rv770_reset_smio_status(rdev); +} + +int cypress_dpm_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps; + struct radeon_ps *old_ps = rdev->pm.dpm.current_ps; + int ret; + + ret = rv770_restrict_performance_levels_before_switch(rdev); + if (ret) { + DRM_ERROR("rv770_restrict_performance_levels_before_switch failed\n"); + return ret; + } + if (eg_pi->pcie_performance_request) + cypress_notify_link_speed_change_before_state_change(rdev, new_ps, old_ps); + + rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps); + ret = rv770_halt_smc(rdev); + if (ret) { + DRM_ERROR("rv770_halt_smc failed\n"); + return ret; + } + ret = cypress_upload_sw_state(rdev, new_ps); + if (ret) { + DRM_ERROR("cypress_upload_sw_state failed\n"); + return ret; + } + if (eg_pi->dynamic_ac_timing) { + ret = cypress_upload_mc_reg_table(rdev, new_ps); + if (ret) { + DRM_ERROR("cypress_upload_mc_reg_table failed\n"); + return ret; + } + } + + cypress_program_memory_timing_parameters(rdev, new_ps); + + ret = rv770_resume_smc(rdev); + if (ret) { + DRM_ERROR("rv770_resume_smc failed\n"); + return ret; + } + ret = rv770_set_sw_state(rdev); + if (ret) { + DRM_ERROR("rv770_set_sw_state failed\n"); + return ret; + } + rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps); + + if (eg_pi->pcie_performance_request) + cypress_notify_link_speed_change_after_state_change(rdev, new_ps, old_ps); + + return 0; +} + +#if 0 +void cypress_dpm_reset_asic(struct radeon_device *rdev) +{ + rv770_restrict_performance_levels_before_switch(rdev); + rv770_set_boot_state(rdev); +} +#endif + +void cypress_dpm_display_configuration_changed(struct radeon_device *rdev) +{ + cypress_program_display_gap(rdev); +} + +int cypress_dpm_init(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi; + struct evergreen_power_info *eg_pi; + struct atom_clock_dividers dividers; + int ret; + + eg_pi = kzalloc(sizeof(struct evergreen_power_info), GFP_KERNEL); + if (eg_pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = eg_pi; + pi = &eg_pi->rv7xx; + + rv770_get_max_vddc(rdev); + + eg_pi->ulv.supported = false; + pi->acpi_vddc = 0; + eg_pi->acpi_vddci = 0; + pi->min_vddc_in_table = 0; + pi->max_vddc_in_table = 0; + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = rv7xx_parse_power_table(rdev); + if (ret) + return ret; + + if (rdev->pm.dpm.voltage_response_time == 0) + rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT; + if (rdev->pm.dpm.backbias_response_time == 0) + rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + 0, false, ÷rs); + if (ret) + pi->ref_div = dividers.ref_div + 1; + else + pi->ref_div = R600_REFERENCEDIVIDER_DFLT; + + pi->mclk_strobe_mode_threshold = 40000; + pi->mclk_edc_enable_threshold = 40000; + eg_pi->mclk_edc_wr_enable_threshold = 40000; + + pi->rlp = RV770_RLP_DFLT; + pi->rmp = RV770_RMP_DFLT; + pi->lhp = RV770_LHP_DFLT; + pi->lmp = RV770_LMP_DFLT; + + pi->voltage_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0); + + pi->mvdd_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0); + + eg_pi->vddci_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0); + + rv770_get_engine_memory_ss(rdev); + + pi->asi = RV770_ASI_DFLT; + pi->pasi = CYPRESS_HASI_DFLT; + pi->vrc = CYPRESS_VRC_DFLT; + + pi->power_gating = false; + + if ((rdev->family == CHIP_CYPRESS) || + (rdev->family == CHIP_HEMLOCK)) + pi->gfx_clock_gating = false; + else + pi->gfx_clock_gating = true; + + pi->mg_clock_gating = true; + pi->mgcgtssm = true; + eg_pi->ls_clock_gating = false; + eg_pi->sclk_deep_sleep = false; + + pi->dynamic_pcie_gen2 = true; + + if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE) + pi->thermal_protection = true; + else + pi->thermal_protection = false; + + pi->display_gap = true; + + if (rdev->flags & RADEON_IS_MOBILITY) + pi->dcodt = true; + else + pi->dcodt = false; + + pi->ulps = true; + + eg_pi->dynamic_ac_timing = true; + eg_pi->abm = true; + eg_pi->mcls = true; + eg_pi->light_sleep = true; + eg_pi->memory_transition = true; +#if defined(CONFIG_ACPI) + eg_pi->pcie_performance_request = + radeon_acpi_is_pcie_performance_request_supported(rdev); +#else + eg_pi->pcie_performance_request = false; +#endif + + if ((rdev->family == CHIP_CYPRESS) || + (rdev->family == CHIP_HEMLOCK) || + (rdev->family == CHIP_JUNIPER)) + eg_pi->dll_default_on = true; + else + eg_pi->dll_default_on = false; + + eg_pi->sclk_deep_sleep = false; + pi->mclk_stutter_mode_threshold = 0; + + pi->sram_end = SMC_RAM_END; + + return 0; +} + +void cypress_dpm_fini(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); +} + +bool cypress_dpm_vblank_too_short(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 vblank_time = r600_dpm_get_vblank_time(rdev); + /* we never hit the non-gddr5 limit so disable it */ + u32 switch_limit = pi->mem_gddr5 ? 450 : 0; + + if (vblank_time < switch_limit) + return true; + else + return false; + +} diff --git a/sys/dev/pci/drm/radeon/cypress_dpm.h b/sys/dev/pci/drm/radeon/cypress_dpm.h new file mode 100644 index 00000000000..4c3f18c69f4 --- /dev/null +++ b/sys/dev/pci/drm/radeon/cypress_dpm.h @@ -0,0 +1,160 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __CYPRESS_DPM_H__ +#define __CYPRESS_DPM_H__ + +#include "rv770_dpm.h" +#include "evergreen_smc.h" + +struct evergreen_mc_reg_entry { + u32 mclk_max; + u32 mc_data[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE]; +}; + +struct evergreen_mc_reg_table { + u8 last; + u8 num_entries; + u16 valid_flag; + struct evergreen_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES]; + SMC_Evergreen_MCRegisterAddress mc_reg_address[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE]; +}; + +struct evergreen_ulv_param { + bool supported; + struct rv7xx_pl *pl; +}; + +struct evergreen_arb_registers { + u32 mc_arb_dram_timing; + u32 mc_arb_dram_timing2; + u32 mc_arb_rfsh_rate; + u32 mc_arb_burst_time; +}; + +struct at { + u32 rlp; + u32 rmp; + u32 lhp; + u32 lmp; +}; + +struct evergreen_power_info { + /* must be first! */ + struct rv7xx_power_info rv7xx; + /* flags */ + bool vddci_control; + bool dynamic_ac_timing; + bool abm; + bool mcls; + bool light_sleep; + bool memory_transition; + bool pcie_performance_request; + bool pcie_performance_request_registered; + bool sclk_deep_sleep; + bool dll_default_on; + bool ls_clock_gating; + bool smu_uvd_hs; + bool uvd_enabled; + /* stored values */ + u16 acpi_vddci; + u8 mvdd_high_index; + u8 mvdd_low_index; + u32 mclk_edc_wr_enable_threshold; + struct evergreen_mc_reg_table mc_reg_table; + struct atom_voltage_table vddc_voltage_table; + struct atom_voltage_table vddci_voltage_table; + struct evergreen_arb_registers bootup_arb_registers; + struct evergreen_ulv_param ulv; + struct at ats[2]; + /* smc offsets */ + u16 mc_reg_table_start; + struct radeon_ps current_rps; + struct rv7xx_ps current_ps; + struct radeon_ps requested_rps; + struct rv7xx_ps requested_ps; +}; + +#define CYPRESS_HASI_DFLT 400000 +#define CYPRESS_MGCGTTLOCAL0_DFLT 0x00000000 +#define CYPRESS_MGCGTTLOCAL1_DFLT 0x00000000 +#define CYPRESS_MGCGTTLOCAL2_DFLT 0x00000000 +#define CYPRESS_MGCGTTLOCAL3_DFLT 0x00000000 +#define CYPRESS_MGCGCGTSSMCTRL_DFLT 0x81944bc0 +#define REDWOOD_MGCGCGTSSMCTRL_DFLT 0x6e944040 +#define CEDAR_MGCGCGTSSMCTRL_DFLT 0x46944040 +#define CYPRESS_VRC_DFLT 0xC00033 + +#define PCIE_PERF_REQ_REMOVE_REGISTRY 0 +#define PCIE_PERF_REQ_FORCE_LOWPOWER 1 +#define PCIE_PERF_REQ_PECI_GEN1 2 +#define PCIE_PERF_REQ_PECI_GEN2 3 +#define PCIE_PERF_REQ_PECI_GEN3 4 + +int cypress_convert_power_level_to_smc(struct radeon_device *rdev, + struct rv7xx_pl *pl, + RV770_SMC_HW_PERFORMANCE_LEVEL *level, + u8 watermark_level); +int cypress_populate_smc_acpi_state(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table); +int cypress_populate_smc_voltage_tables(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table); +int cypress_populate_smc_initial_state(struct radeon_device *rdev, + struct radeon_ps *radeon_initial_state, + RV770_SMC_STATETABLE *table); +u32 cypress_calculate_burst_time(struct radeon_device *rdev, + u32 engine_clock, u32 memory_clock); +void cypress_notify_link_speed_change_before_state_change(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state); +int cypress_upload_sw_state(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state); +int cypress_upload_mc_reg_table(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state); +void cypress_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state); +void cypress_notify_link_speed_change_after_state_change(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state); +int cypress_construct_voltage_tables(struct radeon_device *rdev); +int cypress_get_mvdd_configuration(struct radeon_device *rdev); +void cypress_enable_spread_spectrum(struct radeon_device *rdev, + bool enable); +void cypress_enable_display_gap(struct radeon_device *rdev); +int cypress_get_table_locations(struct radeon_device *rdev); +int cypress_populate_mc_reg_table(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state); +void cypress_program_response_times(struct radeon_device *rdev); +int cypress_notify_smc_display_change(struct radeon_device *rdev, + bool has_display); +void cypress_enable_sclk_control(struct radeon_device *rdev, + bool enable); +void cypress_enable_mclk_control(struct radeon_device *rdev, + bool enable); +void cypress_start_dpm(struct radeon_device *rdev); +void cypress_advertise_gen2_capability(struct radeon_device *rdev); +u32 cypress_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf); +u8 cypress_get_mclk_frequency_ratio(struct radeon_device *rdev, + u32 memory_clock, bool strobe_mode); +u8 cypress_get_strobe_mode_settings(struct radeon_device *rdev, u32 mclk); + +#endif diff --git a/sys/dev/pci/drm/radeon/dce3_1_afmt.c b/sys/dev/pci/drm/radeon/dce3_1_afmt.c new file mode 100644 index 00000000000..228e8ac00f2 --- /dev/null +++ b/sys/dev/pci/drm/radeon/dce3_1_afmt.c @@ -0,0 +1,232 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * Copyright 2014 RafaÅ‚ MiÅ‚ecki + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "radeon_audio.h" +#include "r600d.h" + +void dce3_2_afmt_hdmi_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + u32 tmp; + + /* program the speaker allocation */ + tmp = RREG32_ENDPOINT(0, AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER); + tmp &= ~(DP_CONNECTION | SPEAKER_ALLOCATION_MASK); + /* set HDMI mode */ + tmp |= HDMI_CONNECTION; + if (sad_count) + tmp |= SPEAKER_ALLOCATION(sadb[0]); + else + tmp |= SPEAKER_ALLOCATION(5); /* stereo */ + WREG32_ENDPOINT(0, AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER, tmp); +} + +void dce3_2_afmt_dp_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + u32 tmp; + + /* program the speaker allocation */ + tmp = RREG32_ENDPOINT(0, AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER); + tmp &= ~(HDMI_CONNECTION | SPEAKER_ALLOCATION_MASK); + /* set DP mode */ + tmp |= DP_CONNECTION; + if (sad_count) + tmp |= SPEAKER_ALLOCATION(sadb[0]); + else + tmp |= SPEAKER_ALLOCATION(5); /* stereo */ + WREG32_ENDPOINT(0, AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER, tmp); +} + +void dce3_2_afmt_write_sad_regs(struct drm_encoder *encoder, + struct cea_sad *sads, int sad_count) +{ + int i; + struct radeon_device *rdev = encoder->dev->dev_private; + static const u16 eld_reg_to_type[][2] = { + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR0, HDMI_AUDIO_CODING_TYPE_PCM }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR1, HDMI_AUDIO_CODING_TYPE_AC3 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR2, HDMI_AUDIO_CODING_TYPE_MPEG1 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR3, HDMI_AUDIO_CODING_TYPE_MP3 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR4, HDMI_AUDIO_CODING_TYPE_MPEG2 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR5, HDMI_AUDIO_CODING_TYPE_AAC_LC }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR6, HDMI_AUDIO_CODING_TYPE_DTS }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR7, HDMI_AUDIO_CODING_TYPE_ATRAC }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR9, HDMI_AUDIO_CODING_TYPE_EAC3 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR10, HDMI_AUDIO_CODING_TYPE_DTS_HD }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR11, HDMI_AUDIO_CODING_TYPE_MLP }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR13, HDMI_AUDIO_CODING_TYPE_WMA_PRO }, + }; + + for (i = 0; i < ARRAY_SIZE(eld_reg_to_type); i++) { + u32 value = 0; + u8 stereo_freqs = 0; + int max_channels = -1; + int j; + + for (j = 0; j < sad_count; j++) { + struct cea_sad *sad = &sads[j]; + + if (sad->format == eld_reg_to_type[i][1]) { + if (sad->channels > max_channels) { + value = MAX_CHANNELS(sad->channels) | + DESCRIPTOR_BYTE_2(sad->byte2) | + SUPPORTED_FREQUENCIES(sad->freq); + max_channels = sad->channels; + } + + if (sad->format == HDMI_AUDIO_CODING_TYPE_PCM) + stereo_freqs |= sad->freq; + else + break; + } + } + + value |= SUPPORTED_FREQUENCIES_STEREO(stereo_freqs); + + WREG32_ENDPOINT(0, eld_reg_to_type[i][0], value); + } +} + +void dce3_2_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock) +{ + struct radeon_encoder *radeon_encoder; + struct radeon_encoder_atom_dig *dig; + unsigned int max_ratio = clock / 24000; + u32 dto_phase; + u32 wallclock_ratio; + u32 dto_cntl; + + if (!crtc) + return; + + radeon_encoder = to_radeon_encoder(crtc->encoder); + dig = radeon_encoder->enc_priv; + + if (!dig) + return; + + if (max_ratio >= 8) { + dto_phase = 192 * 1000; + wallclock_ratio = 3; + } else if (max_ratio >= 4) { + dto_phase = 96 * 1000; + wallclock_ratio = 2; + } else if (max_ratio >= 2) { + dto_phase = 48 * 1000; + wallclock_ratio = 1; + } else { + dto_phase = 24 * 1000; + wallclock_ratio = 0; + } + + /* Express [24MHz / target pixel clock] as an exact rational + * number (coefficient of two integer numbers. DCCG_AUDIO_DTOx_PHASE + * is the numerator, DCCG_AUDIO_DTOx_MODULE is the denominator + */ + if (dig->dig_encoder == 0) { + dto_cntl = RREG32(DCCG_AUDIO_DTO0_CNTL) & ~DCCG_AUDIO_DTO_WALLCLOCK_RATIO_MASK; + dto_cntl |= DCCG_AUDIO_DTO_WALLCLOCK_RATIO(wallclock_ratio); + WREG32(DCCG_AUDIO_DTO0_CNTL, dto_cntl); + WREG32(DCCG_AUDIO_DTO0_PHASE, dto_phase); + WREG32(DCCG_AUDIO_DTO0_MODULE, clock); + WREG32(DCCG_AUDIO_DTO_SELECT, 0); /* select DTO0 */ + } else { + dto_cntl = RREG32(DCCG_AUDIO_DTO1_CNTL) & ~DCCG_AUDIO_DTO_WALLCLOCK_RATIO_MASK; + dto_cntl |= DCCG_AUDIO_DTO_WALLCLOCK_RATIO(wallclock_ratio); + WREG32(DCCG_AUDIO_DTO1_CNTL, dto_cntl); + WREG32(DCCG_AUDIO_DTO1_PHASE, dto_phase); + WREG32(DCCG_AUDIO_DTO1_MODULE, clock); + WREG32(DCCG_AUDIO_DTO_SELECT, 1); /* select DTO1 */ + } +} + +void dce3_2_hdmi_update_acr(struct drm_encoder *encoder, long offset, + const struct radeon_hdmi_acr *acr) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + + WREG32(DCE3_HDMI0_ACR_PACKET_CONTROL + offset, + HDMI0_ACR_SOURCE | /* select SW CTS value */ + HDMI0_ACR_AUTO_SEND); /* allow hw to sent ACR packets when required */ + + WREG32_P(HDMI0_ACR_32_0 + offset, + HDMI0_ACR_CTS_32(acr->cts_32khz), + ~HDMI0_ACR_CTS_32_MASK); + WREG32_P(HDMI0_ACR_32_1 + offset, + HDMI0_ACR_N_32(acr->n_32khz), + ~HDMI0_ACR_N_32_MASK); + + WREG32_P(HDMI0_ACR_44_0 + offset, + HDMI0_ACR_CTS_44(acr->cts_44_1khz), + ~HDMI0_ACR_CTS_44_MASK); + WREG32_P(HDMI0_ACR_44_1 + offset, + HDMI0_ACR_N_44(acr->n_44_1khz), + ~HDMI0_ACR_N_44_MASK); + + WREG32_P(HDMI0_ACR_48_0 + offset, + HDMI0_ACR_CTS_48(acr->cts_48khz), + ~HDMI0_ACR_CTS_48_MASK); + WREG32_P(HDMI0_ACR_48_1 + offset, + HDMI0_ACR_N_48(acr->n_48khz), + ~HDMI0_ACR_N_48_MASK); +} + +void dce3_2_set_audio_packet(struct drm_encoder *encoder, u32 offset) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + + WREG32(HDMI0_AUDIO_PACKET_CONTROL + offset, + HDMI0_AUDIO_DELAY_EN(1) | /* default audio delay */ + HDMI0_AUDIO_PACKETS_PER_LINE(3)); /* should be suffient for all audio modes and small enough for all hblanks */ + + WREG32(AFMT_AUDIO_PACKET_CONTROL + offset, + AFMT_AUDIO_SAMPLE_SEND | /* send audio packets */ + AFMT_60958_CS_UPDATE); /* allow 60958 channel status fields to be updated */ + + WREG32_OR(HDMI0_INFOFRAME_CONTROL0 + offset, + HDMI0_AUDIO_INFO_SEND | /* enable audio info frames (frames won't be set until audio is enabled) */ + HDMI0_AUDIO_INFO_CONT); /* send audio info frames every frame/field */ + + WREG32_OR(HDMI0_INFOFRAME_CONTROL1 + offset, + HDMI0_AUDIO_INFO_LINE(2)); /* anything other than 0 */ +} + +void dce3_2_set_mute(struct drm_encoder *encoder, u32 offset, bool mute) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + + if (mute) + WREG32_OR(HDMI0_GC + offset, HDMI0_GC_AVMUTE); + else + WREG32_AND(HDMI0_GC + offset, ~HDMI0_GC_AVMUTE); +} diff --git a/sys/dev/pci/drm/radeon/dce6_afmt.c b/sys/dev/pci/drm/radeon/dce6_afmt.c new file mode 100644 index 00000000000..c342965908b --- /dev/null +++ b/sys/dev/pci/drm/radeon/dce6_afmt.c @@ -0,0 +1,318 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include +#include +#include "radeon.h" +#include "radeon_audio.h" +#include "sid.h" + +#define DCE8_DCCG_AUDIO_DTO1_PHASE 0x05b8 +#define DCE8_DCCG_AUDIO_DTO1_MODULE 0x05bc + +u32 dce6_endpoint_rreg(struct radeon_device *rdev, + u32 block_offset, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->end_idx_lock, flags); + WREG32(AZ_F0_CODEC_ENDPOINT_INDEX + block_offset, reg); + r = RREG32(AZ_F0_CODEC_ENDPOINT_DATA + block_offset); + spin_unlock_irqrestore(&rdev->end_idx_lock, flags); + + return r; +} + +void dce6_endpoint_wreg(struct radeon_device *rdev, + u32 block_offset, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->end_idx_lock, flags); + if (ASIC_IS_DCE8(rdev)) + WREG32(AZ_F0_CODEC_ENDPOINT_INDEX + block_offset, reg); + else + WREG32(AZ_F0_CODEC_ENDPOINT_INDEX + block_offset, + AZ_ENDPOINT_REG_WRITE_EN | AZ_ENDPOINT_REG_INDEX(reg)); + WREG32(AZ_F0_CODEC_ENDPOINT_DATA + block_offset, v); + spin_unlock_irqrestore(&rdev->end_idx_lock, flags); +} + +static void dce6_afmt_get_connected_pins(struct radeon_device *rdev) +{ + int i; + u32 offset, tmp; + + for (i = 0; i < rdev->audio.num_pins; i++) { + offset = rdev->audio.pin[i].offset; + tmp = RREG32_ENDPOINT(offset, + AZ_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT); + if (((tmp & PORT_CONNECTIVITY_MASK) >> PORT_CONNECTIVITY_SHIFT) == 1) + rdev->audio.pin[i].connected = false; + else + rdev->audio.pin[i].connected = true; + } +} + +struct r600_audio_pin *dce6_audio_get_pin(struct radeon_device *rdev) +{ + struct drm_encoder *encoder; + struct radeon_encoder *radeon_encoder; + struct radeon_encoder_atom_dig *dig; + struct r600_audio_pin *pin = NULL; + int i, pin_count; + + dce6_afmt_get_connected_pins(rdev); + + for (i = 0; i < rdev->audio.num_pins; i++) { + if (rdev->audio.pin[i].connected) { + pin = &rdev->audio.pin[i]; + pin_count = 0; + + list_for_each_entry(encoder, &rdev->ddev->mode_config.encoder_list, head) { + if (radeon_encoder_is_digital(encoder)) { + radeon_encoder = to_radeon_encoder(encoder); + dig = radeon_encoder->enc_priv; + if (dig->pin == pin) + pin_count++; + } + } + + if (pin_count == 0) + return pin; + } + } + if (!pin) + DRM_ERROR("No connected audio pins found!\n"); + return pin; +} + +void dce6_afmt_select_pin(struct drm_encoder *encoder) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + + if (!dig || !dig->afmt || !dig->pin) + return; + + WREG32(AFMT_AUDIO_SRC_CONTROL + dig->afmt->offset, + AFMT_AUDIO_SRC_SELECT(dig->pin->id)); +} + +void dce6_afmt_write_latency_fields(struct drm_encoder *encoder, + struct drm_connector *connector, + struct drm_display_mode *mode) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + u32 tmp = 0; + + if (!dig || !dig->afmt || !dig->pin) + return; + + if (mode->flags & DRM_MODE_FLAG_INTERLACE) { + if (connector->latency_present[1]) + tmp = VIDEO_LIPSYNC(connector->video_latency[1]) | + AUDIO_LIPSYNC(connector->audio_latency[1]); + else + tmp = VIDEO_LIPSYNC(0) | AUDIO_LIPSYNC(0); + } else { + if (connector->latency_present[0]) + tmp = VIDEO_LIPSYNC(connector->video_latency[0]) | + AUDIO_LIPSYNC(connector->audio_latency[0]); + else + tmp = VIDEO_LIPSYNC(0) | AUDIO_LIPSYNC(0); + } + WREG32_ENDPOINT(dig->pin->offset, + AZ_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC, tmp); +} + +void dce6_afmt_hdmi_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + u32 tmp; + + if (!dig || !dig->afmt || !dig->pin) + return; + + /* program the speaker allocation */ + tmp = RREG32_ENDPOINT(dig->pin->offset, + AZ_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER); + tmp &= ~(DP_CONNECTION | SPEAKER_ALLOCATION_MASK); + /* set HDMI mode */ + tmp |= HDMI_CONNECTION; + if (sad_count) + tmp |= SPEAKER_ALLOCATION(sadb[0]); + else + tmp |= SPEAKER_ALLOCATION(5); /* stereo */ + WREG32_ENDPOINT(dig->pin->offset, + AZ_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER, tmp); +} + +void dce6_afmt_dp_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + u32 tmp; + + if (!dig || !dig->afmt || !dig->pin) + return; + + /* program the speaker allocation */ + tmp = RREG32_ENDPOINT(dig->pin->offset, + AZ_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER); + tmp &= ~(HDMI_CONNECTION | SPEAKER_ALLOCATION_MASK); + /* set DP mode */ + tmp |= DP_CONNECTION; + if (sad_count) + tmp |= SPEAKER_ALLOCATION(sadb[0]); + else + tmp |= SPEAKER_ALLOCATION(5); /* stereo */ + WREG32_ENDPOINT(dig->pin->offset, + AZ_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER, tmp); +} + +void dce6_afmt_write_sad_regs(struct drm_encoder *encoder, + struct cea_sad *sads, int sad_count) +{ + int i; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + struct radeon_device *rdev = encoder->dev->dev_private; + static const u16 eld_reg_to_type[][2] = { + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0, HDMI_AUDIO_CODING_TYPE_PCM }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1, HDMI_AUDIO_CODING_TYPE_AC3 }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2, HDMI_AUDIO_CODING_TYPE_MPEG1 }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3, HDMI_AUDIO_CODING_TYPE_MP3 }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4, HDMI_AUDIO_CODING_TYPE_MPEG2 }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5, HDMI_AUDIO_CODING_TYPE_AAC_LC }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6, HDMI_AUDIO_CODING_TYPE_DTS }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7, HDMI_AUDIO_CODING_TYPE_ATRAC }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9, HDMI_AUDIO_CODING_TYPE_EAC3 }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10, HDMI_AUDIO_CODING_TYPE_DTS_HD }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11, HDMI_AUDIO_CODING_TYPE_MLP }, + { AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13, HDMI_AUDIO_CODING_TYPE_WMA_PRO }, + }; + + if (!dig || !dig->afmt || !dig->pin) + return; + + for (i = 0; i < ARRAY_SIZE(eld_reg_to_type); i++) { + u32 value = 0; + u8 stereo_freqs = 0; + int max_channels = -1; + int j; + + for (j = 0; j < sad_count; j++) { + struct cea_sad *sad = &sads[j]; + + if (sad->format == eld_reg_to_type[i][1]) { + if (sad->channels > max_channels) { + value = MAX_CHANNELS(sad->channels) | + DESCRIPTOR_BYTE_2(sad->byte2) | + SUPPORTED_FREQUENCIES(sad->freq); + max_channels = sad->channels; + } + + if (sad->format == HDMI_AUDIO_CODING_TYPE_PCM) + stereo_freqs |= sad->freq; + else + break; + } + } + + value |= SUPPORTED_FREQUENCIES_STEREO(stereo_freqs); + + WREG32_ENDPOINT(dig->pin->offset, eld_reg_to_type[i][0], value); + } +} + +void dce6_audio_enable(struct radeon_device *rdev, + struct r600_audio_pin *pin, + u8 enable_mask) +{ + if (!pin) + return; + + WREG32_ENDPOINT(pin->offset, AZ_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL, + enable_mask ? AUDIO_ENABLED : 0); +} + +void dce6_hdmi_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock) +{ + /* Two dtos; generally use dto0 for HDMI */ + u32 value = 0; + + if (crtc) + value |= DCCG_AUDIO_DTO0_SOURCE_SEL(crtc->crtc_id); + + WREG32(DCCG_AUDIO_DTO_SOURCE, value); + + /* Express [24MHz / target pixel clock] as an exact rational + * number (coefficient of two integer numbers. DCCG_AUDIO_DTOx_PHASE + * is the numerator, DCCG_AUDIO_DTOx_MODULE is the denominator + */ + WREG32(DCCG_AUDIO_DTO0_PHASE, 24000); + WREG32(DCCG_AUDIO_DTO0_MODULE, clock); +} + +void dce6_dp_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock) +{ + /* Two dtos; generally use dto1 for DP */ + u32 value = 0; + value |= DCCG_AUDIO_DTO_SEL; + + if (crtc) + value |= DCCG_AUDIO_DTO0_SOURCE_SEL(crtc->crtc_id); + + WREG32(DCCG_AUDIO_DTO_SOURCE, value); + + /* Express [24MHz / target pixel clock] as an exact rational + * number (coefficient of two integer numbers. DCCG_AUDIO_DTOx_PHASE + * is the numerator, DCCG_AUDIO_DTOx_MODULE is the denominator + */ + if (ASIC_IS_DCE8(rdev)) { + unsigned int div = (RREG32(DENTIST_DISPCLK_CNTL) & + DENTIST_DPREFCLK_WDIVIDER_MASK) >> + DENTIST_DPREFCLK_WDIVIDER_SHIFT; + div = radeon_audio_decode_dfs_div(div); + + if (div) + clock = clock * 100 / div; + + WREG32(DCE8_DCCG_AUDIO_DTO1_PHASE, 24000); + WREG32(DCE8_DCCG_AUDIO_DTO1_MODULE, clock); + } else { + WREG32(DCCG_AUDIO_DTO1_PHASE, 24000); + WREG32(DCCG_AUDIO_DTO1_MODULE, clock); + } +} diff --git a/sys/dev/pci/drm/radeon/drm_buffer.h b/sys/dev/pci/drm/radeon/drm_buffer.h new file mode 100644 index 00000000000..4eaa26fbad1 --- /dev/null +++ b/sys/dev/pci/drm/radeon/drm_buffer.h @@ -0,0 +1,148 @@ +/************************************************************************** + * + * Copyright 2010 Pauli Nieminen. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * + **************************************************************************/ +/* + * Multipart buffer for coping data which is larger than the page size. + * + * Authors: + * Pauli Nieminen + */ + +#ifndef _DRM_BUFFER_H_ +#define _DRM_BUFFER_H_ + +#include + +struct drm_buffer { + int iterator; + int size; + char *data[]; +}; + + +/** + * Return the index of page that buffer is currently pointing at. + */ +static inline int drm_buffer_page(struct drm_buffer *buf) +{ + return buf->iterator / PAGE_SIZE; +} +/** + * Return the index of the current byte in the page + */ +static inline int drm_buffer_index(struct drm_buffer *buf) +{ + return buf->iterator & (PAGE_SIZE - 1); +} +/** + * Return number of bytes that is left to process + */ +static inline int drm_buffer_unprocessed(struct drm_buffer *buf) +{ + return buf->size - buf->iterator; +} + +/** + * Advance the buffer iterator number of bytes that is given. + */ +static inline void drm_buffer_advance(struct drm_buffer *buf, int bytes) +{ + buf->iterator += bytes; +} + +/** + * Allocate the drm buffer object. + * + * buf: A pointer to a pointer where the object is stored. + * size: The number of bytes to allocate. + */ +extern int drm_buffer_alloc(struct drm_buffer **buf, int size); + +/** + * Copy the user data to the begin of the buffer and reset the processing + * iterator. + * + * user_data: A pointer the data that is copied to the buffer. + * size: The Number of bytes to copy. + */ +extern int drm_buffer_copy_from_user(struct drm_buffer *buf, + void __user *user_data, int size); + +/** + * Free the drm buffer object + */ +extern void drm_buffer_free(struct drm_buffer *buf); + +/** + * Read an object from buffer that may be split to multiple parts. If object + * is not split function just returns the pointer to object in buffer. But in + * case of split object data is copied to given stack object that is suplied + * by caller. + * + * The processing location of the buffer is also advanced to the next byte + * after the object. + * + * objsize: The size of the objet in bytes. + * stack_obj: A pointer to a memory location where object can be copied. + */ +extern void *drm_buffer_read_object(struct drm_buffer *buf, + int objsize, void *stack_obj); + +/** + * Returns the pointer to the dword which is offset number of elements from the + * current processing location. + * + * Caller must make sure that dword is not split in the buffer. This + * requirement is easily met if all the sizes of objects in buffer are + * multiples of dword and PAGE_SIZE is multiple dword. + * + * Call to this function doesn't change the processing location. + * + * offset: The index of the dword relative to the internat iterator. + */ +static inline void *drm_buffer_pointer_to_dword(struct drm_buffer *buffer, + int offset) +{ + int iter = buffer->iterator + offset * 4; + return &buffer->data[iter / PAGE_SIZE][iter & (PAGE_SIZE - 1)]; +} +/** + * Returns the pointer to the dword which is offset number of elements from + * the current processing location. + * + * Call to this function doesn't change the processing location. + * + * offset: The index of the byte relative to the internat iterator. + */ +static inline void *drm_buffer_pointer_to_byte(struct drm_buffer *buffer, + int offset) +{ + int iter = buffer->iterator + offset; + return &buffer->data[iter / PAGE_SIZE][iter & (PAGE_SIZE - 1)]; +} + +#endif diff --git a/sys/dev/pci/drm/radeon/evergreen.c b/sys/dev/pci/drm/radeon/evergreen.c index e9ae50dbddb..255f6fe90e1 100644 --- a/sys/dev/pci/drm/radeon/evergreen.c +++ b/sys/dev/pci/drm/radeon/evergreen.c @@ -1,4 +1,3 @@ -/* $OpenBSD: evergreen.c,v 1.21 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * @@ -25,15 +24,83 @@ #include #include "radeon.h" #include "radeon_asic.h" +#include "radeon_audio.h" #include #include "evergreend.h" #include "atom.h" #include "avivod.h" #include "evergreen_reg.h" #include "evergreen_blit_shaders.h" +#include "radeon_ucode.h" -#define EVERGREEN_PFP_UCODE_SIZE 1120 -#define EVERGREEN_PM4_UCODE_SIZE 1376 +/* + * Indirect registers accessor + */ +u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->cg_idx_lock, flags); + WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); + r = RREG32(EVERGREEN_CG_IND_DATA); + spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); + return r; +} + +void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->cg_idx_lock, flags); + WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); + WREG32(EVERGREEN_CG_IND_DATA, (v)); + spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); +} + +u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->pif_idx_lock, flags); + WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); + r = RREG32(EVERGREEN_PIF_PHY0_DATA); + spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); + return r; +} + +void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->pif_idx_lock, flags); + WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); + WREG32(EVERGREEN_PIF_PHY0_DATA, (v)); + spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); +} + +u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->pif_idx_lock, flags); + WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); + r = RREG32(EVERGREEN_PIF_PHY1_DATA); + spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); + return r; +} + +void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->pif_idx_lock, flags); + WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); + WREG32(EVERGREEN_PIF_PHY1_DATA, (v)); + spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); +} static const u32 crtc_offsets[6] = { @@ -45,11 +112,994 @@ static const u32 crtc_offsets[6] = EVERGREEN_CRTC5_REGISTER_OFFSET }; +#include "clearstate_evergreen.h" + +static const u32 sumo_rlc_save_restore_register_list[] = +{ + 0x98fc, + 0x9830, + 0x9834, + 0x9838, + 0x9870, + 0x9874, + 0x8a14, + 0x8b24, + 0x8bcc, + 0x8b10, + 0x8d00, + 0x8d04, + 0x8c00, + 0x8c04, + 0x8c08, + 0x8c0c, + 0x8d8c, + 0x8c20, + 0x8c24, + 0x8c28, + 0x8c18, + 0x8c1c, + 0x8cf0, + 0x8e2c, + 0x8e38, + 0x8c30, + 0x9508, + 0x9688, + 0x9608, + 0x960c, + 0x9610, + 0x9614, + 0x88c4, + 0x88d4, + 0xa008, + 0x900c, + 0x9100, + 0x913c, + 0x98f8, + 0x98f4, + 0x9b7c, + 0x3f8c, + 0x8950, + 0x8954, + 0x8a18, + 0x8b28, + 0x9144, + 0x9148, + 0x914c, + 0x3f90, + 0x3f94, + 0x915c, + 0x9160, + 0x9178, + 0x917c, + 0x9180, + 0x918c, + 0x9190, + 0x9194, + 0x9198, + 0x919c, + 0x91a8, + 0x91ac, + 0x91b0, + 0x91b4, + 0x91b8, + 0x91c4, + 0x91c8, + 0x91cc, + 0x91d0, + 0x91d4, + 0x91e0, + 0x91e4, + 0x91ec, + 0x91f0, + 0x91f4, + 0x9200, + 0x9204, + 0x929c, + 0x9150, + 0x802c, +}; + static void evergreen_gpu_init(struct radeon_device *rdev); void evergreen_fini(struct radeon_device *rdev); void evergreen_pcie_gen2_enable(struct radeon_device *rdev); +void evergreen_program_aspm(struct radeon_device *rdev); extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev, int ring, u32 cp_int_cntl); +extern void cayman_vm_decode_fault(struct radeon_device *rdev, + u32 status, u32 addr); +void cik_init_cp_pg_table(struct radeon_device *rdev); + +extern u32 si_get_csb_size(struct radeon_device *rdev); +extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); +extern u32 cik_get_csb_size(struct radeon_device *rdev); +extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); +extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev); + +static const u32 evergreen_golden_registers[] = +{ + 0x3f90, 0xffff0000, 0xff000000, + 0x9148, 0xffff0000, 0xff000000, + 0x3f94, 0xffff0000, 0xff000000, + 0x914c, 0xffff0000, 0xff000000, + 0x9b7c, 0xffffffff, 0x00000000, + 0x8a14, 0xffffffff, 0x00000007, + 0x8b10, 0xffffffff, 0x00000000, + 0x960c, 0xffffffff, 0x54763210, + 0x88c4, 0xffffffff, 0x000000c2, + 0x88d4, 0xffffffff, 0x00000010, + 0x8974, 0xffffffff, 0x00000000, + 0xc78, 0x00000080, 0x00000080, + 0x5eb4, 0xffffffff, 0x00000002, + 0x5e78, 0xffffffff, 0x001000f0, + 0x6104, 0x01000300, 0x00000000, + 0x5bc0, 0x00300000, 0x00000000, + 0x7030, 0xffffffff, 0x00000011, + 0x7c30, 0xffffffff, 0x00000011, + 0x10830, 0xffffffff, 0x00000011, + 0x11430, 0xffffffff, 0x00000011, + 0x12030, 0xffffffff, 0x00000011, + 0x12c30, 0xffffffff, 0x00000011, + 0xd02c, 0xffffffff, 0x08421000, + 0x240c, 0xffffffff, 0x00000380, + 0x8b24, 0xffffffff, 0x00ff0fff, + 0x28a4c, 0x06000000, 0x06000000, + 0x10c, 0x00000001, 0x00000001, + 0x8d00, 0xffffffff, 0x100e4848, + 0x8d04, 0xffffffff, 0x00164745, + 0x8c00, 0xffffffff, 0xe4000003, + 0x8c04, 0xffffffff, 0x40600060, + 0x8c08, 0xffffffff, 0x001c001c, + 0x8cf0, 0xffffffff, 0x08e00620, + 0x8c20, 0xffffffff, 0x00800080, + 0x8c24, 0xffffffff, 0x00800080, + 0x8c18, 0xffffffff, 0x20202078, + 0x8c1c, 0xffffffff, 0x00001010, + 0x28350, 0xffffffff, 0x00000000, + 0xa008, 0xffffffff, 0x00010000, + 0x5c4, 0xffffffff, 0x00000001, + 0x9508, 0xffffffff, 0x00000002, + 0x913c, 0x0000000f, 0x0000000a +}; + +static const u32 evergreen_golden_registers2[] = +{ + 0x2f4c, 0xffffffff, 0x00000000, + 0x54f4, 0xffffffff, 0x00000000, + 0x54f0, 0xffffffff, 0x00000000, + 0x5498, 0xffffffff, 0x00000000, + 0x549c, 0xffffffff, 0x00000000, + 0x5494, 0xffffffff, 0x00000000, + 0x53cc, 0xffffffff, 0x00000000, + 0x53c8, 0xffffffff, 0x00000000, + 0x53c4, 0xffffffff, 0x00000000, + 0x53c0, 0xffffffff, 0x00000000, + 0x53bc, 0xffffffff, 0x00000000, + 0x53b8, 0xffffffff, 0x00000000, + 0x53b4, 0xffffffff, 0x00000000, + 0x53b0, 0xffffffff, 0x00000000 +}; + +static const u32 cypress_mgcg_init[] = +{ + 0x802c, 0xffffffff, 0xc0000000, + 0x5448, 0xffffffff, 0x00000100, + 0x55e4, 0xffffffff, 0x00000100, + 0x160c, 0xffffffff, 0x00000100, + 0x5644, 0xffffffff, 0x00000100, + 0xc164, 0xffffffff, 0x00000100, + 0x8a18, 0xffffffff, 0x00000100, + 0x897c, 0xffffffff, 0x06000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x9a60, 0xffffffff, 0x00000100, + 0x9868, 0xffffffff, 0x00000100, + 0x8d58, 0xffffffff, 0x00000100, + 0x9510, 0xffffffff, 0x00000100, + 0x949c, 0xffffffff, 0x00000100, + 0x9654, 0xffffffff, 0x00000100, + 0x9030, 0xffffffff, 0x00000100, + 0x9034, 0xffffffff, 0x00000100, + 0x9038, 0xffffffff, 0x00000100, + 0x903c, 0xffffffff, 0x00000100, + 0x9040, 0xffffffff, 0x00000100, + 0xa200, 0xffffffff, 0x00000100, + 0xa204, 0xffffffff, 0x00000100, + 0xa208, 0xffffffff, 0x00000100, + 0xa20c, 0xffffffff, 0x00000100, + 0x971c, 0xffffffff, 0x00000100, + 0x977c, 0xffffffff, 0x00000100, + 0x3f80, 0xffffffff, 0x00000100, + 0xa210, 0xffffffff, 0x00000100, + 0xa214, 0xffffffff, 0x00000100, + 0x4d8, 0xffffffff, 0x00000100, + 0x9784, 0xffffffff, 0x00000100, + 0x9698, 0xffffffff, 0x00000100, + 0x4d4, 0xffffffff, 0x00000200, + 0x30cc, 0xffffffff, 0x00000100, + 0xd0c0, 0xffffffff, 0xff000100, + 0x802c, 0xffffffff, 0x40000000, + 0x915c, 0xffffffff, 0x00010000, + 0x9160, 0xffffffff, 0x00030002, + 0x9178, 0xffffffff, 0x00070000, + 0x917c, 0xffffffff, 0x00030002, + 0x9180, 0xffffffff, 0x00050004, + 0x918c, 0xffffffff, 0x00010006, + 0x9190, 0xffffffff, 0x00090008, + 0x9194, 0xffffffff, 0x00070000, + 0x9198, 0xffffffff, 0x00030002, + 0x919c, 0xffffffff, 0x00050004, + 0x91a8, 0xffffffff, 0x00010006, + 0x91ac, 0xffffffff, 0x00090008, + 0x91b0, 0xffffffff, 0x00070000, + 0x91b4, 0xffffffff, 0x00030002, + 0x91b8, 0xffffffff, 0x00050004, + 0x91c4, 0xffffffff, 0x00010006, + 0x91c8, 0xffffffff, 0x00090008, + 0x91cc, 0xffffffff, 0x00070000, + 0x91d0, 0xffffffff, 0x00030002, + 0x91d4, 0xffffffff, 0x00050004, + 0x91e0, 0xffffffff, 0x00010006, + 0x91e4, 0xffffffff, 0x00090008, + 0x91e8, 0xffffffff, 0x00000000, + 0x91ec, 0xffffffff, 0x00070000, + 0x91f0, 0xffffffff, 0x00030002, + 0x91f4, 0xffffffff, 0x00050004, + 0x9200, 0xffffffff, 0x00010006, + 0x9204, 0xffffffff, 0x00090008, + 0x9208, 0xffffffff, 0x00070000, + 0x920c, 0xffffffff, 0x00030002, + 0x9210, 0xffffffff, 0x00050004, + 0x921c, 0xffffffff, 0x00010006, + 0x9220, 0xffffffff, 0x00090008, + 0x9224, 0xffffffff, 0x00070000, + 0x9228, 0xffffffff, 0x00030002, + 0x922c, 0xffffffff, 0x00050004, + 0x9238, 0xffffffff, 0x00010006, + 0x923c, 0xffffffff, 0x00090008, + 0x9240, 0xffffffff, 0x00070000, + 0x9244, 0xffffffff, 0x00030002, + 0x9248, 0xffffffff, 0x00050004, + 0x9254, 0xffffffff, 0x00010006, + 0x9258, 0xffffffff, 0x00090008, + 0x925c, 0xffffffff, 0x00070000, + 0x9260, 0xffffffff, 0x00030002, + 0x9264, 0xffffffff, 0x00050004, + 0x9270, 0xffffffff, 0x00010006, + 0x9274, 0xffffffff, 0x00090008, + 0x9278, 0xffffffff, 0x00070000, + 0x927c, 0xffffffff, 0x00030002, + 0x9280, 0xffffffff, 0x00050004, + 0x928c, 0xffffffff, 0x00010006, + 0x9290, 0xffffffff, 0x00090008, + 0x9294, 0xffffffff, 0x00000000, + 0x929c, 0xffffffff, 0x00000001, + 0x802c, 0xffffffff, 0x40010000, + 0x915c, 0xffffffff, 0x00010000, + 0x9160, 0xffffffff, 0x00030002, + 0x9178, 0xffffffff, 0x00070000, + 0x917c, 0xffffffff, 0x00030002, + 0x9180, 0xffffffff, 0x00050004, + 0x918c, 0xffffffff, 0x00010006, + 0x9190, 0xffffffff, 0x00090008, + 0x9194, 0xffffffff, 0x00070000, + 0x9198, 0xffffffff, 0x00030002, + 0x919c, 0xffffffff, 0x00050004, + 0x91a8, 0xffffffff, 0x00010006, + 0x91ac, 0xffffffff, 0x00090008, + 0x91b0, 0xffffffff, 0x00070000, + 0x91b4, 0xffffffff, 0x00030002, + 0x91b8, 0xffffffff, 0x00050004, + 0x91c4, 0xffffffff, 0x00010006, + 0x91c8, 0xffffffff, 0x00090008, + 0x91cc, 0xffffffff, 0x00070000, + 0x91d0, 0xffffffff, 0x00030002, + 0x91d4, 0xffffffff, 0x00050004, + 0x91e0, 0xffffffff, 0x00010006, + 0x91e4, 0xffffffff, 0x00090008, + 0x91e8, 0xffffffff, 0x00000000, + 0x91ec, 0xffffffff, 0x00070000, + 0x91f0, 0xffffffff, 0x00030002, + 0x91f4, 0xffffffff, 0x00050004, + 0x9200, 0xffffffff, 0x00010006, + 0x9204, 0xffffffff, 0x00090008, + 0x9208, 0xffffffff, 0x00070000, + 0x920c, 0xffffffff, 0x00030002, + 0x9210, 0xffffffff, 0x00050004, + 0x921c, 0xffffffff, 0x00010006, + 0x9220, 0xffffffff, 0x00090008, + 0x9224, 0xffffffff, 0x00070000, + 0x9228, 0xffffffff, 0x00030002, + 0x922c, 0xffffffff, 0x00050004, + 0x9238, 0xffffffff, 0x00010006, + 0x923c, 0xffffffff, 0x00090008, + 0x9240, 0xffffffff, 0x00070000, + 0x9244, 0xffffffff, 0x00030002, + 0x9248, 0xffffffff, 0x00050004, + 0x9254, 0xffffffff, 0x00010006, + 0x9258, 0xffffffff, 0x00090008, + 0x925c, 0xffffffff, 0x00070000, + 0x9260, 0xffffffff, 0x00030002, + 0x9264, 0xffffffff, 0x00050004, + 0x9270, 0xffffffff, 0x00010006, + 0x9274, 0xffffffff, 0x00090008, + 0x9278, 0xffffffff, 0x00070000, + 0x927c, 0xffffffff, 0x00030002, + 0x9280, 0xffffffff, 0x00050004, + 0x928c, 0xffffffff, 0x00010006, + 0x9290, 0xffffffff, 0x00090008, + 0x9294, 0xffffffff, 0x00000000, + 0x929c, 0xffffffff, 0x00000001, + 0x802c, 0xffffffff, 0xc0000000 +}; + +static const u32 redwood_mgcg_init[] = +{ + 0x802c, 0xffffffff, 0xc0000000, + 0x5448, 0xffffffff, 0x00000100, + 0x55e4, 0xffffffff, 0x00000100, + 0x160c, 0xffffffff, 0x00000100, + 0x5644, 0xffffffff, 0x00000100, + 0xc164, 0xffffffff, 0x00000100, + 0x8a18, 0xffffffff, 0x00000100, + 0x897c, 0xffffffff, 0x06000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x9a60, 0xffffffff, 0x00000100, + 0x9868, 0xffffffff, 0x00000100, + 0x8d58, 0xffffffff, 0x00000100, + 0x9510, 0xffffffff, 0x00000100, + 0x949c, 0xffffffff, 0x00000100, + 0x9654, 0xffffffff, 0x00000100, + 0x9030, 0xffffffff, 0x00000100, + 0x9034, 0xffffffff, 0x00000100, + 0x9038, 0xffffffff, 0x00000100, + 0x903c, 0xffffffff, 0x00000100, + 0x9040, 0xffffffff, 0x00000100, + 0xa200, 0xffffffff, 0x00000100, + 0xa204, 0xffffffff, 0x00000100, + 0xa208, 0xffffffff, 0x00000100, + 0xa20c, 0xffffffff, 0x00000100, + 0x971c, 0xffffffff, 0x00000100, + 0x977c, 0xffffffff, 0x00000100, + 0x3f80, 0xffffffff, 0x00000100, + 0xa210, 0xffffffff, 0x00000100, + 0xa214, 0xffffffff, 0x00000100, + 0x4d8, 0xffffffff, 0x00000100, + 0x9784, 0xffffffff, 0x00000100, + 0x9698, 0xffffffff, 0x00000100, + 0x4d4, 0xffffffff, 0x00000200, + 0x30cc, 0xffffffff, 0x00000100, + 0xd0c0, 0xffffffff, 0xff000100, + 0x802c, 0xffffffff, 0x40000000, + 0x915c, 0xffffffff, 0x00010000, + 0x9160, 0xffffffff, 0x00030002, + 0x9178, 0xffffffff, 0x00070000, + 0x917c, 0xffffffff, 0x00030002, + 0x9180, 0xffffffff, 0x00050004, + 0x918c, 0xffffffff, 0x00010006, + 0x9190, 0xffffffff, 0x00090008, + 0x9194, 0xffffffff, 0x00070000, + 0x9198, 0xffffffff, 0x00030002, + 0x919c, 0xffffffff, 0x00050004, + 0x91a8, 0xffffffff, 0x00010006, + 0x91ac, 0xffffffff, 0x00090008, + 0x91b0, 0xffffffff, 0x00070000, + 0x91b4, 0xffffffff, 0x00030002, + 0x91b8, 0xffffffff, 0x00050004, + 0x91c4, 0xffffffff, 0x00010006, + 0x91c8, 0xffffffff, 0x00090008, + 0x91cc, 0xffffffff, 0x00070000, + 0x91d0, 0xffffffff, 0x00030002, + 0x91d4, 0xffffffff, 0x00050004, + 0x91e0, 0xffffffff, 0x00010006, + 0x91e4, 0xffffffff, 0x00090008, + 0x91e8, 0xffffffff, 0x00000000, + 0x91ec, 0xffffffff, 0x00070000, + 0x91f0, 0xffffffff, 0x00030002, + 0x91f4, 0xffffffff, 0x00050004, + 0x9200, 0xffffffff, 0x00010006, + 0x9204, 0xffffffff, 0x00090008, + 0x9294, 0xffffffff, 0x00000000, + 0x929c, 0xffffffff, 0x00000001, + 0x802c, 0xffffffff, 0xc0000000 +}; + +static const u32 cedar_golden_registers[] = +{ + 0x3f90, 0xffff0000, 0xff000000, + 0x9148, 0xffff0000, 0xff000000, + 0x3f94, 0xffff0000, 0xff000000, + 0x914c, 0xffff0000, 0xff000000, + 0x9b7c, 0xffffffff, 0x00000000, + 0x8a14, 0xffffffff, 0x00000007, + 0x8b10, 0xffffffff, 0x00000000, + 0x960c, 0xffffffff, 0x54763210, + 0x88c4, 0xffffffff, 0x000000c2, + 0x88d4, 0xffffffff, 0x00000000, + 0x8974, 0xffffffff, 0x00000000, + 0xc78, 0x00000080, 0x00000080, + 0x5eb4, 0xffffffff, 0x00000002, + 0x5e78, 0xffffffff, 0x001000f0, + 0x6104, 0x01000300, 0x00000000, + 0x5bc0, 0x00300000, 0x00000000, + 0x7030, 0xffffffff, 0x00000011, + 0x7c30, 0xffffffff, 0x00000011, + 0x10830, 0xffffffff, 0x00000011, + 0x11430, 0xffffffff, 0x00000011, + 0xd02c, 0xffffffff, 0x08421000, + 0x240c, 0xffffffff, 0x00000380, + 0x8b24, 0xffffffff, 0x00ff0fff, + 0x28a4c, 0x06000000, 0x06000000, + 0x10c, 0x00000001, 0x00000001, + 0x8d00, 0xffffffff, 0x100e4848, + 0x8d04, 0xffffffff, 0x00164745, + 0x8c00, 0xffffffff, 0xe4000003, + 0x8c04, 0xffffffff, 0x40600060, + 0x8c08, 0xffffffff, 0x001c001c, + 0x8cf0, 0xffffffff, 0x08e00410, + 0x8c20, 0xffffffff, 0x00800080, + 0x8c24, 0xffffffff, 0x00800080, + 0x8c18, 0xffffffff, 0x20202078, + 0x8c1c, 0xffffffff, 0x00001010, + 0x28350, 0xffffffff, 0x00000000, + 0xa008, 0xffffffff, 0x00010000, + 0x5c4, 0xffffffff, 0x00000001, + 0x9508, 0xffffffff, 0x00000002 +}; + +static const u32 cedar_mgcg_init[] = +{ + 0x802c, 0xffffffff, 0xc0000000, + 0x5448, 0xffffffff, 0x00000100, + 0x55e4, 0xffffffff, 0x00000100, + 0x160c, 0xffffffff, 0x00000100, + 0x5644, 0xffffffff, 0x00000100, + 0xc164, 0xffffffff, 0x00000100, + 0x8a18, 0xffffffff, 0x00000100, + 0x897c, 0xffffffff, 0x06000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x9a60, 0xffffffff, 0x00000100, + 0x9868, 0xffffffff, 0x00000100, + 0x8d58, 0xffffffff, 0x00000100, + 0x9510, 0xffffffff, 0x00000100, + 0x949c, 0xffffffff, 0x00000100, + 0x9654, 0xffffffff, 0x00000100, + 0x9030, 0xffffffff, 0x00000100, + 0x9034, 0xffffffff, 0x00000100, + 0x9038, 0xffffffff, 0x00000100, + 0x903c, 0xffffffff, 0x00000100, + 0x9040, 0xffffffff, 0x00000100, + 0xa200, 0xffffffff, 0x00000100, + 0xa204, 0xffffffff, 0x00000100, + 0xa208, 0xffffffff, 0x00000100, + 0xa20c, 0xffffffff, 0x00000100, + 0x971c, 0xffffffff, 0x00000100, + 0x977c, 0xffffffff, 0x00000100, + 0x3f80, 0xffffffff, 0x00000100, + 0xa210, 0xffffffff, 0x00000100, + 0xa214, 0xffffffff, 0x00000100, + 0x4d8, 0xffffffff, 0x00000100, + 0x9784, 0xffffffff, 0x00000100, + 0x9698, 0xffffffff, 0x00000100, + 0x4d4, 0xffffffff, 0x00000200, + 0x30cc, 0xffffffff, 0x00000100, + 0xd0c0, 0xffffffff, 0xff000100, + 0x802c, 0xffffffff, 0x40000000, + 0x915c, 0xffffffff, 0x00010000, + 0x9178, 0xffffffff, 0x00050000, + 0x917c, 0xffffffff, 0x00030002, + 0x918c, 0xffffffff, 0x00010004, + 0x9190, 0xffffffff, 0x00070006, + 0x9194, 0xffffffff, 0x00050000, + 0x9198, 0xffffffff, 0x00030002, + 0x91a8, 0xffffffff, 0x00010004, + 0x91ac, 0xffffffff, 0x00070006, + 0x91e8, 0xffffffff, 0x00000000, + 0x9294, 0xffffffff, 0x00000000, + 0x929c, 0xffffffff, 0x00000001, + 0x802c, 0xffffffff, 0xc0000000 +}; + +static const u32 juniper_mgcg_init[] = +{ + 0x802c, 0xffffffff, 0xc0000000, + 0x5448, 0xffffffff, 0x00000100, + 0x55e4, 0xffffffff, 0x00000100, + 0x160c, 0xffffffff, 0x00000100, + 0x5644, 0xffffffff, 0x00000100, + 0xc164, 0xffffffff, 0x00000100, + 0x8a18, 0xffffffff, 0x00000100, + 0x897c, 0xffffffff, 0x06000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x9a60, 0xffffffff, 0x00000100, + 0x9868, 0xffffffff, 0x00000100, + 0x8d58, 0xffffffff, 0x00000100, + 0x9510, 0xffffffff, 0x00000100, + 0x949c, 0xffffffff, 0x00000100, + 0x9654, 0xffffffff, 0x00000100, + 0x9030, 0xffffffff, 0x00000100, + 0x9034, 0xffffffff, 0x00000100, + 0x9038, 0xffffffff, 0x00000100, + 0x903c, 0xffffffff, 0x00000100, + 0x9040, 0xffffffff, 0x00000100, + 0xa200, 0xffffffff, 0x00000100, + 0xa204, 0xffffffff, 0x00000100, + 0xa208, 0xffffffff, 0x00000100, + 0xa20c, 0xffffffff, 0x00000100, + 0x971c, 0xffffffff, 0x00000100, + 0xd0c0, 0xffffffff, 0xff000100, + 0x802c, 0xffffffff, 0x40000000, + 0x915c, 0xffffffff, 0x00010000, + 0x9160, 0xffffffff, 0x00030002, + 0x9178, 0xffffffff, 0x00070000, + 0x917c, 0xffffffff, 0x00030002, + 0x9180, 0xffffffff, 0x00050004, + 0x918c, 0xffffffff, 0x00010006, + 0x9190, 0xffffffff, 0x00090008, + 0x9194, 0xffffffff, 0x00070000, + 0x9198, 0xffffffff, 0x00030002, + 0x919c, 0xffffffff, 0x00050004, + 0x91a8, 0xffffffff, 0x00010006, + 0x91ac, 0xffffffff, 0x00090008, + 0x91b0, 0xffffffff, 0x00070000, + 0x91b4, 0xffffffff, 0x00030002, + 0x91b8, 0xffffffff, 0x00050004, + 0x91c4, 0xffffffff, 0x00010006, + 0x91c8, 0xffffffff, 0x00090008, + 0x91cc, 0xffffffff, 0x00070000, + 0x91d0, 0xffffffff, 0x00030002, + 0x91d4, 0xffffffff, 0x00050004, + 0x91e0, 0xffffffff, 0x00010006, + 0x91e4, 0xffffffff, 0x00090008, + 0x91e8, 0xffffffff, 0x00000000, + 0x91ec, 0xffffffff, 0x00070000, + 0x91f0, 0xffffffff, 0x00030002, + 0x91f4, 0xffffffff, 0x00050004, + 0x9200, 0xffffffff, 0x00010006, + 0x9204, 0xffffffff, 0x00090008, + 0x9208, 0xffffffff, 0x00070000, + 0x920c, 0xffffffff, 0x00030002, + 0x9210, 0xffffffff, 0x00050004, + 0x921c, 0xffffffff, 0x00010006, + 0x9220, 0xffffffff, 0x00090008, + 0x9224, 0xffffffff, 0x00070000, + 0x9228, 0xffffffff, 0x00030002, + 0x922c, 0xffffffff, 0x00050004, + 0x9238, 0xffffffff, 0x00010006, + 0x923c, 0xffffffff, 0x00090008, + 0x9240, 0xffffffff, 0x00070000, + 0x9244, 0xffffffff, 0x00030002, + 0x9248, 0xffffffff, 0x00050004, + 0x9254, 0xffffffff, 0x00010006, + 0x9258, 0xffffffff, 0x00090008, + 0x925c, 0xffffffff, 0x00070000, + 0x9260, 0xffffffff, 0x00030002, + 0x9264, 0xffffffff, 0x00050004, + 0x9270, 0xffffffff, 0x00010006, + 0x9274, 0xffffffff, 0x00090008, + 0x9278, 0xffffffff, 0x00070000, + 0x927c, 0xffffffff, 0x00030002, + 0x9280, 0xffffffff, 0x00050004, + 0x928c, 0xffffffff, 0x00010006, + 0x9290, 0xffffffff, 0x00090008, + 0x9294, 0xffffffff, 0x00000000, + 0x929c, 0xffffffff, 0x00000001, + 0x802c, 0xffffffff, 0xc0000000, + 0x977c, 0xffffffff, 0x00000100, + 0x3f80, 0xffffffff, 0x00000100, + 0xa210, 0xffffffff, 0x00000100, + 0xa214, 0xffffffff, 0x00000100, + 0x4d8, 0xffffffff, 0x00000100, + 0x9784, 0xffffffff, 0x00000100, + 0x9698, 0xffffffff, 0x00000100, + 0x4d4, 0xffffffff, 0x00000200, + 0x30cc, 0xffffffff, 0x00000100, + 0x802c, 0xffffffff, 0xc0000000 +}; + +static const u32 supersumo_golden_registers[] = +{ + 0x5eb4, 0xffffffff, 0x00000002, + 0x5c4, 0xffffffff, 0x00000001, + 0x7030, 0xffffffff, 0x00000011, + 0x7c30, 0xffffffff, 0x00000011, + 0x6104, 0x01000300, 0x00000000, + 0x5bc0, 0x00300000, 0x00000000, + 0x8c04, 0xffffffff, 0x40600060, + 0x8c08, 0xffffffff, 0x001c001c, + 0x8c20, 0xffffffff, 0x00800080, + 0x8c24, 0xffffffff, 0x00800080, + 0x8c18, 0xffffffff, 0x20202078, + 0x8c1c, 0xffffffff, 0x00001010, + 0x918c, 0xffffffff, 0x00010006, + 0x91a8, 0xffffffff, 0x00010006, + 0x91c4, 0xffffffff, 0x00010006, + 0x91e0, 0xffffffff, 0x00010006, + 0x9200, 0xffffffff, 0x00010006, + 0x9150, 0xffffffff, 0x6e944040, + 0x917c, 0xffffffff, 0x00030002, + 0x9180, 0xffffffff, 0x00050004, + 0x9198, 0xffffffff, 0x00030002, + 0x919c, 0xffffffff, 0x00050004, + 0x91b4, 0xffffffff, 0x00030002, + 0x91b8, 0xffffffff, 0x00050004, + 0x91d0, 0xffffffff, 0x00030002, + 0x91d4, 0xffffffff, 0x00050004, + 0x91f0, 0xffffffff, 0x00030002, + 0x91f4, 0xffffffff, 0x00050004, + 0x915c, 0xffffffff, 0x00010000, + 0x9160, 0xffffffff, 0x00030002, + 0x3f90, 0xffff0000, 0xff000000, + 0x9178, 0xffffffff, 0x00070000, + 0x9194, 0xffffffff, 0x00070000, + 0x91b0, 0xffffffff, 0x00070000, + 0x91cc, 0xffffffff, 0x00070000, + 0x91ec, 0xffffffff, 0x00070000, + 0x9148, 0xffff0000, 0xff000000, + 0x9190, 0xffffffff, 0x00090008, + 0x91ac, 0xffffffff, 0x00090008, + 0x91c8, 0xffffffff, 0x00090008, + 0x91e4, 0xffffffff, 0x00090008, + 0x9204, 0xffffffff, 0x00090008, + 0x3f94, 0xffff0000, 0xff000000, + 0x914c, 0xffff0000, 0xff000000, + 0x929c, 0xffffffff, 0x00000001, + 0x8a18, 0xffffffff, 0x00000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x5644, 0xffffffff, 0x00000100, + 0x9b7c, 0xffffffff, 0x00000000, + 0x8030, 0xffffffff, 0x0000100a, + 0x8a14, 0xffffffff, 0x00000007, + 0x8b24, 0xffffffff, 0x00ff0fff, + 0x8b10, 0xffffffff, 0x00000000, + 0x28a4c, 0x06000000, 0x06000000, + 0x4d8, 0xffffffff, 0x00000100, + 0x913c, 0xffff000f, 0x0100000a, + 0x960c, 0xffffffff, 0x54763210, + 0x88c4, 0xffffffff, 0x000000c2, + 0x88d4, 0xffffffff, 0x00000010, + 0x8974, 0xffffffff, 0x00000000, + 0xc78, 0x00000080, 0x00000080, + 0x5e78, 0xffffffff, 0x001000f0, + 0xd02c, 0xffffffff, 0x08421000, + 0xa008, 0xffffffff, 0x00010000, + 0x8d00, 0xffffffff, 0x100e4848, + 0x8d04, 0xffffffff, 0x00164745, + 0x8c00, 0xffffffff, 0xe4000003, + 0x8cf0, 0x1fffffff, 0x08e00620, + 0x28350, 0xffffffff, 0x00000000, + 0x9508, 0xffffffff, 0x00000002 +}; + +static const u32 sumo_golden_registers[] = +{ + 0x900c, 0x00ffffff, 0x0017071f, + 0x8c18, 0xffffffff, 0x10101060, + 0x8c1c, 0xffffffff, 0x00001010, + 0x8c30, 0x0000000f, 0x00000005, + 0x9688, 0x0000000f, 0x00000007 +}; + +static const u32 wrestler_golden_registers[] = +{ + 0x5eb4, 0xffffffff, 0x00000002, + 0x5c4, 0xffffffff, 0x00000001, + 0x7030, 0xffffffff, 0x00000011, + 0x7c30, 0xffffffff, 0x00000011, + 0x6104, 0x01000300, 0x00000000, + 0x5bc0, 0x00300000, 0x00000000, + 0x918c, 0xffffffff, 0x00010006, + 0x91a8, 0xffffffff, 0x00010006, + 0x9150, 0xffffffff, 0x6e944040, + 0x917c, 0xffffffff, 0x00030002, + 0x9198, 0xffffffff, 0x00030002, + 0x915c, 0xffffffff, 0x00010000, + 0x3f90, 0xffff0000, 0xff000000, + 0x9178, 0xffffffff, 0x00070000, + 0x9194, 0xffffffff, 0x00070000, + 0x9148, 0xffff0000, 0xff000000, + 0x9190, 0xffffffff, 0x00090008, + 0x91ac, 0xffffffff, 0x00090008, + 0x3f94, 0xffff0000, 0xff000000, + 0x914c, 0xffff0000, 0xff000000, + 0x929c, 0xffffffff, 0x00000001, + 0x8a18, 0xffffffff, 0x00000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x9b7c, 0xffffffff, 0x00000000, + 0x8030, 0xffffffff, 0x0000100a, + 0x8a14, 0xffffffff, 0x00000001, + 0x8b24, 0xffffffff, 0x00ff0fff, + 0x8b10, 0xffffffff, 0x00000000, + 0x28a4c, 0x06000000, 0x06000000, + 0x4d8, 0xffffffff, 0x00000100, + 0x913c, 0xffff000f, 0x0100000a, + 0x960c, 0xffffffff, 0x54763210, + 0x88c4, 0xffffffff, 0x000000c2, + 0x88d4, 0xffffffff, 0x00000010, + 0x8974, 0xffffffff, 0x00000000, + 0xc78, 0x00000080, 0x00000080, + 0x5e78, 0xffffffff, 0x001000f0, + 0xd02c, 0xffffffff, 0x08421000, + 0xa008, 0xffffffff, 0x00010000, + 0x8d00, 0xffffffff, 0x100e4848, + 0x8d04, 0xffffffff, 0x00164745, + 0x8c00, 0xffffffff, 0xe4000003, + 0x8cf0, 0x1fffffff, 0x08e00410, + 0x28350, 0xffffffff, 0x00000000, + 0x9508, 0xffffffff, 0x00000002, + 0x900c, 0xffffffff, 0x0017071f, + 0x8c18, 0xffffffff, 0x10101060, + 0x8c1c, 0xffffffff, 0x00001010 +}; + +static const u32 barts_golden_registers[] = +{ + 0x5eb4, 0xffffffff, 0x00000002, + 0x5e78, 0x8f311ff1, 0x001000f0, + 0x3f90, 0xffff0000, 0xff000000, + 0x9148, 0xffff0000, 0xff000000, + 0x3f94, 0xffff0000, 0xff000000, + 0x914c, 0xffff0000, 0xff000000, + 0xc78, 0x00000080, 0x00000080, + 0xbd4, 0x70073777, 0x00010001, + 0xd02c, 0xbfffff1f, 0x08421000, + 0xd0b8, 0x03773777, 0x02011003, + 0x5bc0, 0x00200000, 0x50100000, + 0x98f8, 0x33773777, 0x02011003, + 0x98fc, 0xffffffff, 0x76543210, + 0x7030, 0x31000311, 0x00000011, + 0x2f48, 0x00000007, 0x02011003, + 0x6b28, 0x00000010, 0x00000012, + 0x7728, 0x00000010, 0x00000012, + 0x10328, 0x00000010, 0x00000012, + 0x10f28, 0x00000010, 0x00000012, + 0x11b28, 0x00000010, 0x00000012, + 0x12728, 0x00000010, 0x00000012, + 0x240c, 0x000007ff, 0x00000380, + 0x8a14, 0xf000001f, 0x00000007, + 0x8b24, 0x3fff3fff, 0x00ff0fff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x10c, 0x00000001, 0x00010003, + 0xa02c, 0xffffffff, 0x0000009b, + 0x913c, 0x0000000f, 0x0100000a, + 0x8d00, 0xffff7f7f, 0x100e4848, + 0x8d04, 0x00ffffff, 0x00164745, + 0x8c00, 0xfffc0003, 0xe4000003, + 0x8c04, 0xf8ff00ff, 0x40600060, + 0x8c08, 0x00ff00ff, 0x001c001c, + 0x8cf0, 0x1fff1fff, 0x08e00620, + 0x8c20, 0x0fff0fff, 0x00800080, + 0x8c24, 0x0fff0fff, 0x00800080, + 0x8c18, 0xffffffff, 0x20202078, + 0x8c1c, 0x0000ffff, 0x00001010, + 0x28350, 0x00000f01, 0x00000000, + 0x9508, 0x3700001f, 0x00000002, + 0x960c, 0xffffffff, 0x54763210, + 0x88c4, 0x001f3ae3, 0x000000c2, + 0x88d4, 0x0000001f, 0x00000010, + 0x8974, 0xffffffff, 0x00000000 +}; + +static const u32 turks_golden_registers[] = +{ + 0x5eb4, 0xffffffff, 0x00000002, + 0x5e78, 0x8f311ff1, 0x001000f0, + 0x8c8, 0x00003000, 0x00001070, + 0x8cc, 0x000fffff, 0x00040035, + 0x3f90, 0xffff0000, 0xfff00000, + 0x9148, 0xffff0000, 0xfff00000, + 0x3f94, 0xffff0000, 0xfff00000, + 0x914c, 0xffff0000, 0xfff00000, + 0xc78, 0x00000080, 0x00000080, + 0xbd4, 0x00073007, 0x00010002, + 0xd02c, 0xbfffff1f, 0x08421000, + 0xd0b8, 0x03773777, 0x02010002, + 0x5bc0, 0x00200000, 0x50100000, + 0x98f8, 0x33773777, 0x00010002, + 0x98fc, 0xffffffff, 0x33221100, + 0x7030, 0x31000311, 0x00000011, + 0x2f48, 0x33773777, 0x00010002, + 0x6b28, 0x00000010, 0x00000012, + 0x7728, 0x00000010, 0x00000012, + 0x10328, 0x00000010, 0x00000012, + 0x10f28, 0x00000010, 0x00000012, + 0x11b28, 0x00000010, 0x00000012, + 0x12728, 0x00000010, 0x00000012, + 0x240c, 0x000007ff, 0x00000380, + 0x8a14, 0xf000001f, 0x00000007, + 0x8b24, 0x3fff3fff, 0x00ff0fff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x10c, 0x00000001, 0x00010003, + 0xa02c, 0xffffffff, 0x0000009b, + 0x913c, 0x0000000f, 0x0100000a, + 0x8d00, 0xffff7f7f, 0x100e4848, + 0x8d04, 0x00ffffff, 0x00164745, + 0x8c00, 0xfffc0003, 0xe4000003, + 0x8c04, 0xf8ff00ff, 0x40600060, + 0x8c08, 0x00ff00ff, 0x001c001c, + 0x8cf0, 0x1fff1fff, 0x08e00410, + 0x8c20, 0x0fff0fff, 0x00800080, + 0x8c24, 0x0fff0fff, 0x00800080, + 0x8c18, 0xffffffff, 0x20202078, + 0x8c1c, 0x0000ffff, 0x00001010, + 0x28350, 0x00000f01, 0x00000000, + 0x9508, 0x3700001f, 0x00000002, + 0x960c, 0xffffffff, 0x54763210, + 0x88c4, 0x001f3ae3, 0x000000c2, + 0x88d4, 0x0000001f, 0x00000010, + 0x8974, 0xffffffff, 0x00000000 +}; + +static const u32 caicos_golden_registers[] = +{ + 0x5eb4, 0xffffffff, 0x00000002, + 0x5e78, 0x8f311ff1, 0x001000f0, + 0x8c8, 0x00003420, 0x00001450, + 0x8cc, 0x000fffff, 0x00040035, + 0x3f90, 0xffff0000, 0xfffc0000, + 0x9148, 0xffff0000, 0xfffc0000, + 0x3f94, 0xffff0000, 0xfffc0000, + 0x914c, 0xffff0000, 0xfffc0000, + 0xc78, 0x00000080, 0x00000080, + 0xbd4, 0x00073007, 0x00010001, + 0xd02c, 0xbfffff1f, 0x08421000, + 0xd0b8, 0x03773777, 0x02010001, + 0x5bc0, 0x00200000, 0x50100000, + 0x98f8, 0x33773777, 0x02010001, + 0x98fc, 0xffffffff, 0x33221100, + 0x7030, 0x31000311, 0x00000011, + 0x2f48, 0x33773777, 0x02010001, + 0x6b28, 0x00000010, 0x00000012, + 0x7728, 0x00000010, 0x00000012, + 0x10328, 0x00000010, 0x00000012, + 0x10f28, 0x00000010, 0x00000012, + 0x11b28, 0x00000010, 0x00000012, + 0x12728, 0x00000010, 0x00000012, + 0x240c, 0x000007ff, 0x00000380, + 0x8a14, 0xf000001f, 0x00000001, + 0x8b24, 0x3fff3fff, 0x00ff0fff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x10c, 0x00000001, 0x00010003, + 0xa02c, 0xffffffff, 0x0000009b, + 0x913c, 0x0000000f, 0x0100000a, + 0x8d00, 0xffff7f7f, 0x100e4848, + 0x8d04, 0x00ffffff, 0x00164745, + 0x8c00, 0xfffc0003, 0xe4000003, + 0x8c04, 0xf8ff00ff, 0x40600060, + 0x8c08, 0x00ff00ff, 0x001c001c, + 0x8cf0, 0x1fff1fff, 0x08e00410, + 0x8c20, 0x0fff0fff, 0x00800080, + 0x8c24, 0x0fff0fff, 0x00800080, + 0x8c18, 0xffffffff, 0x20202078, + 0x8c1c, 0x0000ffff, 0x00001010, + 0x28350, 0x00000f01, 0x00000000, + 0x9508, 0x3700001f, 0x00000002, + 0x960c, 0xffffffff, 0x54763210, + 0x88c4, 0x001f3ae3, 0x000000c2, + 0x88d4, 0x0000001f, 0x00000010, + 0x8974, 0xffffffff, 0x00000000 +}; + +static void evergreen_init_golden_registers(struct radeon_device *rdev) +{ + switch (rdev->family) { + case CHIP_CYPRESS: + case CHIP_HEMLOCK: + radeon_program_register_sequence(rdev, + evergreen_golden_registers, + (const u32)ARRAY_SIZE(evergreen_golden_registers)); + radeon_program_register_sequence(rdev, + evergreen_golden_registers2, + (const u32)ARRAY_SIZE(evergreen_golden_registers2)); + radeon_program_register_sequence(rdev, + cypress_mgcg_init, + (const u32)ARRAY_SIZE(cypress_mgcg_init)); + break; + case CHIP_JUNIPER: + radeon_program_register_sequence(rdev, + evergreen_golden_registers, + (const u32)ARRAY_SIZE(evergreen_golden_registers)); + radeon_program_register_sequence(rdev, + evergreen_golden_registers2, + (const u32)ARRAY_SIZE(evergreen_golden_registers2)); + radeon_program_register_sequence(rdev, + juniper_mgcg_init, + (const u32)ARRAY_SIZE(juniper_mgcg_init)); + break; + case CHIP_REDWOOD: + radeon_program_register_sequence(rdev, + evergreen_golden_registers, + (const u32)ARRAY_SIZE(evergreen_golden_registers)); + radeon_program_register_sequence(rdev, + evergreen_golden_registers2, + (const u32)ARRAY_SIZE(evergreen_golden_registers2)); + radeon_program_register_sequence(rdev, + redwood_mgcg_init, + (const u32)ARRAY_SIZE(redwood_mgcg_init)); + break; + case CHIP_CEDAR: + radeon_program_register_sequence(rdev, + cedar_golden_registers, + (const u32)ARRAY_SIZE(cedar_golden_registers)); + radeon_program_register_sequence(rdev, + evergreen_golden_registers2, + (const u32)ARRAY_SIZE(evergreen_golden_registers2)); + radeon_program_register_sequence(rdev, + cedar_mgcg_init, + (const u32)ARRAY_SIZE(cedar_mgcg_init)); + break; + case CHIP_PALM: + radeon_program_register_sequence(rdev, + wrestler_golden_registers, + (const u32)ARRAY_SIZE(wrestler_golden_registers)); + break; + case CHIP_SUMO: + radeon_program_register_sequence(rdev, + supersumo_golden_registers, + (const u32)ARRAY_SIZE(supersumo_golden_registers)); + break; + case CHIP_SUMO2: + radeon_program_register_sequence(rdev, + supersumo_golden_registers, + (const u32)ARRAY_SIZE(supersumo_golden_registers)); + radeon_program_register_sequence(rdev, + sumo_golden_registers, + (const u32)ARRAY_SIZE(sumo_golden_registers)); + break; + case CHIP_BARTS: + radeon_program_register_sequence(rdev, + barts_golden_registers, + (const u32)ARRAY_SIZE(barts_golden_registers)); + break; + case CHIP_TURKS: + radeon_program_register_sequence(rdev, + turks_golden_registers, + (const u32)ARRAY_SIZE(turks_golden_registers)); + break; + case CHIP_CAICOS: + radeon_program_register_sequence(rdev, + caicos_golden_registers, + (const u32)ARRAY_SIZE(caicos_golden_registers)); + break; + default: + break; + } +} + +/** + * evergreen_get_allowed_info_register - fetch the register for the info ioctl + * + * @rdev: radeon_device pointer + * @reg: register offset in bytes + * @val: register value + * + * Returns 0 for success or -EINVAL for an invalid register + * + */ +int evergreen_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val) +{ + switch (reg) { + case GRBM_STATUS: + case GRBM_STATUS_SE0: + case GRBM_STATUS_SE1: + case SRBM_STATUS: + case SRBM_STATUS2: + case DMA_STATUS_REG: + case UVD_STATUS: + *val = RREG32(reg); + return 0; + default: + return -EINVAL; + } +} void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, unsigned *bankh, unsigned *mtaspect, @@ -82,6 +1132,157 @@ void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, } } +static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock, + u32 cntl_reg, u32 status_reg) +{ + int r, i; + struct atom_clock_dividers dividers; + + r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + clock, false, ÷rs); + if (r) + return r; + + WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK)); + + for (i = 0; i < 100; i++) { + if (RREG32(status_reg) & DCLK_STATUS) + break; + mdelay(10); + } + if (i == 100) + return -ETIMEDOUT; + + return 0; +} + +int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) +{ + int r = 0; + u32 cg_scratch = RREG32(CG_SCRATCH1); + + r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); + if (r) + goto done; + cg_scratch &= 0xffff0000; + cg_scratch |= vclk / 100; /* Mhz */ + + r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); + if (r) + goto done; + cg_scratch &= 0x0000ffff; + cg_scratch |= (dclk / 100) << 16; /* Mhz */ + +done: + WREG32(CG_SCRATCH1, cg_scratch); + + return r; +} + +int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) +{ + /* start off with something large */ + unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; + int r; + + /* bypass vclk and dclk with bclk */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), + ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); + + /* put PLL in bypass mode */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); + + if (!vclk || !dclk) { + /* keep the Bypass mode, put PLL to sleep */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); + return 0; + } + + r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, + 16384, 0x03FFFFFF, 0, 128, 5, + &fb_div, &vclk_div, &dclk_div); + if (r) + return r; + + /* set VCO_MODE to 1 */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); + + /* toggle UPLL_SLEEP to 1 then back to 0 */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); + + /* deassert UPLL_RESET */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); + + mdelay(1); + + r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); + if (r) + return r; + + /* assert UPLL_RESET again */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); + + /* disable spread spectrum. */ + WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); + + /* set feedback divider */ + WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); + + /* set ref divider to 0 */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); + + if (fb_div < 307200) + WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); + else + WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); + + /* set PDIV_A and PDIV_B */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), + ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); + + /* give the PLL some time to settle */ + mdelay(15); + + /* deassert PLL_RESET */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); + + mdelay(15); + + /* switch from bypass mode to normal mode */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); + + r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); + if (r) + return r; + + /* switch VCLK and DCLK selection */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), + ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); + + mdelay(100); + + return 0; +} + +#ifdef __linux__ +void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) +{ + int readrq; + u16 v; + + readrq = pcie_get_readrq(rdev->pdev); + v = ffs(readrq) - 8; + /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it + * to avoid hangs or perfomance issues + */ + if ((v == 0) || (v == 6) || (v == 7)) + pcie_set_readrq(rdev->pdev, 512); +} +#else void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) { pcireg_t ctl, v; @@ -104,6 +1305,63 @@ void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) pci_conf_write(rdev->pc, rdev->pa_tag, off + PCI_PCIE_DCSR, ctl); } } +#endif + +void dce4_program_fmt(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + int bpc = 0; + u32 tmp = 0; + enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; + + if (connector) { + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + bpc = radeon_get_monitor_bpc(connector); + dither = radeon_connector->dither; + } + + /* LVDS/eDP FMT is set up by atom */ + if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) + return; + + /* not needed for analog */ + if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || + (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) + return; + + if (bpc == 0) + return; + + switch (bpc) { + case 6: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | + FMT_SPATIAL_DITHER_EN); + else + tmp |= FMT_TRUNCATE_EN; + break; + case 8: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | + FMT_RGB_RANDOM_ENABLE | + FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH); + else + tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH); + break; + case 10: + default: + /* not needed */ + break; + } + + WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); +} static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc) { @@ -162,36 +1420,6 @@ void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc) } } -/** - * radeon_irq_kms_pflip_irq_get - pre-pageflip callback. - * - * @rdev: radeon_device pointer - * @crtc: crtc to prepare for pageflip on - * - * Pre-pageflip callback (evergreen+). - * Enables the pageflip irq (vblank irq). - */ -void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc) -{ - /* enable the pflip int */ - radeon_irq_kms_pflip_irq_get(rdev, crtc); -} - -/** - * evergreen_post_page_flip - pos-pageflip callback. - * - * @rdev: radeon_device pointer - * @crtc: crtc to cleanup pageflip on - * - * Post-pageflip callback (evergreen+). - * Disables the pageflip irq (vblank irq). - */ -void evergreen_post_page_flip(struct radeon_device *rdev, int crtc) -{ - /* disable the pflip int */ - radeon_irq_kms_pflip_irq_put(rdev, crtc); -} - /** * evergreen_page_flip - pageflip callback. * @@ -199,47 +1427,37 @@ void evergreen_post_page_flip(struct radeon_device *rdev, int crtc) * @crtc_id: crtc to cleanup pageflip on * @crtc_base: new address of the crtc (GPU MC address) * - * Does the actual pageflip (evergreen+). - * During vblank we take the crtc lock and wait for the update_pending - * bit to go high, when it does, we release the lock, and allow the - * double buffered update to take place. - * Returns the current update pending status. + * Triggers the actual pageflip by updating the primary + * surface base address (evergreen+). */ -u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) +void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; - u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset); - int i; - - /* Lock the graphics update lock */ - tmp |= EVERGREEN_GRPH_UPDATE_LOCK; - WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); /* update the scanout addresses */ - WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, - upper_32_bits(crtc_base)); - WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, - (u32)crtc_base); - WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, upper_32_bits(crtc_base)); WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, (u32)crtc_base); + /* post the write */ + RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset); +} - /* Wait for update_pending to go high. */ - for (i = 0; i < rdev->usec_timeout; i++) { - if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) - break; - udelay(1); - } - DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n"); - - /* Unlock the lock, so double-buffering can take place inside vblank */ - tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; - WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); +/** + * evergreen_page_flip_pending - check if page flip is still pending + * + * @rdev: radeon_device pointer + * @crtc_id: crtc to check + * + * Returns the current update pending status. + */ +bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id) +{ + struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; /* Return current update_pending status: */ - return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING; + return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & + EVERGREEN_GRPH_SURFACE_UPDATE_PENDING); } /* get temperature in millidegrees */ @@ -423,8 +1641,8 @@ void evergreen_pm_misc(struct radeon_device *rdev) struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; if (voltage->type == VOLTAGE_SW) { - /* 0xff01 is a flag rather then an actual voltage */ - if (voltage->voltage == 0xff01) + /* 0xff0x are flags rather then an actual voltage */ + if ((voltage->voltage & 0xff00) == 0xff00) return; if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); @@ -444,8 +1662,8 @@ void evergreen_pm_misc(struct radeon_device *rdev) voltage = &rdev->pm.power_state[req_ps_idx]. clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage; - /* 0xff01 is a flag rather then an actual voltage */ - if (voltage->vddci == 0xff01) + /* 0xff0x are flags rather then an actual voltage */ + if ((voltage->vddci & 0xff00) == 0xff00) return; if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); @@ -542,7 +1760,7 @@ bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) case RADEON_HPD_6: if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE) connected = true; - break; + break; default: break; } @@ -1059,7 +2277,8 @@ static void evergreen_program_watermarks(struct radeon_device *rdev, u32 lb_size, u32 num_heads) { struct drm_display_mode *mode = &radeon_crtc->base.mode; - struct evergreen_wm_params wm; + struct evergreen_wm_params wm_low, wm_high; + u32 dram_channels; u32 pixel_period; u32 line_time = 0; u32 latency_watermark_a = 0, latency_watermark_b = 0; @@ -1075,39 +2294,81 @@ static void evergreen_program_watermarks(struct radeon_device *rdev, line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); priority_a_cnt = 0; priority_b_cnt = 0; + dram_channels = evergreen_get_number_of_dram_channels(rdev); + + /* watermark for high clocks */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + wm_high.yclk = + radeon_dpm_get_mclk(rdev, false) * 10; + wm_high.sclk = + radeon_dpm_get_sclk(rdev, false) * 10; + } else { + wm_high.yclk = rdev->pm.current_mclk * 10; + wm_high.sclk = rdev->pm.current_sclk * 10; + } + + wm_high.disp_clk = mode->clock; + wm_high.src_width = mode->crtc_hdisplay; + wm_high.active_time = mode->crtc_hdisplay * pixel_period; + wm_high.blank_time = line_time - wm_high.active_time; + wm_high.interlaced = false; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + wm_high.interlaced = true; + wm_high.vsc = radeon_crtc->vsc; + wm_high.vtaps = 1; + if (radeon_crtc->rmx_type != RMX_OFF) + wm_high.vtaps = 2; + wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ + wm_high.lb_size = lb_size; + wm_high.dram_channels = dram_channels; + wm_high.num_heads = num_heads; + + /* watermark for low clocks */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + wm_low.yclk = + radeon_dpm_get_mclk(rdev, true) * 10; + wm_low.sclk = + radeon_dpm_get_sclk(rdev, true) * 10; + } else { + wm_low.yclk = rdev->pm.current_mclk * 10; + wm_low.sclk = rdev->pm.current_sclk * 10; + } - wm.yclk = rdev->pm.current_mclk * 10; - wm.sclk = rdev->pm.current_sclk * 10; - wm.disp_clk = mode->clock; - wm.src_width = mode->crtc_hdisplay; - wm.active_time = mode->crtc_hdisplay * pixel_period; - wm.blank_time = line_time - wm.active_time; - wm.interlaced = false; + wm_low.disp_clk = mode->clock; + wm_low.src_width = mode->crtc_hdisplay; + wm_low.active_time = mode->crtc_hdisplay * pixel_period; + wm_low.blank_time = line_time - wm_low.active_time; + wm_low.interlaced = false; if (mode->flags & DRM_MODE_FLAG_INTERLACE) - wm.interlaced = true; - wm.vsc = radeon_crtc->vsc; - wm.vtaps = 1; + wm_low.interlaced = true; + wm_low.vsc = radeon_crtc->vsc; + wm_low.vtaps = 1; if (radeon_crtc->rmx_type != RMX_OFF) - wm.vtaps = 2; - wm.bytes_per_pixel = 4; /* XXX: get this from fb config */ - wm.lb_size = lb_size; - wm.dram_channels = evergreen_get_number_of_dram_channels(rdev); - wm.num_heads = num_heads; + wm_low.vtaps = 2; + wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ + wm_low.lb_size = lb_size; + wm_low.dram_channels = dram_channels; + wm_low.num_heads = num_heads; /* set for high clocks */ - latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535); + latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535); /* set for low clocks */ - /* wm.yclk = low clk; wm.sclk = low clk */ - latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535); + latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535); /* possibly force display priority to high */ /* should really do this at mode validation time... */ - if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) || - !evergreen_average_bandwidth_vs_available_bandwidth(&wm) || - !evergreen_check_latency_hiding(&wm) || + if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || + !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) || + !evergreen_check_latency_hiding(&wm_high) || (rdev->disp_priority == 2)) { - DRM_DEBUG_KMS("force priority to high\n"); + DRM_DEBUG_KMS("force priority a to high\n"); priority_a_cnt |= PRIORITY_ALWAYS_ON; + } + if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || + !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) || + !evergreen_check_latency_hiding(&wm_low) || + (rdev->disp_priority == 2)) { + DRM_DEBUG_KMS("force priority b to high\n"); priority_b_cnt |= PRIORITY_ALWAYS_ON; } @@ -1134,6 +2395,9 @@ static void evergreen_program_watermarks(struct radeon_device *rdev, c.full = dfixed_div(c, a); priority_b_mark = dfixed_trunc(c); priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; + + /* Save number of lines the linebuffer leads before the scanout */ + radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay); } /* select wm A */ @@ -1160,6 +2424,10 @@ static void evergreen_program_watermarks(struct radeon_device *rdev, WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); + /* save values for DPM */ + radeon_crtc->line_time = line_time; + radeon_crtc->wm_high = latency_watermark_a; + radeon_crtc->wm_low = latency_watermark_b; } /** @@ -1177,6 +2445,9 @@ void evergreen_bandwidth_update(struct radeon_device *rdev) u32 num_heads = 0, lb_size; int i; + if (!rdev->mode_info.mode_config_initialized) + return; + radeon_update_display_priority(rdev); for (i = 0; i < rdev->num_crtc; i++) { @@ -1255,7 +2526,6 @@ static int evergreen_pcie_gart_enable(struct radeon_device *rdev) r = radeon_gart_table_vram_pin(rdev); if (r) return r; - radeon_gart_restore(rdev); /* Setup L2 cache */ WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | @@ -1361,16 +2631,160 @@ static void evergreen_agp_enable(struct radeon_device *rdev) WREG32(VM_CONTEXT1_CNTL, 0); } +static const unsigned ni_dig_offsets[] = +{ + NI_DIG0_REGISTER_OFFSET, + NI_DIG1_REGISTER_OFFSET, + NI_DIG2_REGISTER_OFFSET, + NI_DIG3_REGISTER_OFFSET, + NI_DIG4_REGISTER_OFFSET, + NI_DIG5_REGISTER_OFFSET +}; + +static const unsigned ni_tx_offsets[] = +{ + NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1, + NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1, + NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1, + NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1, + NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1, + NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1 +}; + +static const unsigned evergreen_dp_offsets[] = +{ + EVERGREEN_DP0_REGISTER_OFFSET, + EVERGREEN_DP1_REGISTER_OFFSET, + EVERGREEN_DP2_REGISTER_OFFSET, + EVERGREEN_DP3_REGISTER_OFFSET, + EVERGREEN_DP4_REGISTER_OFFSET, + EVERGREEN_DP5_REGISTER_OFFSET +}; + + +/* + * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc + * We go from crtc to connector and it is not relible since it + * should be an opposite direction .If crtc is enable then + * find the dig_fe which selects this crtc and insure that it enable. + * if such dig_fe is found then find dig_be which selects found dig_be and + * insure that it enable and in DP_SST mode. + * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing + * from dp symbols clocks . + */ +static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev, + unsigned crtc_id, unsigned *ret_dig_fe) +{ + unsigned i; + unsigned dig_fe; + unsigned dig_be; + unsigned dig_en_be; + unsigned uniphy_pll; + unsigned digs_fe_selected; + unsigned dig_be_mode; + unsigned dig_fe_mask; + bool is_enabled = false; + bool found_crtc = false; + + /* loop through all running dig_fe to find selected crtc */ + for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { + dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]); + if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON && + crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) { + /* found running pipe */ + found_crtc = true; + dig_fe_mask = 1 << i; + dig_fe = i; + break; + } + } + + if (found_crtc) { + /* loop through all running dig_be to find selected dig_fe */ + for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { + dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]); + /* if dig_fe_selected by dig_be? */ + digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be); + dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be); + if (dig_fe_mask & digs_fe_selected && + /* if dig_be in sst mode? */ + dig_be_mode == NI_DIG_BE_DPSST) { + dig_en_be = RREG32(NI_DIG_BE_EN_CNTL + + ni_dig_offsets[i]); + uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 + + ni_tx_offsets[i]); + /* dig_be enable and tx is running */ + if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE && + dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON && + uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) { + is_enabled = true; + *ret_dig_fe = dig_fe; + break; + } + } + } + } + + return is_enabled; +} + +/* + * Blank dig when in dp sst mode + * Dig ignores crtc timing + */ +static void evergreen_blank_dp_output(struct radeon_device *rdev, + unsigned dig_fe) +{ + unsigned stream_ctrl; + unsigned fifo_ctrl; + unsigned counter = 0; + + if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) { + DRM_ERROR("invalid dig_fe %d\n", dig_fe); + return; + } + + stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + + evergreen_dp_offsets[dig_fe]); + if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) { + DRM_ERROR("dig %d , should be enable\n", dig_fe); + return; + } + + stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE; + WREG32(EVERGREEN_DP_VID_STREAM_CNTL + + evergreen_dp_offsets[dig_fe], stream_ctrl); + + stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + + evergreen_dp_offsets[dig_fe]); + while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) { + drm_msleep(1); + counter++; + stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + + evergreen_dp_offsets[dig_fe]); + } + if (counter >= 32 ) + DRM_ERROR("counter exceeds %d\n", counter); + + fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]); + fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET; + WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl); + +} + void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save) { u32 crtc_enabled, tmp, frame_count, blackout; int i, j; + unsigned dig_fe; - save->vga_render_control = RREG32(VGA_RENDER_CONTROL); - save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); + if (!ASIC_IS_NODCE(rdev)) { + save->vga_render_control = RREG32(VGA_RENDER_CONTROL); + save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); - /* disable VGA render */ - WREG32(VGA_RENDER_CONTROL, 0); + /* disable VGA render */ + WREG32(VGA_RENDER_CONTROL, 0); + } /* blank the display controllers */ for (i = 0; i < rdev->num_crtc; i++) { crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN; @@ -1383,6 +2797,7 @@ void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *sav WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); + WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); } } else { tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); @@ -1401,7 +2816,17 @@ void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *sav break; udelay(1); } - + /*we should disable dig if it drives dp sst*/ + /*but we are in radeon_device_init and the topology is unknown*/ + /*and it is available after radeon_modeset_init*/ + /*the following method radeon_atom_encoder_dpms_dig*/ + /*does the job if we initialize it properly*/ + /*for now we do it this manually*/ + /**/ + if (ASIC_IS_DCE5(rdev) && + evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe)) + evergreen_blank_dp_output(rdev, dig_fe); + /*we could remove 6 lines below*/ /* XXX this is a hack to avoid strange behavior with EFI on certain systems */ WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); @@ -1461,15 +2886,19 @@ void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *s WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i], (u32)rdev->mc.vram_start); } - WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); - WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); + + if (!ASIC_IS_NODCE(rdev)) { + WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); + WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); + } /* unlock regs and wait for update */ for (i = 0; i < rdev->num_crtc; i++) { if (save->crtc_enabled[i]) { tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]); - if ((tmp & 0x3) != 0) { - tmp &= ~0x3; + if ((tmp & 0x7) != 3) { + tmp &= ~0x7; + tmp |= 0x3; WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp); } tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); @@ -1502,7 +2931,7 @@ void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *s if (save->crtc_enabled[i]) { if (ASIC_IS_DCE6(rdev)) { tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); - tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; + tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN; WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); @@ -1522,10 +2951,12 @@ void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *s } } } - /* Unlock vga access */ - WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); - mdelay(1); - WREG32(VGA_RENDER_CONTROL, save->vga_render_control); + if (!ASIC_IS_NODCE(rdev)) { + /* Unlock vga access */ + WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); + mdelay(1); + WREG32(VGA_RENDER_CONTROL, save->vga_render_control); + } } void evergreen_mc_program(struct radeon_device *rdev) @@ -1658,13 +3089,13 @@ static int evergreen_cp_load_microcode(struct radeon_device *rdev) #endif RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3)); - fw_data = (const __be32 *)rdev->pfp_fw; + fw_data = (const __be32 *)rdev->pfp_fw->data; WREG32(CP_PFP_UCODE_ADDR, 0); for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++) WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); WREG32(CP_PFP_UCODE_ADDR, 0); - fw_data = (const __be32 *)rdev->me_fw; + fw_data = (const __be32 *)rdev->me_fw->data; WREG32(CP_ME_RAM_WADDR, 0); for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++) WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); @@ -1693,7 +3124,7 @@ static int evergreen_cp_start(struct radeon_device *rdev) radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); radeon_ring_write(ring, 0); radeon_ring_write(ring, 0); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); cp_me = 0xff; WREG32(CP_ME_CNTL, cp_me); @@ -1736,7 +3167,7 @@ static int evergreen_cp_start(struct radeon_device *rdev) radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ radeon_ring_write(ring, 0x00000010); /* */ - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); return 0; } @@ -1761,8 +3192,8 @@ static int evergreen_cp_resume(struct radeon_device *rdev) RREG32(GRBM_SOFT_RESET); /* Set ring buffer size */ - rb_bufsz = drm_order(ring->ring_size / 8); - tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; + rb_bufsz = order_base_2(ring->ring_size / 8); + tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; #ifdef __BIG_ENDIAN tmp |= BUF_SWAP_32BIT; #endif @@ -1798,8 +3229,6 @@ static int evergreen_cp_resume(struct radeon_device *rdev) WREG32(CP_RB_BASE, ring->gpu_addr >> 8); WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); - ring->rptr = RREG32(CP_RB_RPTR); - evergreen_cp_start(rdev); ring->ready = true; r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); @@ -1832,7 +3261,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev) u32 vgt_cache_invalidation; u32 hdp_host_path_cntl, tmp; u32 disabled_rb_mask; - int i, j, num_shader_engines, ps_thread_count; + int i, j, ps_thread_count; switch (rdev->family) { case CHIP_CYPRESS: @@ -2075,6 +3504,8 @@ static void evergreen_gpu_init(struct radeon_device *rdev) } WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); + WREG32(SRBM_INT_CNTL, 0x1); + WREG32(SRBM_INT_ACK, 0x1); evergreen_fix_pci_max_read_req_size(rdev); @@ -2130,16 +3561,12 @@ static void evergreen_gpu_init(struct radeon_device *rdev) rdev->config.evergreen.tile_config |= ((gb_addr_config & 0x30000000) >> 28) << 12; - num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1; - if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) { u32 efuse_straps_4; u32 efuse_straps_3; - WREG32(RCU_IND_INDEX, 0x204); - efuse_straps_4 = RREG32(RCU_IND_DATA); - WREG32(RCU_IND_INDEX, 0x203); - efuse_straps_3 = RREG32(RCU_IND_DATA); + efuse_straps_4 = RREG32_RCU(0x204); + efuse_straps_3 = RREG32_RCU(0x203); tmp = (((efuse_straps_4 & 0xf) << 4) | ((efuse_straps_3 & 0xf0000000) >> 28)); } else { @@ -2156,6 +3583,26 @@ static void evergreen_gpu_init(struct radeon_device *rdev) } /* enabled rb are just the one not disabled :) */ disabled_rb_mask = tmp; + tmp = 0; + for (i = 0; i < rdev->config.evergreen.max_backends; i++) + tmp |= (1 << i); + /* if all the backends are disabled, fix it up here */ + if ((disabled_rb_mask & tmp) == tmp) { + for (i = 0; i < rdev->config.evergreen.max_backends; i++) + disabled_rb_mask &= ~(1 << i); + } + + for (i = 0; i < rdev->config.evergreen.num_ses; i++) { + u32 simd_disable_bitmap; + + WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); + WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); + simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; + simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds; + tmp <<= 16; + tmp |= simd_disable_bitmap; + } + rdev->config.evergreen.active_simds = hweight32(~tmp); WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); @@ -2164,6 +3611,9 @@ static void evergreen_gpu_init(struct radeon_device *rdev) WREG32(DMIF_ADDR_CONFIG, gb_addr_config); WREG32(HDP_ADDR_CONFIG, gb_addr_config); WREG32(DMA_TILING_CONFIG, gb_addr_config); + WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); + WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); + WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); if ((rdev->config.evergreen.max_backends == 1) && (rdev->flags & RADEON_IS_IGP)) { @@ -2427,32 +3877,8 @@ int evergreen_mc_init(struct radeon_device *rdev) return 0; } -bool evergreen_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) -{ - u32 srbm_status; - u32 grbm_status; - u32 grbm_status_se0, grbm_status_se1; - - srbm_status = RREG32(SRBM_STATUS); - grbm_status = RREG32(GRBM_STATUS); - grbm_status_se0 = RREG32(GRBM_STATUS_SE0); - grbm_status_se1 = RREG32(GRBM_STATUS_SE1); - if (!(grbm_status & GUI_ACTIVE)) { - radeon_ring_lockup_update(ring); - return false; - } - /* force CP activities */ - radeon_ring_force_activity(rdev, ring); - return radeon_ring_test_lockup(rdev, ring); -} - -static void evergreen_gpu_soft_reset_gfx(struct radeon_device *rdev) +void evergreen_print_gpu_status_regs(struct radeon_device *rdev) { - u32 grbm_reset = 0; - - if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) - return; - dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", RREG32(GRBM_STATUS)); dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", @@ -2461,6 +3887,8 @@ static void evergreen_gpu_soft_reset_gfx(struct radeon_device *rdev) RREG32(GRBM_STATUS_SE1)); dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", RREG32(SRBM_STATUS)); + dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n", + RREG32(SRBM_STATUS2)); dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", RREG32(CP_STALLED_STAT1)); dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", @@ -2469,223 +3897,776 @@ static void evergreen_gpu_soft_reset_gfx(struct radeon_device *rdev) RREG32(CP_BUSY_STAT)); dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", RREG32(CP_STAT)); + dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", + RREG32(DMA_STATUS_REG)); + if (rdev->family >= CHIP_CAYMAN) { + dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n", + RREG32(DMA_STATUS_REG + 0x800)); + } +} - /* Disable CP parsing/prefetching */ - WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); +bool evergreen_is_display_hung(struct radeon_device *rdev) +{ + u32 crtc_hung = 0; + u32 crtc_status[6]; + u32 i, j, tmp; - /* reset all the gfx blocks */ - grbm_reset = (SOFT_RESET_CP | - SOFT_RESET_CB | - SOFT_RESET_DB | - SOFT_RESET_PA | - SOFT_RESET_SC | - SOFT_RESET_SPI | - SOFT_RESET_SH | - SOFT_RESET_SX | - SOFT_RESET_TC | - SOFT_RESET_TA | - SOFT_RESET_VC | - SOFT_RESET_VGT); - - dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset); - WREG32(GRBM_SOFT_RESET, grbm_reset); - (void)RREG32(GRBM_SOFT_RESET); - udelay(50); - WREG32(GRBM_SOFT_RESET, 0); - (void)RREG32(GRBM_SOFT_RESET); + for (i = 0; i < rdev->num_crtc; i++) { + if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) { + crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); + crtc_hung |= (1 << i); + } + } - dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", - RREG32(GRBM_STATUS)); - dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", - RREG32(GRBM_STATUS_SE0)); - dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", - RREG32(GRBM_STATUS_SE1)); - dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", - RREG32(SRBM_STATUS)); - dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", - RREG32(CP_STALLED_STAT1)); - dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", - RREG32(CP_STALLED_STAT2)); - dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", - RREG32(CP_BUSY_STAT)); - dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", - RREG32(CP_STAT)); + for (j = 0; j < 10; j++) { + for (i = 0; i < rdev->num_crtc; i++) { + if (crtc_hung & (1 << i)) { + tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); + if (tmp != crtc_status[i]) + crtc_hung &= ~(1 << i); + } + } + if (crtc_hung == 0) + return false; + udelay(100); + } + + return true; } -static void evergreen_gpu_soft_reset_dma(struct radeon_device *rdev) +u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) { + u32 reset_mask = 0; u32 tmp; - if (RREG32(DMA_STATUS_REG) & DMA_IDLE) - return; + /* GRBM_STATUS */ + tmp = RREG32(GRBM_STATUS); + if (tmp & (PA_BUSY | SC_BUSY | + SH_BUSY | SX_BUSY | + TA_BUSY | VGT_BUSY | + DB_BUSY | CB_BUSY | + SPI_BUSY | VGT_BUSY_NO_DMA)) + reset_mask |= RADEON_RESET_GFX; - dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", - RREG32(DMA_STATUS_REG)); + if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | + CP_BUSY | CP_COHERENCY_BUSY)) + reset_mask |= RADEON_RESET_CP; - /* Disable DMA */ - tmp = RREG32(DMA_RB_CNTL); - tmp &= ~DMA_RB_ENABLE; - WREG32(DMA_RB_CNTL, tmp); + if (tmp & GRBM_EE_BUSY) + reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; - /* Reset dma */ - WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA); - RREG32(SRBM_SOFT_RESET); - udelay(50); - WREG32(SRBM_SOFT_RESET, 0); + /* DMA_STATUS_REG */ + tmp = RREG32(DMA_STATUS_REG); + if (!(tmp & DMA_IDLE)) + reset_mask |= RADEON_RESET_DMA; - dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", - RREG32(DMA_STATUS_REG)); + /* SRBM_STATUS2 */ + tmp = RREG32(SRBM_STATUS2); + if (tmp & DMA_BUSY) + reset_mask |= RADEON_RESET_DMA; + + /* SRBM_STATUS */ + tmp = RREG32(SRBM_STATUS); + if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) + reset_mask |= RADEON_RESET_RLC; + + if (tmp & IH_BUSY) + reset_mask |= RADEON_RESET_IH; + + if (tmp & SEM_BUSY) + reset_mask |= RADEON_RESET_SEM; + + if (tmp & GRBM_RQ_PENDING) + reset_mask |= RADEON_RESET_GRBM; + + if (tmp & VMC_BUSY) + reset_mask |= RADEON_RESET_VMC; + + if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | + MCC_BUSY | MCD_BUSY)) + reset_mask |= RADEON_RESET_MC; + + if (evergreen_is_display_hung(rdev)) + reset_mask |= RADEON_RESET_DISPLAY; + + /* VM_L2_STATUS */ + tmp = RREG32(VM_L2_STATUS); + if (tmp & L2_BUSY) + reset_mask |= RADEON_RESET_VMC; + + /* Skip MC reset as it's mostly likely not hung, just busy */ + if (reset_mask & RADEON_RESET_MC) { + DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); + reset_mask &= ~RADEON_RESET_MC; + } + + return reset_mask; } -static int evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) +static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) { struct evergreen_mc_save save; - - if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) - reset_mask &= ~(RADEON_RESET_GFX | RADEON_RESET_COMPUTE); - - if (RREG32(DMA_STATUS_REG) & DMA_IDLE) - reset_mask &= ~RADEON_RESET_DMA; + u32 grbm_soft_reset = 0, srbm_soft_reset = 0; + u32 tmp; if (reset_mask == 0) - return 0; + return; dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); + evergreen_print_gpu_status_regs(rdev); + + /* Disable CP parsing/prefetching */ + WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); + + if (reset_mask & RADEON_RESET_DMA) { + /* Disable DMA */ + tmp = RREG32(DMA_RB_CNTL); + tmp &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL, tmp); + } + + udelay(50); + evergreen_mc_stop(rdev, &save); if (evergreen_mc_wait_for_idle(rdev)) { dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); } - if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) - evergreen_gpu_soft_reset_gfx(rdev); + if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { + grbm_soft_reset |= SOFT_RESET_DB | + SOFT_RESET_CB | + SOFT_RESET_PA | + SOFT_RESET_SC | + SOFT_RESET_SPI | + SOFT_RESET_SX | + SOFT_RESET_SH | + SOFT_RESET_TC | + SOFT_RESET_TA | + SOFT_RESET_VC | + SOFT_RESET_VGT; + } + + if (reset_mask & RADEON_RESET_CP) { + grbm_soft_reset |= SOFT_RESET_CP | + SOFT_RESET_VGT; + + srbm_soft_reset |= SOFT_RESET_GRBM; + } if (reset_mask & RADEON_RESET_DMA) - evergreen_gpu_soft_reset_dma(rdev); + srbm_soft_reset |= SOFT_RESET_DMA; - /* Wait a little for things to settle down */ - udelay(50); + if (reset_mask & RADEON_RESET_DISPLAY) + srbm_soft_reset |= SOFT_RESET_DC; - evergreen_mc_resume(rdev, &save); - return 0; -} + if (reset_mask & RADEON_RESET_RLC) + srbm_soft_reset |= SOFT_RESET_RLC; -int evergreen_asic_reset(struct radeon_device *rdev) -{ - return evergreen_gpu_soft_reset(rdev, (RADEON_RESET_GFX | - RADEON_RESET_COMPUTE | - RADEON_RESET_DMA)); -} + if (reset_mask & RADEON_RESET_SEM) + srbm_soft_reset |= SOFT_RESET_SEM; -/* Interrupts */ + if (reset_mask & RADEON_RESET_IH) + srbm_soft_reset |= SOFT_RESET_IH; -u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) -{ - if (crtc >= rdev->num_crtc) - return 0; - else - return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]); -} + if (reset_mask & RADEON_RESET_GRBM) + srbm_soft_reset |= SOFT_RESET_GRBM; -void evergreen_disable_interrupt_state(struct radeon_device *rdev) -{ - u32 tmp; + if (reset_mask & RADEON_RESET_VMC) + srbm_soft_reset |= SOFT_RESET_VMC; - if (rdev->family >= CHIP_CAYMAN) { - cayman_cp_int_cntl_setup(rdev, 0, - CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); - cayman_cp_int_cntl_setup(rdev, 1, 0); - cayman_cp_int_cntl_setup(rdev, 2, 0); - tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; - WREG32(CAYMAN_DMA1_CNTL, tmp); - } else - WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); - tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE; - WREG32(DMA_CNTL, tmp); - WREG32(GRBM_INT_CNTL, 0); - WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); - WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); - if (rdev->num_crtc >= 4) { - WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); - WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); - } - if (rdev->num_crtc >= 6) { - WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); - WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); + if (!(rdev->flags & RADEON_IS_IGP)) { + if (reset_mask & RADEON_RESET_MC) + srbm_soft_reset |= SOFT_RESET_MC; } - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); - if (rdev->num_crtc >= 4) { - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); + if (grbm_soft_reset) { + tmp = RREG32(GRBM_SOFT_RESET); + tmp |= grbm_soft_reset; + dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(GRBM_SOFT_RESET, tmp); + tmp = RREG32(GRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~grbm_soft_reset; + WREG32(GRBM_SOFT_RESET, tmp); + tmp = RREG32(GRBM_SOFT_RESET); } - if (rdev->num_crtc >= 6) { - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); + + if (srbm_soft_reset) { + tmp = RREG32(SRBM_SOFT_RESET); + tmp |= srbm_soft_reset; + dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~srbm_soft_reset; + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); } - /* only one DAC on DCE6 */ - if (!ASIC_IS_DCE6(rdev)) - WREG32(DACA_AUTODETECT_INT_CONTROL, 0); - WREG32(DACB_AUTODETECT_INT_CONTROL, 0); + /* Wait a little for things to settle down */ + udelay(50); - tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD1_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD2_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD3_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD4_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD5_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD6_INT_CONTROL, tmp); + evergreen_mc_resume(rdev, &save); + udelay(50); + evergreen_print_gpu_status_regs(rdev); } -int evergreen_irq_set(struct radeon_device *rdev) +void evergreen_gpu_pci_config_reset(struct radeon_device *rdev) { - u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; - u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; - u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; - u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; - u32 grbm_int_cntl = 0; - u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; - u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0; - u32 dma_cntl, dma_cntl1 = 0; + struct evergreen_mc_save save; + u32 tmp, i; - if (!rdev->irq.installed) { - WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); - return -EINVAL; + dev_info(rdev->dev, "GPU pci config reset\n"); + + /* disable dpm? */ + + /* Disable CP parsing/prefetching */ + WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); + udelay(50); + /* Disable DMA */ + tmp = RREG32(DMA_RB_CNTL); + tmp &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL, tmp); + /* XXX other engines? */ + + /* halt the rlc */ + r600_rlc_stop(rdev); + + udelay(50); + + /* set mclk/sclk to bypass */ + rv770_set_clk_bypass_mode(rdev); + /* disable BM */ + pci_clear_master(rdev->pdev); + /* disable mem access */ + evergreen_mc_stop(rdev, &save); + if (evergreen_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); } - /* don't enable anything if the ih is disabled */ - if (!rdev->ih.enabled) { - r600_disable_interrupts(rdev); - /* force the active interrupt state to all disabled */ - evergreen_disable_interrupt_state(rdev); - return 0; + /* reset */ + radeon_pci_config_reset(rdev); + /* wait for asic to come out of reset */ + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) + break; + udelay(1); } +} - hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; +int evergreen_asic_reset(struct radeon_device *rdev) +{ + u32 reset_mask; - afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; - afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; - afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; - afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; - afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; - afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; + reset_mask = evergreen_gpu_check_soft_reset(rdev); - dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; + if (reset_mask) + r600_set_bios_scratch_engine_hung(rdev, true); - if (rdev->family >= CHIP_CAYMAN) { - /* enable CP interrupts on all rings */ + /* try soft reset */ + evergreen_gpu_soft_reset(rdev, reset_mask); + + reset_mask = evergreen_gpu_check_soft_reset(rdev); + + /* try pci config reset */ + if (reset_mask && radeon_hard_reset) + evergreen_gpu_pci_config_reset(rdev); + + reset_mask = evergreen_gpu_check_soft_reset(rdev); + + if (!reset_mask) + r600_set_bios_scratch_engine_hung(rdev, false); + + return 0; +} + +/** + * evergreen_gfx_is_lockup - Check if the GFX engine is locked up + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Check if the GFX engine is locked up. + * Returns true if the engine appears to be locked up, false if not. + */ +bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +{ + u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); + + if (!(reset_mask & (RADEON_RESET_GFX | + RADEON_RESET_COMPUTE | + RADEON_RESET_CP))) { + radeon_ring_lockup_update(rdev, ring); + return false; + } + return radeon_ring_test_lockup(rdev, ring); +} + +/* + * RLC + */ +#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000 +#define RLC_CLEAR_STATE_END_MARKER 0x00000001 + +void sumo_rlc_fini(struct radeon_device *rdev) +{ + int r; + + /* save restore block */ + if (rdev->rlc.save_restore_obj) { + r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); + if (unlikely(r != 0)) + dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); + radeon_bo_unpin(rdev->rlc.save_restore_obj); + radeon_bo_unreserve(rdev->rlc.save_restore_obj); + + radeon_bo_unref(&rdev->rlc.save_restore_obj); + rdev->rlc.save_restore_obj = NULL; + } + + /* clear state block */ + if (rdev->rlc.clear_state_obj) { + r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); + if (unlikely(r != 0)) + dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); + radeon_bo_unpin(rdev->rlc.clear_state_obj); + radeon_bo_unreserve(rdev->rlc.clear_state_obj); + + radeon_bo_unref(&rdev->rlc.clear_state_obj); + rdev->rlc.clear_state_obj = NULL; + } + + /* clear state block */ + if (rdev->rlc.cp_table_obj) { + r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); + if (unlikely(r != 0)) + dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); + radeon_bo_unpin(rdev->rlc.cp_table_obj); + radeon_bo_unreserve(rdev->rlc.cp_table_obj); + + radeon_bo_unref(&rdev->rlc.cp_table_obj); + rdev->rlc.cp_table_obj = NULL; + } +} + +#define CP_ME_TABLE_SIZE 96 + +int sumo_rlc_init(struct radeon_device *rdev) +{ + const u32 *src_ptr; + volatile u32 *dst_ptr; + u32 dws, data, i, j, k, reg_num; + u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0; + u64 reg_list_mc_addr; + const struct cs_section_def *cs_data; + int r; + + src_ptr = rdev->rlc.reg_list; + dws = rdev->rlc.reg_list_size; + if (rdev->family >= CHIP_BONAIRE) { + dws += (5 * 16) + 48 + 48 + 64; + } + cs_data = rdev->rlc.cs_data; + + if (src_ptr) { + /* save restore block */ + if (rdev->rlc.save_restore_obj == NULL) { + r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, + RADEON_GEM_DOMAIN_VRAM, 0, NULL, + NULL, &rdev->rlc.save_restore_obj); + if (r) { + dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); + return r; + } + } + + r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); + if (unlikely(r != 0)) { + sumo_rlc_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, + &rdev->rlc.save_restore_gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->rlc.save_restore_obj); + dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + + r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr); + if (r) { + dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + /* write the sr buffer */ + dst_ptr = rdev->rlc.sr_ptr; + if (rdev->family >= CHIP_TAHITI) { + /* SI */ + for (i = 0; i < rdev->rlc.reg_list_size; i++) + dst_ptr[i] = cpu_to_le32(src_ptr[i]); + } else { + /* ON/LN/TN */ + /* format: + * dw0: (reg2 << 16) | reg1 + * dw1: reg1 save space + * dw2: reg2 save space + */ + for (i = 0; i < dws; i++) { + data = src_ptr[i] >> 2; + i++; + if (i < dws) + data |= (src_ptr[i] >> 2) << 16; + j = (((i - 1) * 3) / 2); + dst_ptr[j] = cpu_to_le32(data); + } + j = ((i * 3) / 2); + dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER); + } + radeon_bo_kunmap(rdev->rlc.save_restore_obj); + radeon_bo_unreserve(rdev->rlc.save_restore_obj); + } + + if (cs_data) { + /* clear state block */ + if (rdev->family >= CHIP_BONAIRE) { + rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev); + } else if (rdev->family >= CHIP_TAHITI) { + rdev->rlc.clear_state_size = si_get_csb_size(rdev); + dws = rdev->rlc.clear_state_size + (256 / 4); + } else { + reg_list_num = 0; + dws = 0; + for (i = 0; cs_data[i].section != NULL; i++) { + for (j = 0; cs_data[i].section[j].extent != NULL; j++) { + reg_list_num++; + dws += cs_data[i].section[j].reg_count; + } + } + reg_list_blk_index = (3 * reg_list_num + 2); + dws += reg_list_blk_index; + rdev->rlc.clear_state_size = dws; + } + + if (rdev->rlc.clear_state_obj == NULL) { + r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, + RADEON_GEM_DOMAIN_VRAM, 0, NULL, + NULL, &rdev->rlc.clear_state_obj); + if (r) { + dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + } + r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); + if (unlikely(r != 0)) { + sumo_rlc_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, + &rdev->rlc.clear_state_gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->rlc.clear_state_obj); + dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + + r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr); + if (r) { + dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + /* set up the cs buffer */ + dst_ptr = rdev->rlc.cs_ptr; + if (rdev->family >= CHIP_BONAIRE) { + cik_get_csb_buffer(rdev, dst_ptr); + } else if (rdev->family >= CHIP_TAHITI) { + reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256; + dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr)); + dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr)); + dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size); + si_get_csb_buffer(rdev, &dst_ptr[(256/4)]); + } else { + reg_list_hdr_blk_index = 0; + reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4); + data = upper_32_bits(reg_list_mc_addr); + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); + reg_list_hdr_blk_index++; + for (i = 0; cs_data[i].section != NULL; i++) { + for (j = 0; cs_data[i].section[j].extent != NULL; j++) { + reg_num = cs_data[i].section[j].reg_count; + data = reg_list_mc_addr & 0xffffffff; + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); + reg_list_hdr_blk_index++; + + data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff; + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); + reg_list_hdr_blk_index++; + + data = 0x08000000 | (reg_num * 4); + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); + reg_list_hdr_blk_index++; + + for (k = 0; k < reg_num; k++) { + data = cs_data[i].section[j].extent[k]; + dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data); + } + reg_list_mc_addr += reg_num * 4; + reg_list_blk_index += reg_num; + } + } + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER); + } + radeon_bo_kunmap(rdev->rlc.clear_state_obj); + radeon_bo_unreserve(rdev->rlc.clear_state_obj); + } + + if (rdev->rlc.cp_table_size) { + if (rdev->rlc.cp_table_obj == NULL) { + r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, + PAGE_SIZE, true, + RADEON_GEM_DOMAIN_VRAM, 0, NULL, + NULL, &rdev->rlc.cp_table_obj); + if (r) { + dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + } + + r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); + if (unlikely(r != 0)) { + dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM, + &rdev->rlc.cp_table_gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->rlc.cp_table_obj); + dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr); + if (r) { + dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + + cik_init_cp_pg_table(rdev); + + radeon_bo_kunmap(rdev->rlc.cp_table_obj); + radeon_bo_unreserve(rdev->rlc.cp_table_obj); + + } + + return 0; +} + +static void evergreen_rlc_start(struct radeon_device *rdev) +{ + u32 mask = RLC_ENABLE; + + if (rdev->flags & RADEON_IS_IGP) { + mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC; + } + + WREG32(RLC_CNTL, mask); +} + +int evergreen_rlc_resume(struct radeon_device *rdev) +{ + u32 i; + const __be32 *fw_data; + + if (!rdev->rlc_fw) + return -EINVAL; + + r600_rlc_stop(rdev); + + WREG32(RLC_HB_CNTL, 0); + + if (rdev->flags & RADEON_IS_IGP) { + if (rdev->family == CHIP_ARUBA) { + u32 always_on_bitmap = + 3 | (3 << (16 * rdev->config.cayman.max_shader_engines)); + /* find out the number of active simds */ + u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; + tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se; + tmp = hweight32(~tmp); + if (tmp == rdev->config.cayman.max_simds_per_se) { + WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap); + WREG32(TN_RLC_LB_PARAMS, 0x00601004); + WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff); + WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000); + WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000); + } + } else { + WREG32(RLC_HB_WPTR_LSB_ADDR, 0); + WREG32(RLC_HB_WPTR_MSB_ADDR, 0); + } + WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); + WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); + } else { + WREG32(RLC_HB_BASE, 0); + WREG32(RLC_HB_RPTR, 0); + WREG32(RLC_HB_WPTR, 0); + WREG32(RLC_HB_WPTR_LSB_ADDR, 0); + WREG32(RLC_HB_WPTR_MSB_ADDR, 0); + } + WREG32(RLC_MC_CNTL, 0); + WREG32(RLC_UCODE_CNTL, 0); + + fw_data = (const __be32 *)rdev->rlc_fw->data; + if (rdev->family >= CHIP_ARUBA) { + for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) { + WREG32(RLC_UCODE_ADDR, i); + WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); + } + } else if (rdev->family >= CHIP_CAYMAN) { + for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) { + WREG32(RLC_UCODE_ADDR, i); + WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); + } + } else { + for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) { + WREG32(RLC_UCODE_ADDR, i); + WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); + } + } + WREG32(RLC_UCODE_ADDR, 0); + + evergreen_rlc_start(rdev); + + return 0; +} + +/* Interrupts */ + +u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) +{ + if (crtc >= rdev->num_crtc) + return 0; + else + return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]); +} + +void evergreen_disable_interrupt_state(struct radeon_device *rdev) +{ + u32 tmp; + + if (rdev->family >= CHIP_CAYMAN) { + cayman_cp_int_cntl_setup(rdev, 0, + CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + cayman_cp_int_cntl_setup(rdev, 1, 0); + cayman_cp_int_cntl_setup(rdev, 2, 0); + tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; + WREG32(CAYMAN_DMA1_CNTL, tmp); + } else + WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE; + WREG32(DMA_CNTL, tmp); + WREG32(GRBM_INT_CNTL, 0); + WREG32(SRBM_INT_CNTL, 0); + WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); + WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); + if (rdev->num_crtc >= 4) { + WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); + WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); + } + if (rdev->num_crtc >= 6) { + WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); + WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); + } + + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); + if (rdev->num_crtc >= 4) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); + } + if (rdev->num_crtc >= 6) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); + } + + /* only one DAC on DCE5 */ + if (!ASIC_IS_DCE5(rdev)) + WREG32(DACA_AUTODETECT_INT_CONTROL, 0); + WREG32(DACB_AUTODETECT_INT_CONTROL, 0); + + tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD1_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD2_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD3_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD4_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD5_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD6_INT_CONTROL, tmp); + +} + +int evergreen_irq_set(struct radeon_device *rdev) +{ + u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; + u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; + u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; + u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; + u32 grbm_int_cntl = 0; + u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0; + u32 dma_cntl, dma_cntl1 = 0; + u32 thermal_int = 0; + + if (!rdev->irq.installed) { + WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); + return -EINVAL; + } + /* don't enable anything if the ih is disabled */ + if (!rdev->ih.enabled) { + r600_disable_interrupts(rdev); + /* force the active interrupt state to all disabled */ + evergreen_disable_interrupt_state(rdev); + return 0; + } + + hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + if (rdev->family == CHIP_ARUBA) + thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) & + ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + else + thermal_int = RREG32(CG_THERMAL_INT) & + ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + + afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; + afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; + afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; + afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; + afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; + afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; + + dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; + + if (rdev->family >= CHIP_CAYMAN) { + /* enable CP interrupts on all rings */ if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); cp_int_cntl |= TIME_STAMP_INT_ENABLE; @@ -2719,6 +4700,11 @@ int evergreen_irq_set(struct radeon_device *rdev) } } + if (rdev->irq.dpm_thermal) { + DRM_DEBUG("dpm thermal\n"); + thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; + } + if (rdev->irq.crtc_vblank_int[0] || atomic_read(&rdev->irq.pflip[0])) { DRM_DEBUG("evergreen_irq_set: vblank 0\n"); @@ -2751,27 +4737,27 @@ int evergreen_irq_set(struct radeon_device *rdev) } if (rdev->irq.hpd[0]) { DRM_DEBUG("evergreen_irq_set: hpd 1\n"); - hpd1 |= DC_HPDx_INT_EN; + hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[1]) { DRM_DEBUG("evergreen_irq_set: hpd 2\n"); - hpd2 |= DC_HPDx_INT_EN; + hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[2]) { DRM_DEBUG("evergreen_irq_set: hpd 3\n"); - hpd3 |= DC_HPDx_INT_EN; + hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[3]) { DRM_DEBUG("evergreen_irq_set: hpd 4\n"); - hpd4 |= DC_HPDx_INT_EN; + hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[4]) { DRM_DEBUG("evergreen_irq_set: hpd 5\n"); - hpd5 |= DC_HPDx_INT_EN; + hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[5]) { DRM_DEBUG("evergreen_irq_set: hpd 6\n"); - hpd6 |= DC_HPDx_INT_EN; + hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.afmt[0]) { DRM_DEBUG("evergreen_irq_set: hdmi 0\n"); @@ -2823,15 +4809,21 @@ int evergreen_irq_set(struct radeon_device *rdev) WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); } - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); if (rdev->num_crtc >= 4) { - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); } if (rdev->num_crtc >= 6) { - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); } WREG32(DC_HPD1_INT_CONTROL, hpd1); @@ -2840,6 +4832,10 @@ int evergreen_irq_set(struct radeon_device *rdev) WREG32(DC_HPD4_INT_CONTROL, hpd4); WREG32(DC_HPD5_INT_CONTROL, hpd5); WREG32(DC_HPD6_INT_CONTROL, hpd6); + if (rdev->family == CHIP_ARUBA) + WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int); + else + WREG32(CG_THERMAL_INT, thermal_int); WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1); WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2); @@ -2848,6 +4844,9 @@ int evergreen_irq_set(struct radeon_device *rdev) WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5); WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6); + /* posting read */ + RREG32(SRBM_STATUS); + return 0; } @@ -2948,10 +4947,42 @@ static void evergreen_irq_ack(struct radeon_device *rdev) WREG32(DC_HPD5_INT_CONTROL, tmp); } if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { - tmp = RREG32(DC_HPD5_INT_CONTROL); + tmp = RREG32(DC_HPD6_INT_CONTROL); tmp |= DC_HPDx_INT_ACK; WREG32(DC_HPD6_INT_CONTROL, tmp); } + + if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) { + tmp = RREG32(DC_HPD1_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD1_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) { + tmp = RREG32(DC_HPD2_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD2_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) { + tmp = RREG32(DC_HPD3_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD3_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) { + tmp = RREG32(DC_HPD4_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD4_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) { + tmp = RREG32(DC_HPD5_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD5_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) { + tmp = RREG32(DC_HPD6_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD6_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET); tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; @@ -3009,12 +5040,13 @@ static u32 evergreen_get_ih_wptr(struct radeon_device *rdev) wptr = RREG32(IH_RB_WPTR); if (wptr & RB_OVERFLOW) { + wptr &= ~RB_OVERFLOW; /* When a ring buffer overflow happen start parsing interrupt * from the last not overwritten vector (wptr + 16). Hopefully * this should allow us to catchup. */ - dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", - wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); + dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; tmp = RREG32(IH_RB_CNTL); tmp |= IH_WPTR_OVERFLOW_CLEAR; @@ -3031,21 +5063,22 @@ int evergreen_irq_process(struct radeon_device *rdev) u32 ring_index; bool queue_hotplug = false; bool queue_hdmi = false; + bool queue_dp = false; + bool queue_thermal = false; + u32 status, addr; if (!rdev->ih.enabled || rdev->shutdown) return IRQ_NONE; wptr = evergreen_get_ih_wptr(rdev); - if (wptr == rdev->ih.rptr) - return IRQ_NONE; restart_ih: /* is somebody else already processing irqs? */ if (atomic_xchg(&rdev->ih.lock, 1)) return IRQ_NONE; rptr = rdev->ih.rptr; - DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr); + DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr); /* Order reading of wptr vs. reading of IH ring data */ rmb(); @@ -3063,23 +5096,27 @@ restart_ih: case 1: /* D1 vblank/vline */ switch (src_data) { case 0: /* D1 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[0]) { - drm_handle_vblank(rdev->ddev, 0); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[0])) - radeon_crtc_handle_flip(rdev, 0); - rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D1 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[0]) { + drm_handle_vblank(rdev->ddev, 0); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[0])) + radeon_crtc_handle_vblank(rdev, 0); + rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D1 vblank\n"); + break; case 1: /* D1 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; - DRM_DEBUG("IH: D1 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)) + DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; + DRM_DEBUG("IH: D1 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3089,23 +5126,27 @@ restart_ih: case 2: /* D2 vblank/vline */ switch (src_data) { case 0: /* D2 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[1]) { - drm_handle_vblank(rdev->ddev, 1); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[1])) - radeon_crtc_handle_flip(rdev, 1); - rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D2 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[1]) { + drm_handle_vblank(rdev->ddev, 1); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[1])) + radeon_crtc_handle_vblank(rdev, 1); + rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D2 vblank\n"); + break; case 1: /* D2 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; - DRM_DEBUG("IH: D2 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)) + DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; + DRM_DEBUG("IH: D2 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3115,23 +5156,27 @@ restart_ih: case 3: /* D3 vblank/vline */ switch (src_data) { case 0: /* D3 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[2]) { - drm_handle_vblank(rdev->ddev, 2); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[2])) - radeon_crtc_handle_flip(rdev, 2); - rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D3 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[2]) { + drm_handle_vblank(rdev->ddev, 2); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[2])) + radeon_crtc_handle_vblank(rdev, 2); + rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D3 vblank\n"); + break; case 1: /* D3 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; - DRM_DEBUG("IH: D3 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)) + DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; + DRM_DEBUG("IH: D3 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3141,23 +5186,27 @@ restart_ih: case 4: /* D4 vblank/vline */ switch (src_data) { case 0: /* D4 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[3]) { - drm_handle_vblank(rdev->ddev, 3); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[3])) - radeon_crtc_handle_flip(rdev, 3); - rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D4 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[3]) { + drm_handle_vblank(rdev->ddev, 3); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[3])) + radeon_crtc_handle_vblank(rdev, 3); + rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D4 vblank\n"); + break; case 1: /* D4 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; - DRM_DEBUG("IH: D4 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)) + DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; + DRM_DEBUG("IH: D4 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3167,23 +5216,27 @@ restart_ih: case 5: /* D5 vblank/vline */ switch (src_data) { case 0: /* D5 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[4]) { - drm_handle_vblank(rdev->ddev, 4); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[4])) - radeon_crtc_handle_flip(rdev, 4); - rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D5 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[4]) { + drm_handle_vblank(rdev->ddev, 4); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[4])) + radeon_crtc_handle_vblank(rdev, 4); + rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D5 vblank\n"); + break; case 1: /* D5 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; - DRM_DEBUG("IH: D5 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)) + DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; + DRM_DEBUG("IH: D5 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3193,72 +5246,140 @@ restart_ih: case 6: /* D6 vblank/vline */ switch (src_data) { case 0: /* D6 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[5]) { - drm_handle_vblank(rdev->ddev, 5); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[5])) - radeon_crtc_handle_flip(rdev, 5); - rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D6 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[5]) { + drm_handle_vblank(rdev->ddev, 5); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[5])) + radeon_crtc_handle_vblank(rdev, 5); + rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D6 vblank\n"); + break; case 1: /* D6 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; - DRM_DEBUG("IH: D6 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)) + DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; + DRM_DEBUG("IH: D6 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); break; } break; + case 8: /* D1 page flip */ + case 10: /* D2 page flip */ + case 12: /* D3 page flip */ + case 14: /* D4 page flip */ + case 16: /* D5 page flip */ + case 18: /* D6 page flip */ + DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); + if (radeon_use_pflipirq > 0) + radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); + break; case 42: /* HPD hotplug */ switch (src_data) { case 0: - if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD1\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD1\n"); break; case 1: - if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD2\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD2\n"); break; case 2: - if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD3\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD3\n"); break; case 3: - if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD4\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD4\n"); break; case 4: - if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD5\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD5\n"); break; case 5: - if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD6\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD6\n"); + break; + case 6: + if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 1\n"); + break; + case 7: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 2\n"); + break; + case 8: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 3\n"); + break; + case 9: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 4\n"); + break; + case 10: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 5\n"); + break; + case 11: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 6\n"); break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3268,61 +5389,79 @@ restart_ih: case 44: /* hdmi */ switch (src_data) { case 0: - if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { - rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG; - queue_hdmi = true; - DRM_DEBUG("IH: HDMI0\n"); - } + if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG; + queue_hdmi = true; + DRM_DEBUG("IH: HDMI0\n"); break; case 1: - if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) { - rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG; - queue_hdmi = true; - DRM_DEBUG("IH: HDMI1\n"); - } + if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG; + queue_hdmi = true; + DRM_DEBUG("IH: HDMI1\n"); break; case 2: - if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) { - rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG; - queue_hdmi = true; - DRM_DEBUG("IH: HDMI2\n"); - } + if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG; + queue_hdmi = true; + DRM_DEBUG("IH: HDMI2\n"); break; case 3: - if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) { - rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG; - queue_hdmi = true; - DRM_DEBUG("IH: HDMI3\n"); - } + if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG; + queue_hdmi = true; + DRM_DEBUG("IH: HDMI3\n"); break; case 4: - if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) { - rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG; - queue_hdmi = true; - DRM_DEBUG("IH: HDMI4\n"); - } + if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG; + queue_hdmi = true; + DRM_DEBUG("IH: HDMI4\n"); break; case 5: - if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) { - rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG; - queue_hdmi = true; - DRM_DEBUG("IH: HDMI5\n"); - } + if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG; + queue_hdmi = true; + DRM_DEBUG("IH: HDMI5\n"); break; default: DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); break; } + case 96: + DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); + WREG32(SRBM_INT_ACK, 0x1); + break; + case 124: /* UVD */ + DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); + radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); break; case 146: case 147: + addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); + status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); + /* reset addr and status */ + WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); + if (addr == 0x0 && status == 0x0) + break; dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", - RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); + addr); dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", - RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); - /* reset addr and status */ - WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); + status); + cayman_vm_decode_fault(rdev, status, addr); break; case 176: /* CP_INT in ring buffer */ case 177: /* CP_INT in IB1 */ @@ -3351,6 +5490,16 @@ restart_ih: DRM_DEBUG("IH: DMA trap\n"); radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); break; + case 230: /* thermal low to high */ + DRM_DEBUG("IH: thermal low to high\n"); + rdev->pm.dpm.thermal.high_to_low = false; + queue_thermal = true; + break; + case 231: /* thermal high to low */ + DRM_DEBUG("IH: thermal high to low\n"); + rdev->pm.dpm.thermal.high_to_low = true; + queue_thermal = true; + break; case 233: /* GUI IDLE */ DRM_DEBUG("IH: GUI idle\n"); break; @@ -3368,13 +5517,17 @@ restart_ih: /* wptr/rptr are in bytes! */ rptr += 16; rptr &= rdev->ih.ptr_mask; + WREG32(IH_RB_RPTR, rptr); } + if (queue_dp) + schedule_work(&rdev->dp_work); if (queue_hotplug) - task_add(systq, &rdev->hotplug_task); + schedule_delayed_work(&rdev->hotplug_work, 0); if (queue_hdmi) - task_add(systq, &rdev->audio_task); + schedule_work(&rdev->audio_work); + if (queue_thermal && rdev->pm.dpm_enabled) + schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; - WREG32(IH_RB_RPTR, rdev->ih.rptr); atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ @@ -3385,180 +5538,31 @@ restart_ih: return IRQ_HANDLED; } -/** - * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring - * - * @rdev: radeon_device pointer - * @fence: radeon fence object - * - * Add a DMA fence packet to the ring to write - * the fence seq number and DMA trap packet to generate - * an interrupt if needed (evergreen-SI). - */ -void evergreen_dma_fence_ring_emit(struct radeon_device *rdev, - struct radeon_fence *fence) -{ - struct radeon_ring *ring = &rdev->ring[fence->ring]; - u64 addr = rdev->fence_drv[fence->ring].gpu_addr; - /* write the fence */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0)); - radeon_ring_write(ring, addr & 0xfffffffc); - radeon_ring_write(ring, (upper_32_bits(addr) & 0xff)); - radeon_ring_write(ring, fence->seq); - /* generate an interrupt */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0)); - /* flush HDP */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); - radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); - radeon_ring_write(ring, 1); -} - -/** - * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine - * - * @rdev: radeon_device pointer - * @ib: IB object to schedule - * - * Schedule an IB in the DMA ring (evergreen). - */ -void evergreen_dma_ring_ib_execute(struct radeon_device *rdev, - struct radeon_ib *ib) -{ - struct radeon_ring *ring = &rdev->ring[ib->ring]; - - if (rdev->wb.enabled) { - u32 next_rptr = ring->wptr + 4; - while ((next_rptr & 7) != 5) - next_rptr++; - next_rptr += 3; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1)); - radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff); - radeon_ring_write(ring, next_rptr); - } - - /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. - * Pad as necessary with NOPs. - */ - while ((ring->wptr & 7) != 5) - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0)); - radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); - radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); - -} - -/** - * evergreen_copy_dma - copy pages using the DMA engine - * - * @rdev: radeon_device pointer - * @src_offset: src GPU address - * @dst_offset: dst GPU address - * @num_gpu_pages: number of GPU pages to xfer - * @fence: radeon fence object - * - * Copy GPU paging using the DMA engine (evergreen-cayman). - * Used by the radeon ttm implementation to move pages if - * registered as the asic copy callback. - */ -int evergreen_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence) -{ - struct radeon_semaphore *sem = NULL; - int ring_index = rdev->asic->copy.dma_ring_index; - struct radeon_ring *ring = &rdev->ring[ring_index]; - u32 size_in_dw, cur_size_in_dw; - int i, num_loops; - int r = 0; - - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return r; - } - - size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; - num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff); - r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); - return r; - } - - if (radeon_fence_need_sync(*fence, ring->idx)) { - radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring, - ring->idx); - radeon_fence_note_sync(*fence, ring->idx); - } else { - radeon_semaphore_free(rdev, &sem, NULL); - } - - for (i = 0; i < num_loops; i++) { - cur_size_in_dw = size_in_dw; - if (cur_size_in_dw > 0xFFFFF) - cur_size_in_dw = 0xFFFFF; - size_in_dw -= cur_size_in_dw; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw)); - radeon_ring_write(ring, dst_offset & 0xfffffffc); - radeon_ring_write(ring, src_offset & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); - radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); - src_offset += cur_size_in_dw * 4; - dst_offset += cur_size_in_dw * 4; - } - - r = radeon_fence_emit(rdev, fence, ring->idx); - if (r) { - radeon_ring_unlock_undo(rdev, ring); - return r; - } - - radeon_ring_unlock_commit(rdev, ring); - radeon_semaphore_free(rdev, &sem, *fence); - - return r; -} - static int evergreen_startup(struct radeon_device *rdev) { - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; + struct radeon_ring *ring; int r; /* enable pcie gen2 link */ evergreen_pcie_gen2_enable(rdev); + /* enable aspm */ + evergreen_program_aspm(rdev); + + /* scratch needs to be initialized before MC */ + r = r600_vram_scratch_init(rdev); + if (r) + return r; evergreen_mc_program(rdev); - if (ASIC_IS_DCE5(rdev)) { - if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { - r = ni_init_microcode(rdev); - if (r) { - DRM_ERROR("Failed to load firmware!\n"); - return r; - } - } + if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) { r = ni_mc_load_microcode(rdev); if (r) { DRM_ERROR("Failed to load MC firmware!\n"); return r; } - } else { - if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { - r = r600_init_microcode(rdev); - if (r) { - DRM_ERROR("Failed to load firmware!\n"); - return r; - } - } } - r = r600_vram_scratch_init(rdev); - if (r) - return r; - if (rdev->flags & RADEON_IS_AGP) { evergreen_agp_enable(rdev); } else { @@ -3568,11 +5572,17 @@ static int evergreen_startup(struct radeon_device *rdev) } evergreen_gpu_init(rdev); - r = evergreen_blit_init(rdev); - if (r) { - r600_blit_fini(rdev); - rdev->asic->copy.copy = NULL; - dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); + /* allocate rlc buffers */ + if (rdev->flags & RADEON_IS_IGP) { + rdev->rlc.reg_list = sumo_rlc_save_restore_register_list; + rdev->rlc.reg_list_size = + (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list); + rdev->rlc.cs_data = evergreen_cs_data; + r = sumo_rlc_init(rdev); + if (r) { + DRM_ERROR("Failed to init rlc BOs!\n"); + return r; + } } /* allocate wb buffer */ @@ -3592,6 +5602,17 @@ static int evergreen_startup(struct radeon_device *rdev) return r; } + r = uvd_v2_2_resume(rdev); + if (!r) { + r = radeon_fence_driver_start_ring(rdev, + R600_RING_TYPE_UVD_INDEX); + if (r) + dev_err(rdev->dev, "UVD fences init error (%d).\n", r); + } + + if (r) + rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; + /* Enable IRQ */ if (!rdev->irq.installed) { r = radeon_irq_kms_init(rdev); @@ -3607,16 +5628,15 @@ static int evergreen_startup(struct radeon_device *rdev) } evergreen_irq_set(rdev); + ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, - R600_CP_RB_RPTR, R600_CP_RB_WPTR, - 0, 0xfffff, RADEON_CP_PACKET2); + RADEON_CP_PACKET2); if (r) return r; ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, - DMA_RB_RPTR, DMA_RB_WPTR, - 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); + DMA_PACKET(DMA_PACKET_NOP, 0, 0)); if (r) return r; @@ -3630,13 +5650,24 @@ static int evergreen_startup(struct radeon_device *rdev) if (r) return r; + ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + if (ring->ring_size) { + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + RADEON_CP_PACKET2); + if (!r) + r = uvd_v1_0_init(rdev); + + if (r) + DRM_ERROR("radeon: error initializing UVD (%d).\n", r); + } + r = radeon_ib_pool_init(rdev); if (r) { dev_err(rdev->dev, "IB initialization failed (%d).\n", r); return r; } - r = r600_audio_init(rdev); + r = radeon_audio_init(rdev); if (r) { DRM_ERROR("radeon: audio init failed\n"); return r; @@ -3661,6 +5692,12 @@ int evergreen_resume(struct radeon_device *rdev) /* post card */ atom_asic_init(rdev->mode_info.atom_context); + /* init golden registers */ + evergreen_init_golden_registers(rdev); + + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_resume(rdev); + rdev->accel_working = true; r = evergreen_startup(rdev); if (r) { @@ -3675,7 +5712,10 @@ int evergreen_resume(struct radeon_device *rdev) int evergreen_suspend(struct radeon_device *rdev) { - r600_audio_fini(rdev); + radeon_pm_suspend(rdev); + radeon_audio_fini(rdev); + uvd_v1_0_fini(rdev); + radeon_uvd_suspend(rdev); r700_cp_stop(rdev); r600_dma_stop(rdev); evergreen_irq_suspend(rdev); @@ -3722,6 +5762,8 @@ int evergreen_init(struct radeon_device *rdev) DRM_INFO("GPU not posted. posting now...\n"); atom_asic_init(rdev->mode_info.atom_context); } + /* init golden registers */ + evergreen_init_golden_registers(rdev); /* Initialize scratch registers */ r600_scratch_init(rdev); /* Initialize surface registers */ @@ -3747,12 +5789,40 @@ int evergreen_init(struct radeon_device *rdev) if (r) return r; + if (ASIC_IS_DCE5(rdev)) { + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { + r = ni_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + } else { + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { + r = r600_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + } + + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); + r = radeon_uvd_init(rdev); + if (!r) { + rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; + r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], + 4096); + } + rdev->ih.ring_obj = NULL; r600_ih_ring_init(rdev, 64 * 1024); @@ -3767,6 +5837,8 @@ int evergreen_init(struct radeon_device *rdev) r700_cp_fini(rdev); r600_dma_fini(rdev); r600_irq_fini(rdev); + if (rdev->flags & RADEON_IS_IGP) + sumo_rlc_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); radeon_irq_kms_fini(rdev); @@ -3790,14 +5862,18 @@ int evergreen_init(struct radeon_device *rdev) void evergreen_fini(struct radeon_device *rdev) { - r600_audio_fini(rdev); - r600_blit_fini(rdev); + radeon_pm_fini(rdev); + radeon_audio_fini(rdev); r700_cp_fini(rdev); r600_dma_fini(rdev); r600_irq_fini(rdev); + if (rdev->flags & RADEON_IS_IGP) + sumo_rlc_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); radeon_irq_kms_fini(rdev); + uvd_v1_0_fini(rdev); + radeon_uvd_fini(rdev); evergreen_pcie_gart_fini(rdev); r600_vram_scratch_fini(rdev); radeon_gem_fini(rdev); @@ -3811,8 +5887,8 @@ void evergreen_fini(struct radeon_device *rdev) void evergreen_pcie_gen2_enable(struct radeon_device *rdev) { - u32 link_width_cntl, speed_cntl, mask; - int ret; + u32 link_width_cntl, speed_cntl; + u32 mask; if (radeon_pcie_gen2 == 0) return; @@ -3826,15 +5902,14 @@ void evergreen_pcie_gen2_enable(struct radeon_device *rdev) /* x2 cards have a special sequence */ if (ASIC_IS_X2(rdev)) return; - - ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); - if (ret != 0) + + if (drm_pcie_get_speed_cap_mask(rdev->ddev, &mask)) return; - if (!(mask & DRM_PCIE_SPEED_50)) + if (!(mask & (DRM_PCIE_SPEED_50|DRM_PCIE_SPEED_80))) return; - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); if (speed_cntl & LC_CURRENT_DATA_RATE) { DRM_INFO("PCIE gen 2 link speeds already enabled\n"); return; @@ -3845,33 +5920,183 @@ void evergreen_pcie_gen2_enable(struct radeon_device *rdev) if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) || (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { - link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); + link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); link_width_cntl &= ~LC_UPCONFIGURE_DIS; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl |= LC_GEN2_EN_STRAP; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); } else { - link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); + link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ if (1) link_width_cntl |= LC_UPCONFIGURE_DIS; else link_width_cntl &= ~LC_UPCONFIGURE_DIS; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + } +} + +void evergreen_program_aspm(struct radeon_device *rdev) +{ + u32 data, orig; + u32 pcie_lc_cntl, pcie_lc_cntl_old; + bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false; + /* fusion_platform = true + * if the system is a fusion system + * (APU or DGPU in a fusion system). + * todo: check if the system is a fusion platform. + */ + bool fusion_platform = false; + + if (radeon_aspm == 0) + return; + + if (!(rdev->flags & RADEON_IS_PCIE)) + return; + + switch (rdev->family) { + case CHIP_CYPRESS: + case CHIP_HEMLOCK: + case CHIP_JUNIPER: + case CHIP_REDWOOD: + case CHIP_CEDAR: + case CHIP_SUMO: + case CHIP_SUMO2: + case CHIP_PALM: + case CHIP_ARUBA: + disable_l0s = true; + break; + default: + disable_l0s = false; + break; } + + if (rdev->flags & RADEON_IS_IGP) + fusion_platform = true; /* XXX also dGPUs in a fusion system */ + + data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING); + if (fusion_platform) + data &= ~MULTI_PIF; + else + data |= MULTI_PIF; + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PAIRING, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING); + if (fusion_platform) + data &= ~MULTI_PIF; + else + data |= MULTI_PIF; + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PAIRING, data); + + pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL); + pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); + if (!disable_l0s) { + if (rdev->family >= CHIP_BARTS) + pcie_lc_cntl |= LC_L0S_INACTIVITY(7); + else + pcie_lc_cntl |= LC_L0S_INACTIVITY(3); + } + + if (!disable_l1) { + if (rdev->family >= CHIP_BARTS) + pcie_lc_cntl |= LC_L1_INACTIVITY(7); + else + pcie_lc_cntl |= LC_L1_INACTIVITY(8); + + if (!disable_plloff_in_l1) { + data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); + data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); + + data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); + data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); + data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); + data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); + + if (rdev->family >= CHIP_BARTS) { + data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); + data &= ~PLL_RAMP_UP_TIME_0_MASK; + data |= PLL_RAMP_UP_TIME_0(4); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); + + data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); + data &= ~PLL_RAMP_UP_TIME_1_MASK; + data |= PLL_RAMP_UP_TIME_1(4); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); + data &= ~PLL_RAMP_UP_TIME_0_MASK; + data |= PLL_RAMP_UP_TIME_0(4); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); + data &= ~PLL_RAMP_UP_TIME_1_MASK; + data |= PLL_RAMP_UP_TIME_1(4); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); + } + + data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); + data &= ~LC_DYN_LANES_PWR_STATE_MASK; + data |= LC_DYN_LANES_PWR_STATE(3); + if (data != orig) + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); + + if (rdev->family >= CHIP_BARTS) { + data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL); + data &= ~LS2_EXIT_TIME_MASK; + data |= LS2_EXIT_TIME(1); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_CNTL, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL); + data &= ~LS2_EXIT_TIME_MASK; + data |= LS2_EXIT_TIME(1); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_CNTL, data); + } + } + } + + /* evergreen parts only */ + if (rdev->family < CHIP_BARTS) + pcie_lc_cntl |= LC_PMI_TO_L1_DIS; + + if (pcie_lc_cntl != pcie_lc_cntl_old) + WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl); } diff --git a/sys/dev/pci/drm/radeon/evergreen_blit_kms.c b/sys/dev/pci/drm/radeon/evergreen_blit_kms.c deleted file mode 100644 index 149bb1de0f6..00000000000 --- a/sys/dev/pci/drm/radeon/evergreen_blit_kms.c +++ /dev/null @@ -1,730 +0,0 @@ -/* $OpenBSD: evergreen_blit_kms.c,v 1.5 2018/04/20 21:12:50 naddy Exp $ */ -/* - * Copyright 2010 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * Alex Deucher - */ - -#include -#include -#include "radeon.h" - -#include "evergreend.h" -#include "evergreen_blit_shaders.h" -#include "cayman_blit_shaders.h" -#include "radeon_blit_common.h" - -/* emits 17 */ -static void -set_render_target(struct radeon_device *rdev, int format, - int w, int h, u64 gpu_addr) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u32 cb_color_info; - int pitch, slice; - - h = roundup2(h, 8); - if (h < 8) - h = 8; - - cb_color_info = CB_FORMAT(format) | - CB_SOURCE_FORMAT(CB_SF_EXPORT_NORM) | - CB_ARRAY_MODE(ARRAY_1D_TILED_THIN1); - pitch = (w / 8) - 1; - slice = ((w * h) / 64) - 1; - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 15)); - radeon_ring_write(ring, (CB_COLOR0_BASE - PACKET3_SET_CONTEXT_REG_START) >> 2); - radeon_ring_write(ring, gpu_addr >> 8); - radeon_ring_write(ring, pitch); - radeon_ring_write(ring, slice); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, cb_color_info); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, (w - 1) | ((h - 1) << 16)); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); -} - -/* emits 5dw */ -static void -cp_set_surface_sync(struct radeon_device *rdev, - u32 sync_type, u32 size, - u64 mc_addr) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u32 cp_coher_size; - - if (size == 0xffffffff) - cp_coher_size = 0xffffffff; - else - cp_coher_size = ((size + 255) >> 8); - - if (rdev->family >= CHIP_CAYMAN) { - /* CP_COHER_CNTL2 has to be set manually when submitting a surface_sync - * to the RB directly. For IBs, the CP programs this as part of the - * surface_sync packet. - */ - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); - radeon_ring_write(ring, (0x85e8 - PACKET3_SET_CONFIG_REG_START) >> 2); - radeon_ring_write(ring, 0); /* CP_COHER_CNTL2 */ - } - radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); - radeon_ring_write(ring, sync_type); - radeon_ring_write(ring, cp_coher_size); - radeon_ring_write(ring, mc_addr >> 8); - radeon_ring_write(ring, 10); /* poll interval */ -} - -/* emits 11dw + 1 surface sync = 16dw */ -static void -set_shaders(struct radeon_device *rdev) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u64 gpu_addr; - - /* VS */ - gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset; - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 3)); - radeon_ring_write(ring, (SQ_PGM_START_VS - PACKET3_SET_CONTEXT_REG_START) >> 2); - radeon_ring_write(ring, gpu_addr >> 8); - radeon_ring_write(ring, 2); - radeon_ring_write(ring, 0); - - /* PS */ - gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.ps_offset; - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 4)); - radeon_ring_write(ring, (SQ_PGM_START_PS - PACKET3_SET_CONTEXT_REG_START) >> 2); - radeon_ring_write(ring, gpu_addr >> 8); - radeon_ring_write(ring, 1); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 2); - - gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset; - cp_set_surface_sync(rdev, PACKET3_SH_ACTION_ENA, 512, gpu_addr); -} - -/* emits 10 + 1 sync (5) = 15 */ -static void -set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u32 sq_vtx_constant_word2, sq_vtx_constant_word3; - - /* high addr, stride */ - sq_vtx_constant_word2 = SQ_VTXC_BASE_ADDR_HI(upper_32_bits(gpu_addr) & 0xff) | - SQ_VTXC_STRIDE(16); -#ifdef __BIG_ENDIAN - sq_vtx_constant_word2 |= SQ_VTXC_ENDIAN_SWAP(SQ_ENDIAN_8IN32); -#endif - /* xyzw swizzles */ - sq_vtx_constant_word3 = SQ_VTCX_SEL_X(SQ_SEL_X) | - SQ_VTCX_SEL_Y(SQ_SEL_Y) | - SQ_VTCX_SEL_Z(SQ_SEL_Z) | - SQ_VTCX_SEL_W(SQ_SEL_W); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_RESOURCE, 8)); - radeon_ring_write(ring, 0x580); - radeon_ring_write(ring, gpu_addr & 0xffffffff); - radeon_ring_write(ring, 48 - 1); /* size */ - radeon_ring_write(ring, sq_vtx_constant_word2); - radeon_ring_write(ring, sq_vtx_constant_word3); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, S__SQ_CONSTANT_TYPE(SQ_TEX_VTX_VALID_BUFFER)); - - if ((rdev->family == CHIP_CEDAR) || - (rdev->family == CHIP_PALM) || - (rdev->family == CHIP_SUMO) || - (rdev->family == CHIP_SUMO2) || - (rdev->family == CHIP_CAICOS)) - cp_set_surface_sync(rdev, - PACKET3_TC_ACTION_ENA, 48, gpu_addr); - else - cp_set_surface_sync(rdev, - PACKET3_VC_ACTION_ENA, 48, gpu_addr); - -} - -/* emits 10 */ -static void -set_tex_resource(struct radeon_device *rdev, - int format, int w, int h, int pitch, - u64 gpu_addr, u32 size) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u32 sq_tex_resource_word0, sq_tex_resource_word1; - u32 sq_tex_resource_word4, sq_tex_resource_word7; - - if (h < 1) - h = 1; - - sq_tex_resource_word0 = TEX_DIM(SQ_TEX_DIM_2D); - sq_tex_resource_word0 |= ((((pitch >> 3) - 1) << 6) | - ((w - 1) << 18)); - sq_tex_resource_word1 = ((h - 1) << 0) | - TEX_ARRAY_MODE(ARRAY_1D_TILED_THIN1); - /* xyzw swizzles */ - sq_tex_resource_word4 = TEX_DST_SEL_X(SQ_SEL_X) | - TEX_DST_SEL_Y(SQ_SEL_Y) | - TEX_DST_SEL_Z(SQ_SEL_Z) | - TEX_DST_SEL_W(SQ_SEL_W); - - sq_tex_resource_word7 = format | - S__SQ_CONSTANT_TYPE(SQ_TEX_VTX_VALID_TEXTURE); - - cp_set_surface_sync(rdev, - PACKET3_TC_ACTION_ENA, size, gpu_addr); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_RESOURCE, 8)); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, sq_tex_resource_word0); - radeon_ring_write(ring, sq_tex_resource_word1); - radeon_ring_write(ring, gpu_addr >> 8); - radeon_ring_write(ring, gpu_addr >> 8); - radeon_ring_write(ring, sq_tex_resource_word4); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, sq_tex_resource_word7); -} - -/* emits 12 */ -static void -set_scissors(struct radeon_device *rdev, int x1, int y1, - int x2, int y2) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - /* workaround some hw bugs */ - if (x2 == 0) - x1 = 1; - if (y2 == 0) - y1 = 1; - if (rdev->family >= CHIP_CAYMAN) { - if ((x2 == 1) && (y2 == 1)) - x2 = 2; - } - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); - radeon_ring_write(ring, (PA_SC_SCREEN_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2); - radeon_ring_write(ring, (x1 << 0) | (y1 << 16)); - radeon_ring_write(ring, (x2 << 0) | (y2 << 16)); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); - radeon_ring_write(ring, (PA_SC_GENERIC_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2); - radeon_ring_write(ring, (x1 << 0) | (y1 << 16) | (1 << 31)); - radeon_ring_write(ring, (x2 << 0) | (y2 << 16)); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); - radeon_ring_write(ring, (PA_SC_WINDOW_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2); - radeon_ring_write(ring, (x1 << 0) | (y1 << 16) | (1 << 31)); - radeon_ring_write(ring, (x2 << 0) | (y2 << 16)); -} - -/* emits 10 */ -static void -draw_auto(struct radeon_device *rdev) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); - radeon_ring_write(ring, (VGT_PRIMITIVE_TYPE - PACKET3_SET_CONFIG_REG_START) >> 2); - radeon_ring_write(ring, DI_PT_RECTLIST); - - radeon_ring_write(ring, PACKET3(PACKET3_INDEX_TYPE, 0)); - radeon_ring_write(ring, -#ifdef __BIG_ENDIAN - (2 << 2) | -#endif - DI_INDEX_SIZE_16_BIT); - - radeon_ring_write(ring, PACKET3(PACKET3_NUM_INSTANCES, 0)); - radeon_ring_write(ring, 1); - - radeon_ring_write(ring, PACKET3(PACKET3_DRAW_INDEX_AUTO, 1)); - radeon_ring_write(ring, 3); - radeon_ring_write(ring, DI_SRC_SEL_AUTO_INDEX); - -} - -/* emits 39 */ -static void -set_default_state(struct radeon_device *rdev) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u32 sq_config, sq_gpr_resource_mgmt_1, sq_gpr_resource_mgmt_2, sq_gpr_resource_mgmt_3; - u32 sq_thread_resource_mgmt, sq_thread_resource_mgmt_2; - u32 sq_stack_resource_mgmt_1, sq_stack_resource_mgmt_2, sq_stack_resource_mgmt_3; - int num_ps_gprs, num_vs_gprs, num_temp_gprs; - int num_gs_gprs, num_es_gprs, num_hs_gprs, num_ls_gprs; - int num_ps_threads, num_vs_threads, num_gs_threads, num_es_threads; - int num_hs_threads, num_ls_threads; - int num_ps_stack_entries, num_vs_stack_entries, num_gs_stack_entries, num_es_stack_entries; - int num_hs_stack_entries, num_ls_stack_entries; - u64 gpu_addr; - int dwords; - - /* set clear context state */ - radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); - radeon_ring_write(ring, 0); - - if (rdev->family < CHIP_CAYMAN) { - switch (rdev->family) { - case CHIP_CEDAR: - default: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 96; - num_vs_threads = 16; - num_gs_threads = 16; - num_es_threads = 16; - num_hs_threads = 16; - num_ls_threads = 16; - num_ps_stack_entries = 42; - num_vs_stack_entries = 42; - num_gs_stack_entries = 42; - num_es_stack_entries = 42; - num_hs_stack_entries = 42; - num_ls_stack_entries = 42; - break; - case CHIP_REDWOOD: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 128; - num_vs_threads = 20; - num_gs_threads = 20; - num_es_threads = 20; - num_hs_threads = 20; - num_ls_threads = 20; - num_ps_stack_entries = 42; - num_vs_stack_entries = 42; - num_gs_stack_entries = 42; - num_es_stack_entries = 42; - num_hs_stack_entries = 42; - num_ls_stack_entries = 42; - break; - case CHIP_JUNIPER: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 128; - num_vs_threads = 20; - num_gs_threads = 20; - num_es_threads = 20; - num_hs_threads = 20; - num_ls_threads = 20; - num_ps_stack_entries = 85; - num_vs_stack_entries = 85; - num_gs_stack_entries = 85; - num_es_stack_entries = 85; - num_hs_stack_entries = 85; - num_ls_stack_entries = 85; - break; - case CHIP_CYPRESS: - case CHIP_HEMLOCK: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 128; - num_vs_threads = 20; - num_gs_threads = 20; - num_es_threads = 20; - num_hs_threads = 20; - num_ls_threads = 20; - num_ps_stack_entries = 85; - num_vs_stack_entries = 85; - num_gs_stack_entries = 85; - num_es_stack_entries = 85; - num_hs_stack_entries = 85; - num_ls_stack_entries = 85; - break; - case CHIP_PALM: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 96; - num_vs_threads = 16; - num_gs_threads = 16; - num_es_threads = 16; - num_hs_threads = 16; - num_ls_threads = 16; - num_ps_stack_entries = 42; - num_vs_stack_entries = 42; - num_gs_stack_entries = 42; - num_es_stack_entries = 42; - num_hs_stack_entries = 42; - num_ls_stack_entries = 42; - break; - case CHIP_SUMO: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 96; - num_vs_threads = 25; - num_gs_threads = 25; - num_es_threads = 25; - num_hs_threads = 25; - num_ls_threads = 25; - num_ps_stack_entries = 42; - num_vs_stack_entries = 42; - num_gs_stack_entries = 42; - num_es_stack_entries = 42; - num_hs_stack_entries = 42; - num_ls_stack_entries = 42; - break; - case CHIP_SUMO2: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 96; - num_vs_threads = 25; - num_gs_threads = 25; - num_es_threads = 25; - num_hs_threads = 25; - num_ls_threads = 25; - num_ps_stack_entries = 85; - num_vs_stack_entries = 85; - num_gs_stack_entries = 85; - num_es_stack_entries = 85; - num_hs_stack_entries = 85; - num_ls_stack_entries = 85; - break; - case CHIP_BARTS: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 128; - num_vs_threads = 20; - num_gs_threads = 20; - num_es_threads = 20; - num_hs_threads = 20; - num_ls_threads = 20; - num_ps_stack_entries = 85; - num_vs_stack_entries = 85; - num_gs_stack_entries = 85; - num_es_stack_entries = 85; - num_hs_stack_entries = 85; - num_ls_stack_entries = 85; - break; - case CHIP_TURKS: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 128; - num_vs_threads = 20; - num_gs_threads = 20; - num_es_threads = 20; - num_hs_threads = 20; - num_ls_threads = 20; - num_ps_stack_entries = 42; - num_vs_stack_entries = 42; - num_gs_stack_entries = 42; - num_es_stack_entries = 42; - num_hs_stack_entries = 42; - num_ls_stack_entries = 42; - break; - case CHIP_CAICOS: - num_ps_gprs = 93; - num_vs_gprs = 46; - num_temp_gprs = 4; - num_gs_gprs = 31; - num_es_gprs = 31; - num_hs_gprs = 23; - num_ls_gprs = 23; - num_ps_threads = 128; - num_vs_threads = 10; - num_gs_threads = 10; - num_es_threads = 10; - num_hs_threads = 10; - num_ls_threads = 10; - num_ps_stack_entries = 42; - num_vs_stack_entries = 42; - num_gs_stack_entries = 42; - num_es_stack_entries = 42; - num_hs_stack_entries = 42; - num_ls_stack_entries = 42; - break; - } - - if ((rdev->family == CHIP_CEDAR) || - (rdev->family == CHIP_PALM) || - (rdev->family == CHIP_SUMO) || - (rdev->family == CHIP_SUMO2) || - (rdev->family == CHIP_CAICOS)) - sq_config = 0; - else - sq_config = VC_ENABLE; - - sq_config |= (EXPORT_SRC_C | - CS_PRIO(0) | - LS_PRIO(0) | - HS_PRIO(0) | - PS_PRIO(0) | - VS_PRIO(1) | - GS_PRIO(2) | - ES_PRIO(3)); - - sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(num_ps_gprs) | - NUM_VS_GPRS(num_vs_gprs) | - NUM_CLAUSE_TEMP_GPRS(num_temp_gprs)); - sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(num_gs_gprs) | - NUM_ES_GPRS(num_es_gprs)); - sq_gpr_resource_mgmt_3 = (NUM_HS_GPRS(num_hs_gprs) | - NUM_LS_GPRS(num_ls_gprs)); - sq_thread_resource_mgmt = (NUM_PS_THREADS(num_ps_threads) | - NUM_VS_THREADS(num_vs_threads) | - NUM_GS_THREADS(num_gs_threads) | - NUM_ES_THREADS(num_es_threads)); - sq_thread_resource_mgmt_2 = (NUM_HS_THREADS(num_hs_threads) | - NUM_LS_THREADS(num_ls_threads)); - sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(num_ps_stack_entries) | - NUM_VS_STACK_ENTRIES(num_vs_stack_entries)); - sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(num_gs_stack_entries) | - NUM_ES_STACK_ENTRIES(num_es_stack_entries)); - sq_stack_resource_mgmt_3 = (NUM_HS_STACK_ENTRIES(num_hs_stack_entries) | - NUM_LS_STACK_ENTRIES(num_ls_stack_entries)); - - /* disable dyn gprs */ - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); - radeon_ring_write(ring, (SQ_DYN_GPR_CNTL_PS_FLUSH_REQ - PACKET3_SET_CONFIG_REG_START) >> 2); - radeon_ring_write(ring, 0); - - /* setup LDS */ - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); - radeon_ring_write(ring, (SQ_LDS_RESOURCE_MGMT - PACKET3_SET_CONFIG_REG_START) >> 2); - radeon_ring_write(ring, 0x10001000); - - /* SQ config */ - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 11)); - radeon_ring_write(ring, (SQ_CONFIG - PACKET3_SET_CONFIG_REG_START) >> 2); - radeon_ring_write(ring, sq_config); - radeon_ring_write(ring, sq_gpr_resource_mgmt_1); - radeon_ring_write(ring, sq_gpr_resource_mgmt_2); - radeon_ring_write(ring, sq_gpr_resource_mgmt_3); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, sq_thread_resource_mgmt); - radeon_ring_write(ring, sq_thread_resource_mgmt_2); - radeon_ring_write(ring, sq_stack_resource_mgmt_1); - radeon_ring_write(ring, sq_stack_resource_mgmt_2); - radeon_ring_write(ring, sq_stack_resource_mgmt_3); - } - - /* CONTEXT_CONTROL */ - radeon_ring_write(ring, 0xc0012800); - radeon_ring_write(ring, 0x80000000); - radeon_ring_write(ring, 0x80000000); - - /* SQ_VTX_BASE_VTX_LOC */ - radeon_ring_write(ring, 0xc0026f00); - radeon_ring_write(ring, 0x00000000); - radeon_ring_write(ring, 0x00000000); - radeon_ring_write(ring, 0x00000000); - - /* SET_SAMPLER */ - radeon_ring_write(ring, 0xc0036e00); - radeon_ring_write(ring, 0x00000000); - radeon_ring_write(ring, 0x00000012); - radeon_ring_write(ring, 0x00000000); - radeon_ring_write(ring, 0x00000000); - - /* set to DX10/11 mode */ - radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0)); - radeon_ring_write(ring, 1); - - /* emit an IB pointing at default state */ - dwords = roundup2(rdev->r600_blit.state_len, 0x10); - gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.state_offset; - radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - radeon_ring_write(ring, gpu_addr & 0xFFFFFFFC); - radeon_ring_write(ring, upper_32_bits(gpu_addr) & 0xFF); - radeon_ring_write(ring, dwords); - -} - -int evergreen_blit_init(struct radeon_device *rdev) -{ - u32 obj_size; - int i, r, dwords; - void *ptr; - u32 packet2s[16]; - int num_packet2s = 0; - - rdev->r600_blit.primitives.set_render_target = set_render_target; - rdev->r600_blit.primitives.cp_set_surface_sync = cp_set_surface_sync; - rdev->r600_blit.primitives.set_shaders = set_shaders; - rdev->r600_blit.primitives.set_vtx_resource = set_vtx_resource; - rdev->r600_blit.primitives.set_tex_resource = set_tex_resource; - rdev->r600_blit.primitives.set_scissors = set_scissors; - rdev->r600_blit.primitives.draw_auto = draw_auto; - rdev->r600_blit.primitives.set_default_state = set_default_state; - - rdev->r600_blit.ring_size_common = 8; /* sync semaphore */ - rdev->r600_blit.ring_size_common += 55; /* shaders + def state */ - rdev->r600_blit.ring_size_common += 16; /* fence emit for VB IB */ - rdev->r600_blit.ring_size_common += 5; /* done copy */ - rdev->r600_blit.ring_size_common += 16; /* fence emit for done copy */ - - rdev->r600_blit.ring_size_per_loop = 74; - if (rdev->family >= CHIP_CAYMAN) - rdev->r600_blit.ring_size_per_loop += 9; /* additional DWs for surface sync */ - - rdev->r600_blit.max_dim = 16384; - - rdev->r600_blit.state_offset = 0; - - if (rdev->family < CHIP_CAYMAN) - rdev->r600_blit.state_len = evergreen_default_size; - else - rdev->r600_blit.state_len = cayman_default_size; - - dwords = rdev->r600_blit.state_len; - while (dwords & 0xf) { - packet2s[num_packet2s++] = cpu_to_le32(PACKET2(0)); - dwords++; - } - - obj_size = dwords * 4; - obj_size = roundup2(obj_size, 256); - - rdev->r600_blit.vs_offset = obj_size; - if (rdev->family < CHIP_CAYMAN) - obj_size += evergreen_vs_size * 4; - else - obj_size += cayman_vs_size * 4; - obj_size = roundup2(obj_size, 256); - - rdev->r600_blit.ps_offset = obj_size; - if (rdev->family < CHIP_CAYMAN) - obj_size += evergreen_ps_size * 4; - else - obj_size += cayman_ps_size * 4; - obj_size = roundup2(obj_size, 256); - - /* pin copy shader into vram if not already initialized */ - if (!rdev->r600_blit.shader_obj) { - r = radeon_bo_create(rdev, obj_size, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_VRAM, - NULL, &rdev->r600_blit.shader_obj); - if (r) { - DRM_ERROR("evergreen failed to allocate shader\n"); - return r; - } - - r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false); - if (unlikely(r != 0)) - return r; - r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM, - &rdev->r600_blit.shader_gpu_addr); - radeon_bo_unreserve(rdev->r600_blit.shader_obj); - if (r) { - dev_err(rdev->dev, "(%d) pin blit object failed\n", r); - return r; - } - } - - DRM_DEBUG("evergreen blit allocated bo %08x vs %08x ps %08x\n", - obj_size, - rdev->r600_blit.vs_offset, rdev->r600_blit.ps_offset); - - r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false); - if (unlikely(r != 0)) - return r; - r = radeon_bo_kmap(rdev->r600_blit.shader_obj, &ptr); - if (r) { - DRM_ERROR("failed to map blit object %d\n", r); - return r; - } - - if (rdev->family < CHIP_CAYMAN) { - memcpy_toio(ptr + rdev->r600_blit.state_offset, - evergreen_default_state, rdev->r600_blit.state_len * 4); - - if (num_packet2s) - memcpy_toio(ptr + rdev->r600_blit.state_offset + (rdev->r600_blit.state_len * 4), - packet2s, num_packet2s * 4); - for (i = 0; i < evergreen_vs_size; i++) - *(u32 *)((unsigned long)ptr + rdev->r600_blit.vs_offset + i * 4) = cpu_to_le32(evergreen_vs[i]); - for (i = 0; i < evergreen_ps_size; i++) - *(u32 *)((unsigned long)ptr + rdev->r600_blit.ps_offset + i * 4) = cpu_to_le32(evergreen_ps[i]); - } else { - memcpy_toio(ptr + rdev->r600_blit.state_offset, - cayman_default_state, rdev->r600_blit.state_len * 4); - - if (num_packet2s) - memcpy_toio(ptr + rdev->r600_blit.state_offset + (rdev->r600_blit.state_len * 4), - packet2s, num_packet2s * 4); - for (i = 0; i < cayman_vs_size; i++) - *(u32 *)((unsigned long)ptr + rdev->r600_blit.vs_offset + i * 4) = cpu_to_le32(cayman_vs[i]); - for (i = 0; i < cayman_ps_size; i++) - *(u32 *)((unsigned long)ptr + rdev->r600_blit.ps_offset + i * 4) = cpu_to_le32(cayman_ps[i]); - } - radeon_bo_kunmap(rdev->r600_blit.shader_obj); - radeon_bo_unreserve(rdev->r600_blit.shader_obj); - - radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); - return 0; -} diff --git a/sys/dev/pci/drm/radeon/evergreen_blit_shaders.c b/sys/dev/pci/drm/radeon/evergreen_blit_shaders.c index 85a58332e15..cf426879fbb 100644 --- a/sys/dev/pci/drm/radeon/evergreen_blit_shaders.c +++ b/sys/dev/pci/drm/radeon/evergreen_blit_shaders.c @@ -1,4 +1,3 @@ -/* $OpenBSD: evergreen_blit_shaders.c,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * @@ -25,15 +24,13 @@ * Alex Deucher */ -#include - -#include +#include /* * evergreen cards need to use the 3D engine to blit data which requires * quite a bit of hw state setup. Rather than pull the whole 3D driver * (which normally generates the 3D state) into the DRM, we opt to use - * statically generated state tables. The regsiter state and shaders + * statically generated state tables. The register state and shaders * were hand generated to support blitting functionality. See the 3D * driver or documentation for descriptions of the registers and * shader instructions. @@ -301,58 +298,4 @@ const u32 evergreen_default_state[] = 0x00000010, /* */ }; -const u32 evergreen_vs[] = -{ - 0x00000004, - 0x80800400, - 0x0000a03c, - 0x95000688, - 0x00004000, - 0x15200688, - 0x00000000, - 0x00000000, - 0x3c000000, - 0x67961001, -#ifdef __BIG_ENDIAN - 0x000a0000, -#else - 0x00080000, -#endif - 0x00000000, - 0x1c000000, - 0x67961000, -#ifdef __BIG_ENDIAN - 0x00020008, -#else - 0x00000008, -#endif - 0x00000000, -}; - -const u32 evergreen_ps[] = -{ - 0x00000003, - 0xa00c0000, - 0x00000008, - 0x80400000, - 0x00000000, - 0x95200688, - 0x00380400, - 0x00146b10, - 0x00380000, - 0x20146b10, - 0x00380400, - 0x40146b00, - 0x80380000, - 0x60146b00, - 0x00000000, - 0x00000000, - 0x00000010, - 0x000d1000, - 0xb0800000, - 0x00000000, -}; - -const u32 evergreen_ps_size = ARRAY_SIZE(evergreen_ps); -const u32 evergreen_vs_size = ARRAY_SIZE(evergreen_vs); const u32 evergreen_default_size = ARRAY_SIZE(evergreen_default_state); diff --git a/sys/dev/pci/drm/radeon/evergreen_blit_shaders.h b/sys/dev/pci/drm/radeon/evergreen_blit_shaders.h index 32d549cfb57..bb8d6c75159 100644 --- a/sys/dev/pci/drm/radeon/evergreen_blit_shaders.h +++ b/sys/dev/pci/drm/radeon/evergreen_blit_shaders.h @@ -1,4 +1,3 @@ -/* $OpenBSD: evergreen_blit_shaders.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2009 Advanced Micro Devices, Inc. * diff --git a/sys/dev/pci/drm/radeon/evergreen_cs.c b/sys/dev/pci/drm/radeon/evergreen_cs.c index 285f681d960..a48d9bd90aa 100644 --- a/sys/dev/pci/drm/radeon/evergreen_cs.c +++ b/sys/dev/pci/drm/radeon/evergreen_cs.c @@ -1,4 +1,3 @@ -/* $OpenBSD: evergreen_cs.c,v 1.8 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -35,11 +34,10 @@ #define MAX(a,b) (((a)>(b))?(a):(b)) #define MIN(a,b) (((a)<(b))?(a):(b)) -int r600_dma_cs_next_reloc(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc); -static int evergreen_cs_packet_next_reloc(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc); +#define REG_SAFE_BM_SIZE ARRAY_SIZE(evergreen_reg_safe_bm) +int r600_dma_cs_next_reloc(struct radeon_cs_parser *p, + struct radeon_bo_list **cs_reloc); struct evergreen_cs_track { u32 group_size; u32 nbanks; @@ -87,6 +85,8 @@ struct evergreen_cs_track { u32 htile_offset; u32 htile_surface; struct radeon_bo *htile_bo; + unsigned long indirect_draw_buffer_size; + const unsigned *reg_safe_bm; }; static u32 evergreen_cs_get_aray_mode(u32 tiling_flags) @@ -447,7 +447,7 @@ static int evergreen_cs_track_validate_cb(struct radeon_cs_parser *p, unsigned i * command stream. */ if (!surf.mode) { - volatile u32 *ib = p->ib.ptr; + uint32_t *ib = p->ib.ptr; unsigned long tmp, nby, bsize, size, min = 0; /* find the height the ddx wants */ @@ -822,7 +822,7 @@ static int evergreen_cs_track_validate_texture(struct radeon_cs_parser *p, /* align height */ evergreen_surface_check(p, &surf, NULL); - surf.nby = roundup(surf.nby, surf.halign); + surf.nby = roundup2(surf.nby, surf.halign); r = evergreen_surface_check(p, &surf, "texture"); if (r) { @@ -838,7 +838,7 @@ static int evergreen_cs_track_validate_texture(struct radeon_cs_parser *p, __func__, __LINE__, toffset, surf.base_align); return -EINVAL; } - if (moffset & (surf.base_align - 1)) { + if (surf.nsamples <= 1 && moffset & (surf.base_align - 1)) { dev_warn(p->dev, "%s:%d mipmap bo base %ld not aligned with %ld\n", __func__, __LINE__, moffset, surf.base_align); return -EINVAL; @@ -895,8 +895,8 @@ static int evergreen_cs_track_validate_texture(struct radeon_cs_parser *p, __func__, __LINE__, surf.mode); return -EINVAL; } - surf.nbx = roundup(surf.nbx, surf.palign); - surf.nby = roundup(surf.nby, surf.halign); + surf.nbx = roundup2(surf.nbx, surf.palign); + surf.nby = roundup2(surf.nby, surf.halign); r = evergreen_surface_check(p, &surf, "mipmap"); if (r) { @@ -1013,223 +1013,35 @@ static int evergreen_cs_track_check(struct radeon_cs_parser *p) } /** - * evergreen_cs_packet_parse() - parse cp packet and point ib index to next packet - * @parser: parser structure holding parsing context. - * @pkt: where to store packet informations - * - * Assume that chunk_ib_index is properly set. Will return -EINVAL - * if packet is bigger than remaining ib size. or if packets is unknown. - **/ -static int evergreen_cs_packet_parse(struct radeon_cs_parser *p, - struct radeon_cs_packet *pkt, - unsigned idx) -{ - struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx]; - uint32_t header; - - if (idx >= ib_chunk->length_dw) { - DRM_ERROR("Can not parse packet at %d after CS end %d !\n", - idx, ib_chunk->length_dw); - return -EINVAL; - } - header = radeon_get_ib_value(p, idx); - pkt->idx = idx; - pkt->type = CP_PACKET_GET_TYPE(header); - pkt->count = CP_PACKET_GET_COUNT(header); - pkt->one_reg_wr = 0; - switch (pkt->type) { - case PACKET_TYPE0: - pkt->reg = CP_PACKET0_GET_REG(header); - break; - case PACKET_TYPE3: - pkt->opcode = CP_PACKET3_GET_OPCODE(header); - break; - case PACKET_TYPE2: - pkt->count = -1; - break; - default: - DRM_ERROR("Unknown packet type %d at %d !\n", pkt->type, idx); - return -EINVAL; - } - if ((pkt->count + 1 + pkt->idx) >= ib_chunk->length_dw) { - DRM_ERROR("Packet (%d:%d:%d) end after CS buffer (%d) !\n", - pkt->idx, pkt->type, pkt->count, ib_chunk->length_dw); - return -EINVAL; - } - return 0; -} - -/** - * evergreen_cs_packet_next_reloc() - parse next packet which should be reloc packet3 - * @parser: parser structure holding parsing context. - * @data: pointer to relocation data - * @offset_start: starting offset - * @offset_mask: offset mask (to align start offset on) - * @reloc: reloc informations - * - * Check next packet is relocation packet3, do bo validation and compute - * GPU offset using the provided start. - **/ -static int evergreen_cs_packet_next_reloc(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc) -{ - struct radeon_cs_chunk *relocs_chunk; - struct radeon_cs_packet p3reloc; - unsigned idx; - int r; - - if (p->chunk_relocs_idx == -1) { - DRM_ERROR("No relocation chunk !\n"); - return -EINVAL; - } - *cs_reloc = NULL; - relocs_chunk = &p->chunks[p->chunk_relocs_idx]; - r = evergreen_cs_packet_parse(p, &p3reloc, p->idx); - if (r) { - return r; - } - p->idx += p3reloc.count + 2; - if (p3reloc.type != PACKET_TYPE3 || p3reloc.opcode != PACKET3_NOP) { - DRM_ERROR("No packet3 for relocation for packet at %d.\n", - p3reloc.idx); - return -EINVAL; - } - idx = radeon_get_ib_value(p, p3reloc.idx + 1); - if (idx >= relocs_chunk->length_dw) { - DRM_ERROR("Relocs at %d after relocations chunk end %d !\n", - idx, relocs_chunk->length_dw); - return -EINVAL; - } - /* FIXME: we assume reloc size is 4 dwords */ - *cs_reloc = p->relocs_ptr[(idx / 4)]; - return 0; -} - -/** - * evergreen_cs_packet_next_is_pkt3_nop() - test if the next packet is NOP - * @p: structure holding the parser context. - * - * Check if the next packet is a relocation packet3. - **/ -static bool evergreen_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p) -{ - struct radeon_cs_packet p3reloc; - int r; - - r = evergreen_cs_packet_parse(p, &p3reloc, p->idx); - if (r) { - return false; - } - if (p3reloc.type != PACKET_TYPE3 || p3reloc.opcode != PACKET3_NOP) { - return false; - } - return true; -} - -/** - * evergreen_cs_packet_next_vline() - parse userspace VLINE packet + * evergreen_cs_packet_parse_vline() - parse userspace VLINE packet * @parser: parser structure holding parsing context. * - * Userspace sends a special sequence for VLINE waits. - * PACKET0 - VLINE_START_END + value - * PACKET3 - WAIT_REG_MEM poll vline status reg - * RELOC (P3) - crtc_id in reloc. - * - * This function parses this and relocates the VLINE START END - * and WAIT_REG_MEM packets to the correct crtc. - * It also detects a switched off crtc and nulls out the - * wait in that case. + * This is an Evergreen(+)-specific function for parsing VLINE packets. + * Real work is done by r600_cs_common_vline_parse function. + * Here we just set up ASIC-specific register table and call + * the common implementation function. */ static int evergreen_cs_packet_parse_vline(struct radeon_cs_parser *p) { - struct drm_mode_object *obj; - struct drm_crtc *crtc; - struct radeon_crtc *radeon_crtc; - struct radeon_cs_packet p3reloc, wait_reg_mem; - int crtc_id; - int r; - uint32_t header, h_idx, reg, wait_reg_mem_info; - volatile uint32_t *ib; - - ib = p->ib.ptr; - - /* parse the WAIT_REG_MEM */ - r = evergreen_cs_packet_parse(p, &wait_reg_mem, p->idx); - if (r) - return r; - - /* check its a WAIT_REG_MEM */ - if (wait_reg_mem.type != PACKET_TYPE3 || - wait_reg_mem.opcode != PACKET3_WAIT_REG_MEM) { - DRM_ERROR("vline wait missing WAIT_REG_MEM segment\n"); - return -EINVAL; - } - - wait_reg_mem_info = radeon_get_ib_value(p, wait_reg_mem.idx + 1); - /* bit 4 is reg (0) or mem (1) */ - if (wait_reg_mem_info & 0x10) { - DRM_ERROR("vline WAIT_REG_MEM waiting on MEM rather than REG\n"); - return -EINVAL; - } - /* waiting for value to be equal */ - if ((wait_reg_mem_info & 0x7) != 0x3) { - DRM_ERROR("vline WAIT_REG_MEM function not equal\n"); - return -EINVAL; - } - if ((radeon_get_ib_value(p, wait_reg_mem.idx + 2) << 2) != EVERGREEN_VLINE_STATUS) { - DRM_ERROR("vline WAIT_REG_MEM bad reg\n"); - return -EINVAL; - } - if (radeon_get_ib_value(p, wait_reg_mem.idx + 5) != EVERGREEN_VLINE_STAT) { - DRM_ERROR("vline WAIT_REG_MEM bad bit mask\n"); - return -EINVAL; - } - - /* jump over the NOP */ - r = evergreen_cs_packet_parse(p, &p3reloc, p->idx + wait_reg_mem.count + 2); - if (r) - return r; - - h_idx = p->idx - 2; - p->idx += wait_reg_mem.count + 2; - p->idx += p3reloc.count + 2; - - header = radeon_get_ib_value(p, h_idx); - crtc_id = radeon_get_ib_value(p, h_idx + 2 + 7 + 1); - reg = CP_PACKET0_GET_REG(header); - obj = drm_mode_object_find(p->rdev->ddev, crtc_id, DRM_MODE_OBJECT_CRTC); - if (!obj) { - DRM_ERROR("cannot find crtc %d\n", crtc_id); - return -EINVAL; - } - crtc = obj_to_crtc(obj); - radeon_crtc = to_radeon_crtc(crtc); - crtc_id = radeon_crtc->crtc_id; - - if (!crtc->enabled) { - /* if the CRTC isn't enabled - we need to nop out the WAIT_REG_MEM */ - ib[h_idx + 2] = PACKET2(0); - ib[h_idx + 3] = PACKET2(0); - ib[h_idx + 4] = PACKET2(0); - ib[h_idx + 5] = PACKET2(0); - ib[h_idx + 6] = PACKET2(0); - ib[h_idx + 7] = PACKET2(0); - ib[h_idx + 8] = PACKET2(0); - } else { - switch (reg) { - case EVERGREEN_VLINE_START_END: - header &= ~R600_CP_PACKET0_REG_MASK; - header |= (EVERGREEN_VLINE_START_END + radeon_crtc->crtc_offset) >> 2; - ib[h_idx] = header; - ib[h_idx + 4] = (EVERGREEN_VLINE_STATUS + radeon_crtc->crtc_offset) >> 2; - break; - default: - DRM_ERROR("unknown crtc reloc\n"); - return -EINVAL; - } - } - return 0; + static uint32_t vline_start_end[6] = { + EVERGREEN_VLINE_START_END + EVERGREEN_CRTC0_REGISTER_OFFSET, + EVERGREEN_VLINE_START_END + EVERGREEN_CRTC1_REGISTER_OFFSET, + EVERGREEN_VLINE_START_END + EVERGREEN_CRTC2_REGISTER_OFFSET, + EVERGREEN_VLINE_START_END + EVERGREEN_CRTC3_REGISTER_OFFSET, + EVERGREEN_VLINE_START_END + EVERGREEN_CRTC4_REGISTER_OFFSET, + EVERGREEN_VLINE_START_END + EVERGREEN_CRTC5_REGISTER_OFFSET + }; + static uint32_t vline_status[6] = { + EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, + EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, + EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, + EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, + EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, + EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET + }; + + return r600_cs_common_vline_parse(p, vline_start_end, vline_status); } static int evergreen_packet0_check(struct radeon_cs_parser *p, @@ -1274,41 +1086,18 @@ static int evergreen_cs_parse_packet0(struct radeon_cs_parser *p, } /** - * evergreen_cs_check_reg() - check if register is authorized or not + * evergreen_cs_handle_reg() - process registers that need special handling. * @parser: parser structure holding parsing context * @reg: register we are testing * @idx: index into the cs buffer - * - * This function will test against evergreen_reg_safe_bm and return 0 - * if register is safe. If register is not flag as safe this function - * will test it against a list of register needind special handling. */ -static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) +static int evergreen_cs_handle_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) { struct evergreen_cs_track *track = (struct evergreen_cs_track *)p->track; - struct radeon_cs_reloc *reloc; - u32 last_reg; - u32 m, i, tmp, *ib; + struct radeon_bo_list *reloc; + u32 tmp, *ib; int r; - if (p->rdev->family >= CHIP_CAYMAN) - last_reg = ARRAY_SIZE(cayman_reg_safe_bm); - else - last_reg = ARRAY_SIZE(evergreen_reg_safe_bm); - - i = (reg >> 7); - if (i >= last_reg) { - dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx); - return -EINVAL; - } - m = 1 << ((reg >> 2) & 31); - if (p->rdev->family >= CHIP_CAYMAN) { - if (!(cayman_reg_safe_bm[i] & m)) - return 0; - } else { - if (!(evergreen_reg_safe_bm[i] & m)) - return 0; - } ib = p->ib.ptr; switch (reg) { /* force following reg to 0 in an attempt to disable out buffer @@ -1351,13 +1140,13 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case SQ_LSTMP_RING_BASE: case SQ_PSTMP_RING_BASE: case SQ_VSTMP_RING_BASE: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); break; case DB_DEPTH_CONTROL: track->db_depth_control = radeon_get_ib_value(p, idx); @@ -1380,7 +1169,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case DB_Z_INFO: track->db_z_info = radeon_get_ib_value(p, idx); if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); @@ -1388,12 +1177,12 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) } ib[idx] &= ~Z_ARRAY_MODE(0xf); track->db_z_info &= ~Z_ARRAY_MODE(0xf); - ib[idx] |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags)); - track->db_z_info |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags)); - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) { + ib[idx] |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); + track->db_z_info |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); + if (reloc->tiling_flags & RADEON_TILING_MACRO) { unsigned bankw, bankh, mtaspect, tile_split; - evergreen_tiling_fields(reloc->lobj.tiling_flags, + evergreen_tiling_fields(reloc->tiling_flags, &bankw, &bankh, &mtaspect, &tile_split); ib[idx] |= DB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks)); @@ -1422,50 +1211,50 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) track->db_dirty = true; break; case DB_Z_READ_BASE: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } track->db_z_read_offset = radeon_get_ib_value(p, idx); - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->db_z_read_bo = reloc->robj; track->db_dirty = true; break; case DB_Z_WRITE_BASE: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } track->db_z_write_offset = radeon_get_ib_value(p, idx); - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->db_z_write_bo = reloc->robj; track->db_dirty = true; break; case DB_STENCIL_READ_BASE: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } track->db_s_read_offset = radeon_get_ib_value(p, idx); - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->db_s_read_bo = reloc->robj; track->db_dirty = true; break; case DB_STENCIL_WRITE_BASE: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } track->db_s_write_offset = radeon_get_ib_value(p, idx); - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->db_s_write_bo = reloc->robj; track->db_dirty = true; break; @@ -1481,7 +1270,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case VGT_STRMOUT_BUFFER_BASE_1: case VGT_STRMOUT_BUFFER_BASE_2: case VGT_STRMOUT_BUFFER_BASE_3: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); @@ -1489,7 +1278,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) } tmp = (reg - VGT_STRMOUT_BUFFER_BASE_0) / 16; track->vgt_strmout_bo_offset[tmp] = radeon_get_ib_value(p, idx) << 8; - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->vgt_strmout_bo[tmp] = reloc->robj; track->streamout_dirty = true; break; @@ -1503,13 +1292,13 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) track->streamout_dirty = true; break; case CP_COHER_BASE: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "missing reloc for CP_COHER_BASE " "0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); case CB_TARGET_MASK: track->cb_target_mask = radeon_get_ib_value(p, idx); track->cb_dirty = true; @@ -1567,14 +1356,14 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) tmp = (reg - CB_COLOR0_INFO) / 0x3c; track->cb_color_info[tmp] = radeon_get_ib_value(p, idx); if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } - ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags)); - track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags)); + ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); + track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); } track->cb_dirty = true; break; @@ -1585,14 +1374,14 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) tmp = ((reg - CB_COLOR8_INFO) / 0x1c) + 8; track->cb_color_info[tmp] = radeon_get_ib_value(p, idx); if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } - ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags)); - track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags)); + ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); + track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); } track->cb_dirty = true; break; @@ -1646,17 +1435,17 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case CB_COLOR5_ATTRIB: case CB_COLOR6_ATTRIB: case CB_COLOR7_ATTRIB: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) { + if (reloc->tiling_flags & RADEON_TILING_MACRO) { unsigned bankw, bankh, mtaspect, tile_split; - evergreen_tiling_fields(reloc->lobj.tiling_flags, + evergreen_tiling_fields(reloc->tiling_flags, &bankw, &bankh, &mtaspect, &tile_split); ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks)); @@ -1674,17 +1463,17 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case CB_COLOR9_ATTRIB: case CB_COLOR10_ATTRIB: case CB_COLOR11_ATTRIB: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) { + if (reloc->tiling_flags & RADEON_TILING_MACRO) { unsigned bankw, bankh, mtaspect, tile_split; - evergreen_tiling_fields(reloc->lobj.tiling_flags, + evergreen_tiling_fields(reloc->tiling_flags, &bankw, &bankh, &mtaspect, &tile_split); ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks)); @@ -1707,12 +1496,12 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case CB_COLOR6_FMASK: case CB_COLOR7_FMASK: tmp = (reg - CB_COLOR0_FMASK) / 0x3c; - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->cb_color_fmask_bo[tmp] = reloc->robj; break; case CB_COLOR0_CMASK: @@ -1724,12 +1513,12 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case CB_COLOR6_CMASK: case CB_COLOR7_CMASK: tmp = (reg - CB_COLOR0_CMASK) / 0x3c; - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->cb_color_cmask_bo[tmp] = reloc->robj; break; case CB_COLOR0_FMASK_SLICE: @@ -1762,7 +1551,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case CB_COLOR5_BASE: case CB_COLOR6_BASE: case CB_COLOR7_BASE: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); @@ -1770,7 +1559,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) } tmp = (reg - CB_COLOR0_BASE) / 0x3c; track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx); - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->cb_color_bo[tmp] = reloc->robj; track->cb_dirty = true; break; @@ -1778,7 +1567,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case CB_COLOR9_BASE: case CB_COLOR10_BASE: case CB_COLOR11_BASE: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); @@ -1786,19 +1575,19 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) } tmp = ((reg - CB_COLOR8_BASE) / 0x1c) + 8; track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx); - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->cb_color_bo[tmp] = reloc->robj; track->cb_dirty = true; break; case DB_HTILE_DATA_BASE: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } track->htile_offset = radeon_get_ib_value(p, idx); - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->htile_bo = reloc->robj; track->db_dirty = true; break; @@ -1909,13 +1698,13 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case SQ_ALU_CONST_CACHE_LS_13: case SQ_ALU_CONST_CACHE_LS_14: case SQ_ALU_CONST_CACHE_LS_15: - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); break; case SX_MEMORY_EXPORT_BASE: if (p->rdev->family >= CHIP_CAYMAN) { @@ -1923,13 +1712,13 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) "0x%04X\n", reg); return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONFIG_REG " "0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); break; case CAYMAN_SX_SCATTER_EXPORT_BASE: if (p->rdev->family < CHIP_CAYMAN) { @@ -1937,13 +1726,13 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) "0x%04X\n", reg); return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); break; case SX_MISC: track->sx_misc_kill_all_prims = (radeon_get_ib_value(p, idx) & 0x1) != 0; @@ -1955,38 +1744,36 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) return 0; } -static bool evergreen_is_safe_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) +/** + * evergreen_is_safe_reg() - check if register is authorized or not + * @parser: parser structure holding parsing context + * @reg: register we are testing + * + * This function will test against reg_safe_bm and return true + * if register is safe or false otherwise. + */ +static inline bool evergreen_is_safe_reg(struct radeon_cs_parser *p, u32 reg) { - u32 last_reg, m, i; - - if (p->rdev->family >= CHIP_CAYMAN) - last_reg = ARRAY_SIZE(cayman_reg_safe_bm); - else - last_reg = ARRAY_SIZE(evergreen_reg_safe_bm); + struct evergreen_cs_track *track = p->track; + u32 m, i; i = (reg >> 7); - if (i >= last_reg) { - dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx); + if (unlikely(i >= REG_SAFE_BM_SIZE)) { return false; } m = 1 << ((reg >> 2) & 31); - if (p->rdev->family >= CHIP_CAYMAN) { - if (!(cayman_reg_safe_bm[i] & m)) - return true; - } else { - if (!(evergreen_reg_safe_bm[i] & m)) - return true; - } - dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx); + if (!(track->reg_safe_bm[i] & m)) + return true; + return false; } static int evergreen_packet3_check(struct radeon_cs_parser *p, struct radeon_cs_packet *pkt) { - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; struct evergreen_cs_track *track; - volatile u32 *ib; + uint32_t *ib; unsigned idx; unsigned i; unsigned start_reg, end_reg, reg; @@ -2022,13 +1809,13 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad SET PREDICATION\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + (idx_value & 0xfffffff0) + ((u64)(tmp & 0xff) << 32); @@ -2068,13 +1855,13 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad INDEX_BASE\n"); return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad INDEX_BASE\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + idx_value + ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32); @@ -2088,6 +1875,14 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, } break; } + case PACKET3_INDEX_BUFFER_SIZE: + { + if (pkt->count != 0) { + DRM_ERROR("bad INDEX_BUFFER_SIZE\n"); + return -EINVAL; + } + break; + } case PACKET3_DRAW_INDEX: { uint64_t offset; @@ -2095,13 +1890,13 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad DRAW_INDEX\n"); return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad DRAW_INDEX\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + idx_value + ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32); @@ -2123,13 +1918,13 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad DRAW_INDEX_2\n"); return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad DRAW_INDEX_2\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + radeon_get_ib_value(p, idx+1) + ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); @@ -2198,6 +1993,67 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, return r; } break; + case PACKET3_SET_BASE: + { + /* + DW 1 HEADER Header of the packet. Shader_Type in bit 1 of the Header will correspond to the shader type of the Load, see Type-3 Packet. + 2 BASE_INDEX Bits [3:0] BASE_INDEX - Base Index specifies which base address is specified in the last two DWs. + 0001: DX11 Draw_Index_Indirect Patch Table Base: Base address for Draw_Index_Indirect data. + 3 ADDRESS_LO Bits [31:3] - Lower bits of QWORD-Aligned Address. Bits [2:0] - Reserved + 4 ADDRESS_HI Bits [31:8] - Reserved. Bits [7:0] - Upper bits of Address [47:32] + */ + if (pkt->count != 2) { + DRM_ERROR("bad SET_BASE\n"); + return -EINVAL; + } + + /* currently only supporting setting indirect draw buffer base address */ + if (idx_value != 1) { + DRM_ERROR("bad SET_BASE\n"); + return -EINVAL; + } + + r = radeon_cs_packet_next_reloc(p, &reloc, 0); + if (r) { + DRM_ERROR("bad SET_BASE\n"); + return -EINVAL; + } + + track->indirect_draw_buffer_size = radeon_bo_size(reloc->robj); + + ib[idx+1] = reloc->gpu_offset; + ib[idx+2] = upper_32_bits(reloc->gpu_offset) & 0xff; + + break; + } + case PACKET3_DRAW_INDIRECT: + case PACKET3_DRAW_INDEX_INDIRECT: + { + u64 size = pkt->opcode == PACKET3_DRAW_INDIRECT ? 16 : 20; + + /* + DW 1 HEADER + 2 DATA_OFFSET Bits [31:0] + byte aligned offset where the required data structure starts. Bits 1:0 are zero + 3 DRAW_INITIATOR Draw Initiator Register. Written to the VGT_DRAW_INITIATOR register for the assigned context + */ + if (pkt->count != 1) { + DRM_ERROR("bad DRAW_INDIRECT\n"); + return -EINVAL; + } + + if (idx_value + size > track->indirect_draw_buffer_size) { + dev_warn(p->dev, "DRAW_INDIRECT buffer too small %u + %llu > %lu\n", + idx_value, size, track->indirect_draw_buffer_size); + return -EINVAL; + } + + r = evergreen_cs_track_check(p); + if (r) { + dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); + return r; + } + break; + } case PACKET3_DISPATCH_DIRECT: if (pkt->count != 3) { DRM_ERROR("bad DISPATCH_DIRECT\n"); @@ -2214,12 +2070,12 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad DISPATCH_INDIRECT\n"); return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad DISPATCH_INDIRECT\n"); return -EINVAL; } - ib[idx+0] = idx_value + (u32)(reloc->lobj.gpu_offset & 0xffffffff); + ib[idx+0] = idx_value + (u32)(reloc->gpu_offset & 0xffffffff); r = evergreen_cs_track_check(p); if (r) { dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); @@ -2235,18 +2091,21 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, if (idx_value & 0x10) { uint64_t offset; - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad WAIT_REG_MEM\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + (radeon_get_ib_value(p, idx+1) & 0xfffffffc) + ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); ib[idx+1] = (ib[idx+1] & 0x3) | (offset & 0xfffffffc); ib[idx+2] = upper_32_bits(offset) & 0xff; + } else if (idx_value & 0x100) { + DRM_ERROR("cannot use PFP on REG wait\n"); + return -EINVAL; } break; case PACKET3_CP_DMA: @@ -2286,7 +2145,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, } /* src address space is memory */ if (((info & 0x60000000) >> 29) == 0) { - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad CP DMA SRC\n"); return -EINVAL; @@ -2295,7 +2154,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, tmp = radeon_get_ib_value(p, idx) + ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32); - offset = reloc->lobj.gpu_offset + tmp; + offset = reloc->gpu_offset + tmp; if ((tmp + size) > radeon_bo_size(reloc->robj)) { dev_warn(p->dev, "CP DMA src buffer too small (%llu %lu)\n", @@ -2324,7 +2183,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, return -EINVAL; } if (((info & 0x00300000) >> 20) == 0) { - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad CP DMA DST\n"); return -EINVAL; @@ -2333,7 +2192,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, tmp = radeon_get_ib_value(p, idx+2) + ((u64)(radeon_get_ib_value(p, idx+3) & 0xff) << 32); - offset = reloc->lobj.gpu_offset + tmp; + offset = reloc->gpu_offset + tmp; if ((tmp + size) > radeon_bo_size(reloc->robj)) { dev_warn(p->dev, "CP DMA dst buffer too small (%llu %lu)\n", @@ -2358,12 +2217,12 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, /* 0xffffffff/0x0 is flush all cache flag */ if (radeon_get_ib_value(p, idx + 1) != 0xffffffff || radeon_get_ib_value(p, idx + 2) != 0) { - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad SURFACE_SYNC\n"); return -EINVAL; } - ib[idx+2] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx+2] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); } break; case PACKET3_EVENT_WRITE: @@ -2374,12 +2233,12 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, if (pkt->count) { uint64_t offset; - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad EVENT_WRITE\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + (radeon_get_ib_value(p, idx+1) & 0xfffffff8) + ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); @@ -2395,13 +2254,13 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad EVENT_WRITE_EOP\n"); return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad EVENT_WRITE_EOP\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + (radeon_get_ib_value(p, idx+1) & 0xfffffffc) + ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); @@ -2417,13 +2276,13 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad EVENT_WRITE_EOS\n"); return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad EVENT_WRITE_EOS\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + (radeon_get_ib_value(p, idx+1) & 0xfffffffc) + ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); @@ -2440,9 +2299,10 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n"); return -EINVAL; } - for (i = 0; i < pkt->count; i++) { - reg = start_reg + (4 * i); - r = evergreen_cs_check_reg(p, reg, idx+1+i); + for (reg = start_reg, idx++; reg <= end_reg; reg += 4, idx++) { + if (evergreen_is_safe_reg(p, reg)) + continue; + r = evergreen_cs_handle_reg(p, reg, idx); if (r) return r; } @@ -2456,9 +2316,10 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad PACKET3_SET_CONTEXT_REG\n"); return -EINVAL; } - for (i = 0; i < pkt->count; i++) { - reg = start_reg + (4 * i); - r = evergreen_cs_check_reg(p, reg, idx+1+i); + for (reg = start_reg, idx++; reg <= end_reg; reg += 4, idx++) { + if (evergreen_is_safe_reg(p, reg)) + continue; + r = evergreen_cs_handle_reg(p, reg, idx); if (r) return r; } @@ -2484,18 +2345,18 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, switch (G__SQ_CONSTANT_TYPE(radeon_get_ib_value(p, idx+1+(i*8)+7))) { case SQ_TEX_VTX_VALID_TEXTURE: /* tex base */ - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad SET_RESOURCE (tex)\n"); return -EINVAL; } if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { ib[idx+1+(i*8)+1] |= - TEX_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags)); - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) { + TEX_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); + if (reloc->tiling_flags & RADEON_TILING_MACRO) { unsigned bankw, bankh, mtaspect, tile_split; - evergreen_tiling_fields(reloc->lobj.tiling_flags, + evergreen_tiling_fields(reloc->tiling_flags, &bankw, &bankh, &mtaspect, &tile_split); ib[idx+1+(i*8)+6] |= TEX_TILE_SPLIT(tile_split); @@ -2507,7 +2368,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, } } texture = reloc->robj; - toffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + toffset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff); /* tex mip base */ tex_dim = ib[idx+1+(i*8)+0] & 0x7; @@ -2515,18 +2376,18 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, if ((tex_dim == SQ_TEX_DIM_2D_MSAA || tex_dim == SQ_TEX_DIM_2D_ARRAY_MSAA) && !mip_address && - !evergreen_cs_packet_next_is_pkt3_nop(p)) { + !radeon_cs_packet_next_is_pkt3_nop(p)) { /* MIP_ADDRESS should point to FMASK for an MSAA texture. * It should be 0 if FMASK is disabled. */ moffset = 0; mipmap = NULL; } else { - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad SET_RESOURCE (tex)\n"); return -EINVAL; } - moffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + moffset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff); mipmap = reloc->robj; } @@ -2540,7 +2401,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, { uint64_t offset64; /* vtx base */ - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad SET_RESOURCE (vtx)\n"); return -EINVAL; @@ -2553,7 +2414,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, ib[idx+1+(i*8)+1] = radeon_bo_size(reloc->robj) - offset; } - offset64 = reloc->lobj.gpu_offset + offset; + offset64 = reloc->gpu_offset + offset; ib[idx+1+(i*8)+0] = offset64; ib[idx+1+(i*8)+2] = (ib[idx+1+(i*8)+2] & 0xffffff00) | (upper_32_bits(offset64) & 0xff); @@ -2622,7 +2483,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, /* Updating memory at DST_ADDRESS. */ if (idx_value & 0x1) { u64 offset; - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing dst reloc)\n"); return -EINVAL; @@ -2634,14 +2495,14 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, offset + 4, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+1] = offset; ib[idx+2] = upper_32_bits(offset) & 0xff; } /* Reading data from SRC_ADDRESS. */ if (((idx_value >> 1) & 0x3) == 2) { u64 offset; - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing src reloc)\n"); return -EINVAL; @@ -2653,7 +2514,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, offset + 4, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+3] = offset; ib[idx+4] = upper_32_bits(offset) & 0xff; } @@ -2666,7 +2527,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad MEM_WRITE (invalid count)\n"); return -EINVAL; } - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad MEM_WRITE (missing reloc)\n"); return -EINVAL; @@ -2682,7 +2543,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, offset + 8, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+0] = offset; ib[idx+1] = upper_32_bits(offset) & 0xff; break; @@ -2695,7 +2556,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, if (idx_value & 0x1) { u64 offset; /* SRC is memory. */ - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad COPY_DW (missing src reloc)\n"); return -EINVAL; @@ -2707,19 +2568,22 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, offset + 4, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+1] = offset; ib[idx+2] = upper_32_bits(offset) & 0xff; } else { /* SRC is a reg. */ reg = radeon_get_ib_value(p, idx+1) << 2; - if (!evergreen_is_safe_reg(p, reg, idx+1)) + if (!evergreen_is_safe_reg(p, reg)) { + dev_warn(p->dev, "forbidden register 0x%08x at %d\n", + reg, idx + 1); return -EINVAL; + } } if (idx_value & 0x2) { u64 offset; /* DST is memory. */ - r = evergreen_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("bad COPY_DW (missing dst reloc)\n"); return -EINVAL; @@ -2731,14 +2595,17 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, offset + 4, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+3] = offset; ib[idx+4] = upper_32_bits(offset) & 0xff; } else { /* DST is a reg. */ reg = radeon_get_ib_value(p, idx+3) << 2; - if (!evergreen_is_safe_reg(p, reg, idx+3)) + if (!evergreen_is_safe_reg(p, reg)) { + dev_warn(p->dev, "forbidden register 0x%08x at %d\n", + reg, idx + 3); return -EINVAL; + } } break; case PACKET3_NOP: @@ -2763,11 +2630,15 @@ int evergreen_cs_parse(struct radeon_cs_parser *p) if (track == NULL) return -ENOMEM; evergreen_cs_track_init(track); - if (p->rdev->family >= CHIP_CAYMAN) + if (p->rdev->family >= CHIP_CAYMAN) { tmp = p->rdev->config.cayman.tile_config; - else + track->reg_safe_bm = cayman_reg_safe_bm; + } else { tmp = p->rdev->config.evergreen.tile_config; - + track->reg_safe_bm = evergreen_reg_safe_bm; + } + BUILD_BUG_ON(ARRAY_SIZE(cayman_reg_safe_bm) != REG_SAFE_BM_SIZE); + BUILD_BUG_ON(ARRAY_SIZE(evergreen_reg_safe_bm) != REG_SAFE_BM_SIZE); switch (tmp & 0xf) { case 0: track->npipes = 1; @@ -2823,7 +2694,7 @@ int evergreen_cs_parse(struct radeon_cs_parser *p) p->track = track; } do { - r = evergreen_cs_packet_parse(p, &pkt, p->idx); + r = radeon_cs_packet_parse(p, &pkt, p->idx); if (r) { kfree(p->track); p->track = NULL; @@ -2831,12 +2702,12 @@ int evergreen_cs_parse(struct radeon_cs_parser *p) } p->idx += pkt.count + 2; switch (pkt.type) { - case PACKET_TYPE0: + case RADEON_PACKET_TYPE0: r = evergreen_cs_parse_packet0(p, &pkt); break; - case PACKET_TYPE2: + case RADEON_PACKET_TYPE2: break; - case PACKET_TYPE3: + case RADEON_PACKET_TYPE3: r = evergreen_packet3_check(p, &pkt); break; default: @@ -2850,7 +2721,7 @@ int evergreen_cs_parse(struct radeon_cs_parser *p) p->track = NULL; return r; } - } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw); + } while (p->idx < p->chunk_ib->length_dw); #if 0 for (r = 0; r < p->ib.length_dw; r++) { printk(KERN_INFO "%05d 0x%08X\n", r, p->ib.ptr[r]); @@ -2862,16 +2733,6 @@ int evergreen_cs_parse(struct radeon_cs_parser *p) return 0; } -/* - * DMA - */ - -#define GET_DMA_CMD(h) (((h) & 0xf0000000) >> 28) -#define GET_DMA_COUNT(h) ((h) & 0x000fffff) -#define GET_DMA_T(h) (((h) & 0x00800000) >> 23) -#define GET_DMA_NEW(h) (((h) & 0x04000000) >> 26) -#define GET_DMA_MISC(h) (((h) & 0x0700000) >> 20) - /** * evergreen_dma_cs_parse() - parse the DMA IB * @p: parser structure holding parsing context. @@ -2883,11 +2744,11 @@ int evergreen_cs_parse(struct radeon_cs_parser *p) **/ int evergreen_dma_cs_parse(struct radeon_cs_parser *p) { - struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx]; - struct radeon_cs_reloc *src_reloc, *dst_reloc, *dst2_reloc; - u32 header, cmd, count, tiled, new_cmd, misc; - volatile u32 *ib = p->ib.ptr; - u32 idx, idx_value; + struct radeon_cs_chunk *ib_chunk = p->chunk_ib; + struct radeon_bo_list *src_reloc, *dst_reloc, *dst2_reloc; + u32 header, cmd, count, sub_cmd; + uint32_t *ib = p->ib.ptr; + u32 idx; u64 src_offset, dst_offset, dst2_offset; int r; @@ -2901,9 +2762,7 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p) header = radeon_get_ib_value(p, idx); cmd = GET_DMA_CMD(header); count = GET_DMA_COUNT(header); - tiled = GET_DMA_T(header); - new_cmd = GET_DMA_NEW(header); - misc = GET_DMA_MISC(header); + sub_cmd = GET_DMA_SUB_CMD(header); switch (cmd) { case DMA_PACKET_WRITE: @@ -2912,19 +2771,27 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p) DRM_ERROR("bad DMA_PACKET_WRITE\n"); return -EINVAL; } - if (tiled) { + switch (sub_cmd) { + /* tiled */ + case 8: dst_offset = radeon_get_ib_value(p, idx+1); dst_offset <<= 8; - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8); + ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); p->idx += count + 7; - } else { + break; + /* linear */ + case 0: dst_offset = radeon_get_ib_value(p, idx+1); dst_offset |= ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32; - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+2] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; + ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+2] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; p->idx += count + 3; + break; + default: + DRM_ERROR("bad DMA_PACKET_WRITE [%6d] 0x%08x sub cmd is not 0 or 8\n", idx, header); + return -EINVAL; } if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { dev_warn(p->dev, "DMA write buffer too small (%llu %lu)\n", @@ -2943,338 +2810,330 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p) DRM_ERROR("bad DMA_PACKET_COPY\n"); return -EINVAL; } - if (tiled) { - idx_value = radeon_get_ib_value(p, idx + 2); - if (new_cmd) { - switch (misc) { - case 0: - /* L2T, frame to fields */ - if (idx_value & (1 << 31)) { - DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n"); - return -EINVAL; - } - r = r600_dma_cs_next_reloc(p, &dst2_reloc); - if (r) { - DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n"); - return -EINVAL; - } - dst_offset = radeon_get_ib_value(p, idx+1); - dst_offset <<= 8; - dst2_offset = radeon_get_ib_value(p, idx+2); - dst2_offset <<= 8; - src_offset = radeon_get_ib_value(p, idx+8); - src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32; - if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, frame to fields src buffer too small (%llu %lu)\n", - src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); - return -EINVAL; - } - if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n", - dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); - return -EINVAL; - } - if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n", - dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); - return -EINVAL; - } - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8); - ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8); - ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - p->idx += 10; - break; - case 1: - /* L2T, T2L partial */ - if (p->family < CHIP_CAYMAN) { - DRM_ERROR("L2T, T2L Partial is cayman only !\n"); - return -EINVAL; - } - /* detile bit */ - if (idx_value & (1 << 31)) { - /* tiled src, linear dst */ - ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8); - - ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; - } else { - /* linear src, tiled dst */ - ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8); - } - p->idx += 12; - break; - case 3: - /* L2T, broadcast */ - if (idx_value & (1 << 31)) { - DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); - return -EINVAL; - } - r = r600_dma_cs_next_reloc(p, &dst2_reloc); - if (r) { - DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); - return -EINVAL; - } - dst_offset = radeon_get_ib_value(p, idx+1); - dst_offset <<= 8; - dst2_offset = radeon_get_ib_value(p, idx+2); - dst2_offset <<= 8; - src_offset = radeon_get_ib_value(p, idx+8); - src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32; - if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n", - src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); - return -EINVAL; - } - if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n", - dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); - return -EINVAL; - } - if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n", - dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); - return -EINVAL; - } - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8); - ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8); - ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - p->idx += 10; - break; - case 4: - /* L2T, T2L */ - /* detile bit */ - if (idx_value & (1 << 31)) { - /* tiled src, linear dst */ - src_offset = radeon_get_ib_value(p, idx+1); - src_offset <<= 8; - ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8); - - dst_offset = radeon_get_ib_value(p, idx+7); - dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; - ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; - } else { - /* linear src, tiled dst */ - src_offset = radeon_get_ib_value(p, idx+7); - src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; - ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - - dst_offset = radeon_get_ib_value(p, idx+1); - dst_offset <<= 8; - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8); - } - if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, T2L src buffer too small (%llu %lu)\n", - src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); - return -EINVAL; - } - if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, T2L dst buffer too small (%llu %lu)\n", - dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); - return -EINVAL; - } - p->idx += 9; - break; - case 5: - /* T2T partial */ - if (p->family < CHIP_CAYMAN) { - DRM_ERROR("L2T, T2L Partial is cayman only !\n"); - return -EINVAL; - } - ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8); - ib[idx+4] += (u32)(dst_reloc->lobj.gpu_offset >> 8); - p->idx += 13; - break; - case 7: - /* L2T, broadcast */ - if (idx_value & (1 << 31)) { - DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); - return -EINVAL; - } - r = r600_dma_cs_next_reloc(p, &dst2_reloc); - if (r) { - DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); - return -EINVAL; - } - dst_offset = radeon_get_ib_value(p, idx+1); - dst_offset <<= 8; - dst2_offset = radeon_get_ib_value(p, idx+2); - dst2_offset <<= 8; - src_offset = radeon_get_ib_value(p, idx+8); - src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32; - if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n", - src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); - return -EINVAL; - } - if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n", - dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); - return -EINVAL; - } - if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n", - dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); - return -EINVAL; - } - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8); - ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8); - ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - p->idx += 10; - break; - default: - DRM_ERROR("bad DMA_PACKET_COPY misc %u\n", misc); - return -EINVAL; - } + switch (sub_cmd) { + /* Copy L2L, DW aligned */ + case 0x00: + /* L2L, dw */ + src_offset = radeon_get_ib_value(p, idx+2); + src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; + dst_offset = radeon_get_ib_value(p, idx+1); + dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32; + if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { + dev_warn(p->dev, "DMA L2L, dw src buffer too small (%llu %lu)\n", + src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); + return -EINVAL; + } + if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { + dev_warn(p->dev, "DMA L2L, dw dst buffer too small (%llu %lu)\n", + dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); + return -EINVAL; + } + ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+3] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; + ib[idx+4] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + p->idx += 5; + break; + /* Copy L2T/T2L */ + case 0x08: + /* detile bit */ + if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { + /* tiled src, linear dst */ + src_offset = radeon_get_ib_value(p, idx+1); + src_offset <<= 8; + ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8); + + dst_offset = radeon_get_ib_value(p, idx + 7); + dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; + ib[idx+7] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+8] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; } else { - switch (misc) { - case 0: - /* detile bit */ - if (idx_value & (1 << 31)) { - /* tiled src, linear dst */ - src_offset = radeon_get_ib_value(p, idx+1); - src_offset <<= 8; - ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8); - - dst_offset = radeon_get_ib_value(p, idx+7); - dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; - ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; - } else { - /* linear src, tiled dst */ - src_offset = radeon_get_ib_value(p, idx+7); - src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; - ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - - dst_offset = radeon_get_ib_value(p, idx+1); - dst_offset <<= 8; - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8); - } - if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n", - src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); - return -EINVAL; - } - if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { - dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n", - dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); - return -EINVAL; - } - p->idx += 9; - break; - default: - DRM_ERROR("bad DMA_PACKET_COPY misc %u\n", misc); - return -EINVAL; - } + /* linear src, tiled dst */ + src_offset = radeon_get_ib_value(p, idx+7); + src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; + ib[idx+7] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+8] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + + dst_offset = radeon_get_ib_value(p, idx+1); + dst_offset <<= 8; + ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); } - } else { - if (new_cmd) { - switch (misc) { - case 0: - /* L2L, byte */ - src_offset = radeon_get_ib_value(p, idx+2); - src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; - dst_offset = radeon_get_ib_value(p, idx+1); - dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32; - if ((src_offset + count) > radeon_bo_size(src_reloc->robj)) { - dev_warn(p->dev, "DMA L2L, byte src buffer too small (%llu %lu)\n", - src_offset + count, radeon_bo_size(src_reloc->robj)); - return -EINVAL; - } - if ((dst_offset + count) > radeon_bo_size(dst_reloc->robj)) { - dev_warn(p->dev, "DMA L2L, byte dst buffer too small (%llu %lu)\n", - dst_offset + count, radeon_bo_size(dst_reloc->robj)); - return -EINVAL; - } - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xffffffff); - ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xffffffff); - ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; - ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - p->idx += 5; - break; - case 1: - /* L2L, partial */ - if (p->family < CHIP_CAYMAN) { - DRM_ERROR("L2L Partial is cayman only !\n"); - return -EINVAL; - } - ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset & 0xffffffff); - ib[idx+2] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - ib[idx+4] += (u32)(dst_reloc->lobj.gpu_offset & 0xffffffff); - ib[idx+5] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; - - p->idx += 9; - break; - case 4: - /* L2L, dw, broadcast */ - r = r600_dma_cs_next_reloc(p, &dst2_reloc); - if (r) { - DRM_ERROR("bad L2L, dw, broadcast DMA_PACKET_COPY\n"); - return -EINVAL; - } - dst_offset = radeon_get_ib_value(p, idx+1); - dst_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; - dst2_offset = radeon_get_ib_value(p, idx+2); - dst2_offset |= ((u64)(radeon_get_ib_value(p, idx+5) & 0xff)) << 32; - src_offset = radeon_get_ib_value(p, idx+3); - src_offset |= ((u64)(radeon_get_ib_value(p, idx+6) & 0xff)) << 32; - if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { - dev_warn(p->dev, "DMA L2L, dw, broadcast src buffer too small (%llu %lu)\n", - src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); - return -EINVAL; - } - if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { - dev_warn(p->dev, "DMA L2L, dw, broadcast dst buffer too small (%llu %lu)\n", - dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); - return -EINVAL; - } - if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { - dev_warn(p->dev, "DMA L2L, dw, broadcast dst2 buffer too small (%llu %lu)\n", - dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); - return -EINVAL; - } - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+3] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+4] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; - ib[idx+5] += upper_32_bits(dst2_reloc->lobj.gpu_offset) & 0xff; - ib[idx+6] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - p->idx += 7; - break; - default: - DRM_ERROR("bad DMA_PACKET_COPY misc %u\n", misc); - return -EINVAL; - } + if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, src buffer too small (%llu %lu)\n", + src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); + return -EINVAL; + } + if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, dst buffer too small (%llu %lu)\n", + dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); + return -EINVAL; + } + p->idx += 9; + break; + /* Copy L2L, byte aligned */ + case 0x40: + /* L2L, byte */ + src_offset = radeon_get_ib_value(p, idx+2); + src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; + dst_offset = radeon_get_ib_value(p, idx+1); + dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32; + if ((src_offset + count) > radeon_bo_size(src_reloc->robj)) { + dev_warn(p->dev, "DMA L2L, byte src buffer too small (%llu %lu)\n", + src_offset + count, radeon_bo_size(src_reloc->robj)); + return -EINVAL; + } + if ((dst_offset + count) > radeon_bo_size(dst_reloc->robj)) { + dev_warn(p->dev, "DMA L2L, byte dst buffer too small (%llu %lu)\n", + dst_offset + count, radeon_bo_size(dst_reloc->robj)); + return -EINVAL; + } + ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xffffffff); + ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xffffffff); + ib[idx+3] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; + ib[idx+4] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + p->idx += 5; + break; + /* Copy L2L, partial */ + case 0x41: + /* L2L, partial */ + if (p->family < CHIP_CAYMAN) { + DRM_ERROR("L2L Partial is cayman only !\n"); + return -EINVAL; + } + ib[idx+1] += (u32)(src_reloc->gpu_offset & 0xffffffff); + ib[idx+2] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + ib[idx+4] += (u32)(dst_reloc->gpu_offset & 0xffffffff); + ib[idx+5] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; + + p->idx += 9; + break; + /* Copy L2L, DW aligned, broadcast */ + case 0x44: + /* L2L, dw, broadcast */ + r = r600_dma_cs_next_reloc(p, &dst2_reloc); + if (r) { + DRM_ERROR("bad L2L, dw, broadcast DMA_PACKET_COPY\n"); + return -EINVAL; + } + dst_offset = radeon_get_ib_value(p, idx+1); + dst_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; + dst2_offset = radeon_get_ib_value(p, idx+2); + dst2_offset |= ((u64)(radeon_get_ib_value(p, idx+5) & 0xff)) << 32; + src_offset = radeon_get_ib_value(p, idx+3); + src_offset |= ((u64)(radeon_get_ib_value(p, idx+6) & 0xff)) << 32; + if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { + dev_warn(p->dev, "DMA L2L, dw, broadcast src buffer too small (%llu %lu)\n", + src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); + return -EINVAL; + } + if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { + dev_warn(p->dev, "DMA L2L, dw, broadcast dst buffer too small (%llu %lu)\n", + dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); + return -EINVAL; + } + if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { + dev_warn(p->dev, "DMA L2L, dw, broadcast dst2 buffer too small (%llu %lu)\n", + dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); + return -EINVAL; + } + ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+2] += (u32)(dst2_reloc->gpu_offset & 0xfffffffc); + ib[idx+3] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+4] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; + ib[idx+5] += upper_32_bits(dst2_reloc->gpu_offset) & 0xff; + ib[idx+6] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + p->idx += 7; + break; + /* Copy L2T Frame to Field */ + case 0x48: + if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { + DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n"); + return -EINVAL; + } + r = r600_dma_cs_next_reloc(p, &dst2_reloc); + if (r) { + DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n"); + return -EINVAL; + } + dst_offset = radeon_get_ib_value(p, idx+1); + dst_offset <<= 8; + dst2_offset = radeon_get_ib_value(p, idx+2); + dst2_offset <<= 8; + src_offset = radeon_get_ib_value(p, idx+8); + src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32; + if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, frame to fields src buffer too small (%llu %lu)\n", + src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); + return -EINVAL; + } + if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n", + dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); + return -EINVAL; + } + if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n", + dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); + return -EINVAL; + } + ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); + ib[idx+2] += (u32)(dst2_reloc->gpu_offset >> 8); + ib[idx+8] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+9] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + p->idx += 10; + break; + /* Copy L2T/T2L, partial */ + case 0x49: + /* L2T, T2L partial */ + if (p->family < CHIP_CAYMAN) { + DRM_ERROR("L2T, T2L Partial is cayman only !\n"); + return -EINVAL; + } + /* detile bit */ + if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { + /* tiled src, linear dst */ + ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8); + + ib[idx+7] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+8] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; + } else { + /* linear src, tiled dst */ + ib[idx+7] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+8] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + + ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); + } + p->idx += 12; + break; + /* Copy L2T broadcast */ + case 0x4b: + /* L2T, broadcast */ + if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { + DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); + return -EINVAL; + } + r = r600_dma_cs_next_reloc(p, &dst2_reloc); + if (r) { + DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); + return -EINVAL; + } + dst_offset = radeon_get_ib_value(p, idx+1); + dst_offset <<= 8; + dst2_offset = radeon_get_ib_value(p, idx+2); + dst2_offset <<= 8; + src_offset = radeon_get_ib_value(p, idx+8); + src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32; + if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n", + src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); + return -EINVAL; + } + if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n", + dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); + return -EINVAL; + } + if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n", + dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); + return -EINVAL; + } + ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); + ib[idx+2] += (u32)(dst2_reloc->gpu_offset >> 8); + ib[idx+8] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+9] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + p->idx += 10; + break; + /* Copy L2T/T2L (tile units) */ + case 0x4c: + /* L2T, T2L */ + /* detile bit */ + if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { + /* tiled src, linear dst */ + src_offset = radeon_get_ib_value(p, idx+1); + src_offset <<= 8; + ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8); + + dst_offset = radeon_get_ib_value(p, idx+7); + dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; + ib[idx+7] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+8] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; } else { - /* L2L, dw */ - src_offset = radeon_get_ib_value(p, idx+2); - src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; + /* linear src, tiled dst */ + src_offset = radeon_get_ib_value(p, idx+7); + src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; + ib[idx+7] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+8] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + dst_offset = radeon_get_ib_value(p, idx+1); - dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32; - if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { - dev_warn(p->dev, "DMA L2L, dw src buffer too small (%llu %lu)\n", - src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); - return -EINVAL; - } - if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { - dev_warn(p->dev, "DMA L2L, dw dst buffer too small (%llu %lu)\n", - dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); - return -EINVAL; - } - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; - ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - p->idx += 5; + dst_offset <<= 8; + ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); } + if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, T2L src buffer too small (%llu %lu)\n", + src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); + return -EINVAL; + } + if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, T2L dst buffer too small (%llu %lu)\n", + dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); + return -EINVAL; + } + p->idx += 9; + break; + /* Copy T2T, partial (tile units) */ + case 0x4d: + /* T2T partial */ + if (p->family < CHIP_CAYMAN) { + DRM_ERROR("L2T, T2L Partial is cayman only !\n"); + return -EINVAL; + } + ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8); + ib[idx+4] += (u32)(dst_reloc->gpu_offset >> 8); + p->idx += 13; + break; + /* Copy L2T broadcast (tile units) */ + case 0x4f: + /* L2T, broadcast */ + if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { + DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); + return -EINVAL; + } + r = r600_dma_cs_next_reloc(p, &dst2_reloc); + if (r) { + DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); + return -EINVAL; + } + dst_offset = radeon_get_ib_value(p, idx+1); + dst_offset <<= 8; + dst2_offset = radeon_get_ib_value(p, idx+2); + dst2_offset <<= 8; + src_offset = radeon_get_ib_value(p, idx+8); + src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32; + if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n", + src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); + return -EINVAL; + } + if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n", + dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); + return -EINVAL; + } + if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { + dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n", + dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); + return -EINVAL; + } + ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); + ib[idx+2] += (u32)(dst2_reloc->gpu_offset >> 8); + ib[idx+8] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+9] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + p->idx += 10; + break; + default: + DRM_ERROR("bad DMA_PACKET_COPY [%6d] 0x%08x invalid sub cmd\n", idx, header); + return -EINVAL; } break; case DMA_PACKET_CONSTANT_FILL: @@ -3290,8 +3149,8 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p) dst_offset, radeon_bo_size(dst_reloc->robj)); return -EINVAL; } - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+3] += (upper_32_bits(dst_reloc->lobj.gpu_offset) << 16) & 0x00ff0000; + ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+3] += (upper_32_bits(dst_reloc->gpu_offset) << 16) & 0x00ff0000; p->idx += 4; break; case DMA_PACKET_NOP: @@ -3301,7 +3160,7 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p) DRM_ERROR("Unknown packet type %d at %d !\n", cmd, idx); return -EINVAL; } - } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw); + } while (p->idx < p->chunk_ib->length_dw); #if 0 for (r = 0; r < p->ib->length_dw; r++) { printk(KERN_INFO "%05d 0x%08X\n", r, p->ib.ptr[r]); @@ -3444,7 +3303,13 @@ static int evergreen_vm_packet3_check(struct radeon_device *rdev, switch (pkt->opcode) { case PACKET3_NOP: + break; case PACKET3_SET_BASE: + if (idx_value != 1) { + DRM_ERROR("bad SET_BASE"); + return -EINVAL; + } + break; case PACKET3_CLEAR_STATE: case PACKET3_INDEX_BUFFER_SIZE: case PACKET3_DISPATCH_DIRECT: @@ -3587,19 +3452,19 @@ int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) do { pkt.idx = idx; - pkt.type = CP_PACKET_GET_TYPE(ib->ptr[idx]); - pkt.count = CP_PACKET_GET_COUNT(ib->ptr[idx]); + pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]); + pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]); pkt.one_reg_wr = 0; switch (pkt.type) { - case PACKET_TYPE0: + case RADEON_PACKET_TYPE0: dev_err(rdev->dev, "Packet0 not allowed!\n"); ret = -EINVAL; break; - case PACKET_TYPE2: + case RADEON_PACKET_TYPE2: idx += 1; break; - case PACKET_TYPE3: - pkt.opcode = CP_PACKET3_GET_OPCODE(ib->ptr[idx]); + case RADEON_PACKET_TYPE3: + pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]); ret = evergreen_vm_packet3_check(rdev, ib->ptr, &pkt); idx += pkt.count + 2; break; @@ -3627,88 +3492,79 @@ int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) int evergreen_dma_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) { u32 idx = 0; - u32 header, cmd, count, tiled, new_cmd, misc; + u32 header, cmd, count, sub_cmd; do { header = ib->ptr[idx]; cmd = GET_DMA_CMD(header); count = GET_DMA_COUNT(header); - tiled = GET_DMA_T(header); - new_cmd = GET_DMA_NEW(header); - misc = GET_DMA_MISC(header); + sub_cmd = GET_DMA_SUB_CMD(header); switch (cmd) { case DMA_PACKET_WRITE: - if (tiled) + switch (sub_cmd) { + /* tiled */ + case 8: idx += count + 7; - else + break; + /* linear */ + case 0: idx += count + 3; + break; + default: + DRM_ERROR("bad DMA_PACKET_WRITE [%6d] 0x%08x sub cmd is not 0 or 8\n", idx, ib->ptr[idx]); + return -EINVAL; + } break; case DMA_PACKET_COPY: - if (tiled) { - if (new_cmd) { - switch (misc) { - case 0: - /* L2T, frame to fields */ - idx += 10; - break; - case 1: - /* L2T, T2L partial */ - idx += 12; - break; - case 3: - /* L2T, broadcast */ - idx += 10; - break; - case 4: - /* L2T, T2L */ - idx += 9; - break; - case 5: - /* T2T partial */ - idx += 13; - break; - case 7: - /* L2T, broadcast */ - idx += 10; - break; - default: - DRM_ERROR("bad DMA_PACKET_COPY misc %u\n", misc); - return -EINVAL; - } - } else { - switch (misc) { - case 0: - idx += 9; - break; - default: - DRM_ERROR("bad DMA_PACKET_COPY misc %u\n", misc); - return -EINVAL; - } - } - } else { - if (new_cmd) { - switch (misc) { - case 0: - /* L2L, byte */ - idx += 5; - break; - case 1: - /* L2L, partial */ - idx += 9; - break; - case 4: - /* L2L, dw, broadcast */ - idx += 7; - break; - default: - DRM_ERROR("bad DMA_PACKET_COPY misc %u\n", misc); - return -EINVAL; - } - } else { - /* L2L, dw */ - idx += 5; - } + switch (sub_cmd) { + /* Copy L2L, DW aligned */ + case 0x00: + idx += 5; + break; + /* Copy L2T/T2L */ + case 0x08: + idx += 9; + break; + /* Copy L2L, byte aligned */ + case 0x40: + idx += 5; + break; + /* Copy L2L, partial */ + case 0x41: + idx += 9; + break; + /* Copy L2L, DW aligned, broadcast */ + case 0x44: + idx += 7; + break; + /* Copy L2T Frame to Field */ + case 0x48: + idx += 10; + break; + /* Copy L2T/T2L, partial */ + case 0x49: + idx += 12; + break; + /* Copy L2T broadcast */ + case 0x4b: + idx += 10; + break; + /* Copy L2T/T2L (tile units) */ + case 0x4c: + idx += 9; + break; + /* Copy T2T, partial (tile units) */ + case 0x4d: + idx += 13; + break; + /* Copy L2T broadcast (tile units) */ + case 0x4f: + idx += 10; + break; + default: + DRM_ERROR("bad DMA_PACKET_COPY [%6d] 0x%08x invalid sub cmd\n", idx, ib->ptr[idx]); + return -EINVAL; } break; case DMA_PACKET_CONSTANT_FILL: diff --git a/sys/dev/pci/drm/radeon/evergreen_dma.c b/sys/dev/pci/drm/radeon/evergreen_dma.c new file mode 100644 index 00000000000..a28b426eb8a --- /dev/null +++ b/sys/dev/pci/drm/radeon/evergreen_dma.c @@ -0,0 +1,182 @@ +/* + * Copyright 2010 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "evergreend.h" + +u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev); + +/** + * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring + * + * @rdev: radeon_device pointer + * @fence: radeon fence object + * + * Add a DMA fence packet to the ring to write + * the fence seq number and DMA trap packet to generate + * an interrupt if needed (evergreen-SI). + */ +void evergreen_dma_fence_ring_emit(struct radeon_device *rdev, + struct radeon_fence *fence) +{ + struct radeon_ring *ring = &rdev->ring[fence->ring]; + u64 addr = rdev->fence_drv[fence->ring].gpu_addr; + /* write the fence */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0)); + radeon_ring_write(ring, addr & 0xfffffffc); + radeon_ring_write(ring, (upper_32_bits(addr) & 0xff)); + radeon_ring_write(ring, fence->seq); + /* generate an interrupt */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0)); + /* flush HDP */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0)); + radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); + radeon_ring_write(ring, 1); +} + +/** + * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine + * + * @rdev: radeon_device pointer + * @ib: IB object to schedule + * + * Schedule an IB in the DMA ring (evergreen). + */ +void evergreen_dma_ring_ib_execute(struct radeon_device *rdev, + struct radeon_ib *ib) +{ + struct radeon_ring *ring = &rdev->ring[ib->ring]; + + if (rdev->wb.enabled) { + u32 next_rptr = ring->wptr + 4; + while ((next_rptr & 7) != 5) + next_rptr++; + next_rptr += 3; + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1)); + radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); + radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff); + radeon_ring_write(ring, next_rptr); + } + + /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. + * Pad as necessary with NOPs. + */ + while ((ring->wptr & 7) != 5) + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0)); + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0)); + radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); + radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); + +} + +/** + * evergreen_copy_dma - copy pages using the DMA engine + * + * @rdev: radeon_device pointer + * @src_offset: src GPU address + * @dst_offset: dst GPU address + * @num_gpu_pages: number of GPU pages to xfer + * @fence: radeon fence object + * + * Copy GPU paging using the DMA engine (evergreen-cayman). + * Used by the radeon ttm implementation to move pages if + * registered as the asic copy callback. + */ +struct radeon_fence *evergreen_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, + uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv) +{ + struct radeon_fence *fence; + struct radeon_sync sync; + int ring_index = rdev->asic->copy.dma_ring_index; + struct radeon_ring *ring = &rdev->ring[ring_index]; + u32 size_in_dw, cur_size_in_dw; + int i, num_loops; + int r = 0; + + radeon_sync_create(&sync); + + size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; + num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff); + r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); + if (r) { + DRM_ERROR("radeon: moving bo (%d).\n", r); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); + + for (i = 0; i < num_loops; i++) { + cur_size_in_dw = size_in_dw; + if (cur_size_in_dw > 0xFFFFF) + cur_size_in_dw = 0xFFFFF; + size_in_dw -= cur_size_in_dw; + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw)); + radeon_ring_write(ring, dst_offset & 0xfffffffc); + radeon_ring_write(ring, src_offset & 0xfffffffc); + radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); + radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); + src_offset += cur_size_in_dw * 4; + dst_offset += cur_size_in_dw * 4; + } + + r = radeon_fence_emit(rdev, &fence, ring->idx); + if (r) { + radeon_ring_unlock_undo(rdev, ring); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_ring_unlock_commit(rdev, ring, false); + radeon_sync_free(rdev, &sync, fence); + + return fence; +} + +/** + * evergreen_dma_is_lockup - Check if the DMA engine is locked up + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Check if the async DMA engine is locked up. + * Returns true if the engine appears to be locked up, false if not. + */ +bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +{ + u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); + + if (!(reset_mask & RADEON_RESET_DMA)) { + radeon_ring_lockup_update(rdev, ring); + return false; + } + return radeon_ring_test_lockup(rdev, ring); +} + + diff --git a/sys/dev/pci/drm/radeon/evergreen_hdmi.c b/sys/dev/pci/drm/radeon/evergreen_hdmi.c index 4ff587b075d..025c30122af 100644 --- a/sys/dev/pci/drm/radeon/evergreen_hdmi.c +++ b/sys/dev/pci/drm/radeon/evergreen_hdmi.c @@ -1,4 +1,3 @@ -/* $OpenBSD: evergreen_hdmi.c,v 1.4 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -25,116 +24,191 @@ * Authors: Christian König * RafaÅ‚ MiÅ‚ecki */ +#include #include #include #include "radeon.h" #include "radeon_asic.h" +#include "radeon_audio.h" #include "evergreend.h" #include "atom.h" -/* - * update the N and CTS parameters for a given pixel clock rate - */ -static void evergreen_hdmi_update_ACR(struct drm_encoder *encoder, uint32_t clock) +/* enable the audio stream */ +void dce4_audio_enable(struct radeon_device *rdev, + struct r600_audio_pin *pin, + u8 enable_mask) +{ + u32 tmp = RREG32(AZ_HOT_PLUG_CONTROL); + + if (!pin) + return; + + if (enable_mask) { + tmp |= AUDIO_ENABLED; + if (enable_mask & 1) + tmp |= PIN0_AUDIO_ENABLED; + if (enable_mask & 2) + tmp |= PIN1_AUDIO_ENABLED; + if (enable_mask & 4) + tmp |= PIN2_AUDIO_ENABLED; + if (enable_mask & 8) + tmp |= PIN3_AUDIO_ENABLED; + } else { + tmp &= ~(AUDIO_ENABLED | + PIN0_AUDIO_ENABLED | + PIN1_AUDIO_ENABLED | + PIN2_AUDIO_ENABLED | + PIN3_AUDIO_ENABLED); + } + + WREG32(AZ_HOT_PLUG_CONTROL, tmp); +} + +void evergreen_hdmi_update_acr(struct drm_encoder *encoder, long offset, + const struct radeon_hdmi_acr *acr) { struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; - struct radeon_hdmi_acr acr = r600_hdmi_acr(clock); - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - uint32_t offset = dig->afmt->offset; + int bpc = 8; + + if (encoder->crtc) { + struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); + bpc = radeon_crtc->bpc; + } - WREG32(HDMI_ACR_32_0 + offset, HDMI_ACR_CTS_32(acr.cts_32khz)); - WREG32(HDMI_ACR_32_1 + offset, acr.n_32khz); + if (bpc > 8) + WREG32(HDMI_ACR_PACKET_CONTROL + offset, + HDMI_ACR_AUTO_SEND); /* allow hw to sent ACR packets when required */ + else + WREG32(HDMI_ACR_PACKET_CONTROL + offset, + HDMI_ACR_SOURCE | /* select SW CTS value */ + HDMI_ACR_AUTO_SEND); /* allow hw to sent ACR packets when required */ - WREG32(HDMI_ACR_44_0 + offset, HDMI_ACR_CTS_44(acr.cts_44_1khz)); - WREG32(HDMI_ACR_44_1 + offset, acr.n_44_1khz); + WREG32(HDMI_ACR_32_0 + offset, HDMI_ACR_CTS_32(acr->cts_32khz)); + WREG32(HDMI_ACR_32_1 + offset, acr->n_32khz); - WREG32(HDMI_ACR_48_0 + offset, HDMI_ACR_CTS_48(acr.cts_48khz)); - WREG32(HDMI_ACR_48_1 + offset, acr.n_48khz); + WREG32(HDMI_ACR_44_0 + offset, HDMI_ACR_CTS_44(acr->cts_44_1khz)); + WREG32(HDMI_ACR_44_1 + offset, acr->n_44_1khz); + + WREG32(HDMI_ACR_48_0 + offset, HDMI_ACR_CTS_48(acr->cts_48khz)); + WREG32(HDMI_ACR_48_1 + offset, acr->n_48khz); } -/* - * calculate the crc for a given info frame - */ -static void evergreen_hdmi_infoframe_checksum(uint8_t packetType, - uint8_t versionNumber, - uint8_t length, - uint8_t *frame) +void dce4_afmt_write_latency_fields(struct drm_encoder *encoder, + struct drm_connector *connector, struct drm_display_mode *mode) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + u32 tmp = 0; + + if (mode->flags & DRM_MODE_FLAG_INTERLACE) { + if (connector->latency_present[1]) + tmp = VIDEO_LIPSYNC(connector->video_latency[1]) | + AUDIO_LIPSYNC(connector->audio_latency[1]); + else + tmp = VIDEO_LIPSYNC(255) | AUDIO_LIPSYNC(255); + } else { + if (connector->latency_present[0]) + tmp = VIDEO_LIPSYNC(connector->video_latency[0]) | + AUDIO_LIPSYNC(connector->audio_latency[0]); + else + tmp = VIDEO_LIPSYNC(255) | AUDIO_LIPSYNC(255); + } + WREG32_ENDPOINT(0, AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_LIPSYNC, tmp); +} + +void dce4_afmt_hdmi_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + u32 tmp; + + /* program the speaker allocation */ + tmp = RREG32_ENDPOINT(0, AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER); + tmp &= ~(DP_CONNECTION | SPEAKER_ALLOCATION_MASK); + /* set HDMI mode */ + tmp |= HDMI_CONNECTION; + if (sad_count) + tmp |= SPEAKER_ALLOCATION(sadb[0]); + else + tmp |= SPEAKER_ALLOCATION(5); /* stereo */ + WREG32_ENDPOINT(0, AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER, tmp); +} + +void dce4_afmt_dp_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + u32 tmp; + + /* program the speaker allocation */ + tmp = RREG32_ENDPOINT(0, AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER); + tmp &= ~(HDMI_CONNECTION | SPEAKER_ALLOCATION_MASK); + /* set DP mode */ + tmp |= DP_CONNECTION; + if (sad_count) + tmp |= SPEAKER_ALLOCATION(sadb[0]); + else + tmp |= SPEAKER_ALLOCATION(5); /* stereo */ + WREG32_ENDPOINT(0, AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER, tmp); +} + +void evergreen_hdmi_write_sad_regs(struct drm_encoder *encoder, + struct cea_sad *sads, int sad_count) { int i; - frame[0] = packetType + versionNumber + length; - for (i = 1; i <= length; i++) - frame[0] += frame[i]; - frame[0] = 0x100 - frame[0]; + struct radeon_device *rdev = encoder->dev->dev_private; + static const u16 eld_reg_to_type[][2] = { + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR0, HDMI_AUDIO_CODING_TYPE_PCM }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR1, HDMI_AUDIO_CODING_TYPE_AC3 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR2, HDMI_AUDIO_CODING_TYPE_MPEG1 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR3, HDMI_AUDIO_CODING_TYPE_MP3 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR4, HDMI_AUDIO_CODING_TYPE_MPEG2 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR5, HDMI_AUDIO_CODING_TYPE_AAC_LC }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR6, HDMI_AUDIO_CODING_TYPE_DTS }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR7, HDMI_AUDIO_CODING_TYPE_ATRAC }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR9, HDMI_AUDIO_CODING_TYPE_EAC3 }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR10, HDMI_AUDIO_CODING_TYPE_DTS_HD }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR11, HDMI_AUDIO_CODING_TYPE_MLP }, + { AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR13, HDMI_AUDIO_CODING_TYPE_WMA_PRO }, + }; + + for (i = 0; i < ARRAY_SIZE(eld_reg_to_type); i++) { + u32 value = 0; + u8 stereo_freqs = 0; + int max_channels = -1; + int j; + + for (j = 0; j < sad_count; j++) { + struct cea_sad *sad = &sads[j]; + + if (sad->format == eld_reg_to_type[i][1]) { + if (sad->channels > max_channels) { + value = MAX_CHANNELS(sad->channels) | + DESCRIPTOR_BYTE_2(sad->byte2) | + SUPPORTED_FREQUENCIES(sad->freq); + max_channels = sad->channels; + } + + if (sad->format == HDMI_AUDIO_CODING_TYPE_PCM) + stereo_freqs |= sad->freq; + else + break; + } + } + + value |= SUPPORTED_FREQUENCIES_STEREO(stereo_freqs); + + WREG32_ENDPOINT(0, eld_reg_to_type[i][0], value); + } } /* - * build a HDMI Video Info Frame + * build a AVI Info Frame */ -static void evergreen_hdmi_videoinfoframe( - struct drm_encoder *encoder, - uint8_t color_format, - int active_information_present, - uint8_t active_format_aspect_ratio, - uint8_t scan_information, - uint8_t colorimetry, - uint8_t ex_colorimetry, - uint8_t quantization, - int ITC, - uint8_t picture_aspect_ratio, - uint8_t video_format_identification, - uint8_t pixel_repetition, - uint8_t non_uniform_picture_scaling, - uint8_t bar_info_data_valid, - uint16_t top_bar, - uint16_t bottom_bar, - uint16_t left_bar, - uint16_t right_bar -) +void evergreen_set_avi_packet(struct radeon_device *rdev, u32 offset, + unsigned char *buffer, size_t size) { - struct drm_device *dev = encoder->dev; - struct radeon_device *rdev = dev->dev_private; - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - uint32_t offset = dig->afmt->offset; - - uint8_t frame[14]; - - frame[0x0] = 0; - frame[0x1] = - (scan_information & 0x3) | - ((bar_info_data_valid & 0x3) << 2) | - ((active_information_present & 0x1) << 4) | - ((color_format & 0x3) << 5); - frame[0x2] = - (active_format_aspect_ratio & 0xF) | - ((picture_aspect_ratio & 0x3) << 4) | - ((colorimetry & 0x3) << 6); - frame[0x3] = - (non_uniform_picture_scaling & 0x3) | - ((quantization & 0x3) << 2) | - ((ex_colorimetry & 0x7) << 4) | - ((ITC & 0x1) << 7); - frame[0x4] = (video_format_identification & 0x7F); - frame[0x5] = (pixel_repetition & 0xF); - frame[0x6] = (top_bar & 0xFF); - frame[0x7] = (top_bar >> 8); - frame[0x8] = (bottom_bar & 0xFF); - frame[0x9] = (bottom_bar >> 8); - frame[0xA] = (left_bar & 0xFF); - frame[0xB] = (left_bar >> 8); - frame[0xC] = (right_bar & 0xFF); - frame[0xD] = (right_bar >> 8); - - evergreen_hdmi_infoframe_checksum(0x82, 0x02, 0x0D, frame); - /* Our header values (type, version, length) should be alright, Intel - * is using the same. Checksum function also seems to be OK, it works - * fine for audio infoframe. However calculated value is always lower - * by 2 in comparison to fglrx. It breaks displaying anything in case - * of TVs that strictly check the checksum. Hack it manually here to - * workaround this issue. */ - frame[0x0] += 2; + uint8_t *frame = buffer + 3; WREG32(AFMT_AVI_INFO0 + offset, frame[0x0] | (frame[0x1] << 8) | (frame[0x2] << 16) | (frame[0x3] << 24)); @@ -143,75 +217,275 @@ static void evergreen_hdmi_videoinfoframe( WREG32(AFMT_AVI_INFO2 + offset, frame[0x8] | (frame[0x9] << 8) | (frame[0xA] << 16) | (frame[0xB] << 24)); WREG32(AFMT_AVI_INFO3 + offset, - frame[0xC] | (frame[0xD] << 8)); + frame[0xC] | (frame[0xD] << 8) | (buffer[1] << 24)); + + WREG32_P(HDMI_INFOFRAME_CONTROL1 + offset, + HDMI_AVI_INFO_LINE(2), /* anything other than 0 */ + ~HDMI_AVI_INFO_LINE_MASK); } -/* - * update the info frames with the data from the current display mode - */ -void evergreen_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode) +void dce4_hdmi_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock) +{ + unsigned int max_ratio = clock / 24000; + u32 dto_phase; + u32 wallclock_ratio; + u32 value; + + if (max_ratio >= 8) { + dto_phase = 192 * 1000; + wallclock_ratio = 3; + } else if (max_ratio >= 4) { + dto_phase = 96 * 1000; + wallclock_ratio = 2; + } else if (max_ratio >= 2) { + dto_phase = 48 * 1000; + wallclock_ratio = 1; + } else { + dto_phase = 24 * 1000; + wallclock_ratio = 0; + } + + value = RREG32(DCCG_AUDIO_DTO0_CNTL) & ~DCCG_AUDIO_DTO_WALLCLOCK_RATIO_MASK; + value |= DCCG_AUDIO_DTO_WALLCLOCK_RATIO(wallclock_ratio); + value &= ~DCCG_AUDIO_DTO1_USE_512FBR_DTO; + WREG32(DCCG_AUDIO_DTO0_CNTL, value); + + /* Two dtos; generally use dto0 for HDMI */ + value = 0; + + if (crtc) + value |= DCCG_AUDIO_DTO0_SOURCE_SEL(crtc->crtc_id); + + WREG32(DCCG_AUDIO_DTO_SOURCE, value); + + /* Express [24MHz / target pixel clock] as an exact rational + * number (coefficient of two integer numbers. DCCG_AUDIO_DTOx_PHASE + * is the numerator, DCCG_AUDIO_DTOx_MODULE is the denominator + */ + WREG32(DCCG_AUDIO_DTO0_PHASE, dto_phase); + WREG32(DCCG_AUDIO_DTO0_MODULE, clock); +} + +void dce4_dp_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock) +{ + u32 value; + + value = RREG32(DCCG_AUDIO_DTO1_CNTL) & ~DCCG_AUDIO_DTO_WALLCLOCK_RATIO_MASK; + value |= DCCG_AUDIO_DTO1_USE_512FBR_DTO; + WREG32(DCCG_AUDIO_DTO1_CNTL, value); + + /* Two dtos; generally use dto1 for DP */ + value = 0; + value |= DCCG_AUDIO_DTO_SEL; + + if (crtc) + value |= DCCG_AUDIO_DTO0_SOURCE_SEL(crtc->crtc_id); + + WREG32(DCCG_AUDIO_DTO_SOURCE, value); + + /* Express [24MHz / target pixel clock] as an exact rational + * number (coefficient of two integer numbers. DCCG_AUDIO_DTOx_PHASE + * is the numerator, DCCG_AUDIO_DTOx_MODULE is the denominator + */ + if (ASIC_IS_DCE41(rdev)) { + unsigned int div = (RREG32(DCE41_DENTIST_DISPCLK_CNTL) & + DENTIST_DPREFCLK_WDIVIDER_MASK) >> + DENTIST_DPREFCLK_WDIVIDER_SHIFT; + div = radeon_audio_decode_dfs_div(div); + + if (div) + clock = 100 * clock / div; + } + + WREG32(DCCG_AUDIO_DTO1_PHASE, 24000); + WREG32(DCCG_AUDIO_DTO1_MODULE, clock); +} + +void dce4_set_vbi_packet(struct drm_encoder *encoder, u32 offset) { struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - uint32_t offset; - if (!dig || !dig->afmt) - return; + WREG32(HDMI_VBI_PACKET_CONTROL + offset, + HDMI_NULL_SEND | /* send null packets when required */ + HDMI_GC_SEND | /* send general control packets */ + HDMI_GC_CONT); /* send general control packets every frame */ +} - /* Silent, r600_hdmi_enable will raise WARN for us */ - if (!dig->afmt->enabled) - return; - offset = dig->afmt->offset; +void dce4_hdmi_set_color_depth(struct drm_encoder *encoder, u32 offset, int bpc) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; +#ifdef DRMDEBUG + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); +#endif + uint32_t val; - r600_audio_set_clock(encoder, mode->clock); + val = RREG32(HDMI_CONTROL + offset); + val &= ~HDMI_DEEP_COLOR_ENABLE; + val &= ~HDMI_DEEP_COLOR_DEPTH_MASK; - WREG32(HDMI_VBI_PACKET_CONTROL + offset, - HDMI_NULL_SEND); /* send null packets when required */ + switch (bpc) { + case 0: + case 6: + case 8: + case 16: + default: + DRM_DEBUG("%s: Disabling hdmi deep color for %d bpc.\n", + connector->name, bpc); + break; + case 10: + val |= HDMI_DEEP_COLOR_ENABLE; + val |= HDMI_DEEP_COLOR_DEPTH(HDMI_30BIT_DEEP_COLOR); + DRM_DEBUG("%s: Enabling hdmi deep color 30 for 10 bpc.\n", + connector->name); + break; + case 12: + val |= HDMI_DEEP_COLOR_ENABLE; + val |= HDMI_DEEP_COLOR_DEPTH(HDMI_36BIT_DEEP_COLOR); + DRM_DEBUG("%s: Enabling hdmi deep color 36 for 12 bpc.\n", + connector->name); + break; + } + + WREG32(HDMI_CONTROL + offset, val); +} + +void dce4_set_audio_packet(struct drm_encoder *encoder, u32 offset) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + + WREG32(AFMT_INFOFRAME_CONTROL0 + offset, + AFMT_AUDIO_INFO_UPDATE); /* required for audio info values to be updated */ - WREG32(AFMT_AUDIO_CRC_CONTROL + offset, 0x1000); + WREG32(AFMT_60958_0 + offset, + AFMT_60958_CS_CHANNEL_NUMBER_L(1)); + + WREG32(AFMT_60958_1 + offset, + AFMT_60958_CS_CHANNEL_NUMBER_R(2)); + + WREG32(AFMT_60958_2 + offset, + AFMT_60958_CS_CHANNEL_NUMBER_2(3) | + AFMT_60958_CS_CHANNEL_NUMBER_3(4) | + AFMT_60958_CS_CHANNEL_NUMBER_4(5) | + AFMT_60958_CS_CHANNEL_NUMBER_5(6) | + AFMT_60958_CS_CHANNEL_NUMBER_6(7) | + AFMT_60958_CS_CHANNEL_NUMBER_7(8)); + + WREG32(AFMT_AUDIO_PACKET_CONTROL2 + offset, + AFMT_AUDIO_CHANNEL_ENABLE(0xff)); WREG32(HDMI_AUDIO_PACKET_CONTROL + offset, HDMI_AUDIO_DELAY_EN(1) | /* set the default audio delay */ HDMI_AUDIO_PACKETS_PER_LINE(3)); /* should be suffient for all audio modes and small enough for all hblanks */ - WREG32(AFMT_AUDIO_PACKET_CONTROL + offset, - AFMT_AUDIO_SAMPLE_SEND | /* send audio packets */ - AFMT_60958_CS_UPDATE); /* allow 60958 channel status fields to be updated */ + /* allow 60958 channel status and send audio packets fields to be updated */ + WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + offset, + AFMT_RESET_FIFO_WHEN_AUDIO_DIS | AFMT_60958_CS_UPDATE); +} - WREG32(HDMI_ACR_PACKET_CONTROL + offset, - HDMI_ACR_AUTO_SEND | /* allow hw to sent ACR packets when required */ - HDMI_ACR_SOURCE); /* select SW CTS value */ - WREG32(HDMI_VBI_PACKET_CONTROL + offset, - HDMI_NULL_SEND | /* send null packets when required */ - HDMI_GC_SEND | /* send general control packets */ - HDMI_GC_CONT); /* send general control packets every frame */ +void dce4_set_mute(struct drm_encoder *encoder, u32 offset, bool mute) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; - WREG32(HDMI_INFOFRAME_CONTROL0 + offset, - HDMI_AVI_INFO_SEND | /* enable AVI info frames */ - HDMI_AVI_INFO_CONT | /* send AVI info frames every frame/field */ - HDMI_AUDIO_INFO_SEND | /* enable audio info frames (frames won't be set until audio is enabled) */ - HDMI_AUDIO_INFO_CONT); /* required for audio info values to be updated */ + if (mute) + WREG32_OR(HDMI_GC + offset, HDMI_GC_AVMUTE); + else + WREG32_AND(HDMI_GC + offset, ~HDMI_GC_AVMUTE); +} - WREG32(AFMT_INFOFRAME_CONTROL0 + offset, - AFMT_AUDIO_INFO_UPDATE); /* required for audio info values to be updated */ +void evergreen_hdmi_enable(struct drm_encoder *encoder, bool enable) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + + if (!dig || !dig->afmt) + return; + + if (enable) { + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + + if (connector && drm_detect_monitor_audio(radeon_connector_edid(connector))) { + WREG32(HDMI_INFOFRAME_CONTROL0 + dig->afmt->offset, + HDMI_AVI_INFO_SEND | /* enable AVI info frames */ + HDMI_AVI_INFO_CONT | /* required for audio info values to be updated */ + HDMI_AUDIO_INFO_SEND | /* enable audio info frames (frames won't be set until audio is enabled) */ + HDMI_AUDIO_INFO_CONT); /* required for audio info values to be updated */ + WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + dig->afmt->offset, + AFMT_AUDIO_SAMPLE_SEND); + } else { + WREG32(HDMI_INFOFRAME_CONTROL0 + dig->afmt->offset, + HDMI_AVI_INFO_SEND | /* enable AVI info frames */ + HDMI_AVI_INFO_CONT); /* required for audio info values to be updated */ + WREG32_AND(AFMT_AUDIO_PACKET_CONTROL + dig->afmt->offset, + ~AFMT_AUDIO_SAMPLE_SEND); + } + } else { + WREG32_AND(AFMT_AUDIO_PACKET_CONTROL + dig->afmt->offset, + ~AFMT_AUDIO_SAMPLE_SEND); + WREG32(HDMI_INFOFRAME_CONTROL0 + dig->afmt->offset, 0); + } + + dig->afmt->enabled = enable; + + DRM_DEBUG("%sabling HDMI interface @ 0x%04X for encoder 0x%x\n", + enable ? "En" : "Dis", dig->afmt->offset, radeon_encoder->encoder_id); +} + +void evergreen_dp_enable(struct drm_encoder *encoder, bool enable) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + + if (!dig || !dig->afmt) + return; + + if (enable && connector && + drm_detect_monitor_audio(radeon_connector_edid(connector))) { + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + struct radeon_connector_atom_dig *dig_connector; + uint32_t val; + + WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + dig->afmt->offset, + AFMT_AUDIO_SAMPLE_SEND); + + WREG32(EVERGREEN_DP_SEC_TIMESTAMP + dig->afmt->offset, + EVERGREEN_DP_SEC_TIMESTAMP_MODE(1)); - WREG32(HDMI_INFOFRAME_CONTROL1 + offset, - HDMI_AVI_INFO_LINE(2) | /* anything other than 0 */ - HDMI_AUDIO_INFO_LINE(2)); /* anything other than 0 */ + if (!ASIC_IS_DCE6(rdev) && radeon_connector->con_priv) { + dig_connector = radeon_connector->con_priv; + val = RREG32(EVERGREEN_DP_SEC_AUD_N + dig->afmt->offset); + val &= ~EVERGREEN_DP_SEC_N_BASE_MULTIPLE(0xf); - WREG32(HDMI_GC + offset, 0); /* unset HDMI_GC_AVMUTE */ + if (dig_connector->dp_clock == 162000) + val |= EVERGREEN_DP_SEC_N_BASE_MULTIPLE(3); + else + val |= EVERGREEN_DP_SEC_N_BASE_MULTIPLE(5); - evergreen_hdmi_videoinfoframe(encoder, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0); + WREG32(EVERGREEN_DP_SEC_AUD_N + dig->afmt->offset, val); + } - evergreen_hdmi_update_ACR(encoder, mode->clock); + WREG32(EVERGREEN_DP_SEC_CNTL + dig->afmt->offset, + EVERGREEN_DP_SEC_ASP_ENABLE | /* Audio packet transmission */ + EVERGREEN_DP_SEC_ATP_ENABLE | /* Audio timestamp packet transmission */ + EVERGREEN_DP_SEC_AIP_ENABLE | /* Audio infoframe packet transmission */ + EVERGREEN_DP_SEC_STREAM_ENABLE); /* Master enable for secondary stream engine */ + } else { + WREG32(EVERGREEN_DP_SEC_CNTL + dig->afmt->offset, 0); + WREG32_AND(AFMT_AUDIO_PACKET_CONTROL + dig->afmt->offset, + ~AFMT_AUDIO_SAMPLE_SEND); + } - /* it's unknown what these bits do excatly, but it's indeed quite useful for debugging */ - WREG32(AFMT_RAMP_CONTROL0 + offset, 0x00FFFFFF); - WREG32(AFMT_RAMP_CONTROL1 + offset, 0x007FFFFF); - WREG32(AFMT_RAMP_CONTROL2 + offset, 0x00000001); - WREG32(AFMT_RAMP_CONTROL3 + offset, 0x00000001); + dig->afmt->enabled = enable; } diff --git a/sys/dev/pci/drm/radeon/evergreen_reg.h b/sys/dev/pci/drm/radeon/evergreen_reg.h index 584417f1f52..b436badf9ef 100644 --- a/sys/dev/pci/drm/radeon/evergreen_reg.h +++ b/sys/dev/pci/drm/radeon/evergreen_reg.h @@ -1,4 +1,3 @@ -/* $OpenBSD: evergreen_reg.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * @@ -25,7 +24,17 @@ #ifndef __EVERGREEN_REG_H__ #define __EVERGREEN_REG_H__ +/* trinity */ +#define TN_SMC_IND_INDEX_0 0x200 +#define TN_SMC_IND_DATA_0 0x204 + /* evergreen */ +#define EVERGREEN_PIF_PHY0_INDEX 0x8 +#define EVERGREEN_PIF_PHY0_DATA 0xc +#define EVERGREEN_PIF_PHY1_INDEX 0x10 +#define EVERGREEN_PIF_PHY1_DATA 0x14 +#define EVERGREEN_MM_INDEX_HI 0x18 + #define EVERGREEN_VGA_MEMORY_BASE_ADDRESS 0x310 #define EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH 0x324 #define EVERGREEN_D3VGA_CONTROL 0x3e0 @@ -41,6 +50,9 @@ #define EVERGREEN_AUDIO_PLL1_DIV 0x5b4 #define EVERGREEN_AUDIO_PLL1_UNK 0x5bc +#define EVERGREEN_CG_IND_ADDR 0x8f8 +#define EVERGREEN_CG_IND_DATA 0x8fc + #define EVERGREEN_AUDIO_ENABLE 0x5e78 #define EVERGREEN_AUDIO_VENDOR_ID 0x5ec0 @@ -104,6 +116,8 @@ # define EVERGREEN_GRPH_ARRAY_LINEAR_ALIGNED 1 # define EVERGREEN_GRPH_ARRAY_1D_TILED_THIN1 2 # define EVERGREEN_GRPH_ARRAY_2D_TILED_THIN1 4 +#define EVERGREEN_GRPH_LUT_10BIT_BYPASS_CONTROL 0x6808 +# define EVERGREEN_LUT_10BIT_BYPASS_EN (1 << 8) #define EVERGREEN_GRPH_SWAP_CONTROL 0x680c # define EVERGREEN_GRPH_ENDIAN_SWAP(x) (((x) & 0x3) << 0) # define EVERGREEN_GRPH_ENDIAN_NONE 0 @@ -224,7 +238,7 @@ #define EVERGREEN_CRTC_STATUS 0x6e8c # define EVERGREEN_CRTC_V_BLANK (1 << 0) #define EVERGREEN_CRTC_STATUS_POSITION 0x6e90 -#define EVERGREEN_MASTER_UPDATE_MODE 0x6ef8 +#define EVERGREEN_CRTC_STATUS_HV_COUNT 0x6ea0 #define EVERGREEN_CRTC_UPDATE_LOCK 0x6ed4 #define EVERGREEN_MASTER_UPDATE_LOCK 0x6ef4 #define EVERGREEN_MASTER_UPDATE_MODE 0x6ef8 @@ -236,5 +250,66 @@ /* HDMI blocks at 0x7030, 0x7c30, 0x10830, 0x11430, 0x12030, 0x12c30 */ #define EVERGREEN_HDMI_BASE 0x7030 +/*DIG block*/ +#define NI_DIG0_REGISTER_OFFSET (0x7000 - 0x7000) +#define NI_DIG1_REGISTER_OFFSET (0x7C00 - 0x7000) +#define NI_DIG2_REGISTER_OFFSET (0x10800 - 0x7000) +#define NI_DIG3_REGISTER_OFFSET (0x11400 - 0x7000) +#define NI_DIG4_REGISTER_OFFSET (0x12000 - 0x7000) +#define NI_DIG5_REGISTER_OFFSET (0x12C00 - 0x7000) + + +#define NI_DIG_FE_CNTL 0x7000 +# define NI_DIG_FE_CNTL_SOURCE_SELECT(x) ((x) & 0x3) +# define NI_DIG_FE_CNTL_SYMCLK_FE_ON (1<<24) + + +#define NI_DIG_BE_CNTL 0x7140 +# define NI_DIG_BE_CNTL_FE_SOURCE_SELECT(x) (((x) >> 8 ) & 0x3F) +# define NI_DIG_FE_CNTL_MODE(x) (((x) >> 16) & 0x7 ) + +#define NI_DIG_BE_EN_CNTL 0x7144 +# define NI_DIG_BE_EN_CNTL_ENABLE (1 << 0) +# define NI_DIG_BE_EN_CNTL_SYMBCLK_ON (1 << 8) +# define NI_DIG_BE_DPSST 0 + +/* Display Port block */ +#define EVERGREEN_DP0_REGISTER_OFFSET (0x730C - 0x730C) +#define EVERGREEN_DP1_REGISTER_OFFSET (0x7F0C - 0x730C) +#define EVERGREEN_DP2_REGISTER_OFFSET (0x10B0C - 0x730C) +#define EVERGREEN_DP3_REGISTER_OFFSET (0x1170C - 0x730C) +#define EVERGREEN_DP4_REGISTER_OFFSET (0x1230C - 0x730C) +#define EVERGREEN_DP5_REGISTER_OFFSET (0x12F0C - 0x730C) + + +#define EVERGREEN_DP_VID_STREAM_CNTL 0x730C +# define EVERGREEN_DP_VID_STREAM_CNTL_ENABLE (1 << 0) +# define EVERGREEN_DP_VID_STREAM_STATUS (1 <<16) +#define EVERGREEN_DP_STEER_FIFO 0x7310 +# define EVERGREEN_DP_STEER_FIFO_RESET (1 << 0) +#define EVERGREEN_DP_SEC_CNTL 0x7280 +# define EVERGREEN_DP_SEC_STREAM_ENABLE (1 << 0) +# define EVERGREEN_DP_SEC_ASP_ENABLE (1 << 4) +# define EVERGREEN_DP_SEC_ATP_ENABLE (1 << 8) +# define EVERGREEN_DP_SEC_AIP_ENABLE (1 << 12) +# define EVERGREEN_DP_SEC_GSP_ENABLE (1 << 20) +# define EVERGREEN_DP_SEC_AVI_ENABLE (1 << 24) +# define EVERGREEN_DP_SEC_MPG_ENABLE (1 << 28) +#define EVERGREEN_DP_SEC_TIMESTAMP 0x72a4 +# define EVERGREEN_DP_SEC_TIMESTAMP_MODE(x) (((x) & 0x3) << 0) +#define EVERGREEN_DP_SEC_AUD_N 0x7294 +# define EVERGREEN_DP_SEC_N_BASE_MULTIPLE(x) (((x) & 0xf) << 24) +# define EVERGREEN_DP_SEC_SS_EN (1 << 28) + +/*DCIO_UNIPHY block*/ +#define NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1 (0x6600 -0x6600) +#define NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1 (0x6640 -0x6600) +#define NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1 (0x6680 - 0x6600) +#define NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1 (0x66C0 - 0x6600) +#define NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1 (0x6700 - 0x6600) +#define NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1 (0x6740 - 0x6600) + +#define NI_DCIO_UNIPHY0_PLL_CONTROL1 0x6618 +# define NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE (1 << 0) #endif diff --git a/sys/dev/pci/drm/radeon/evergreen_smc.h b/sys/dev/pci/drm/radeon/evergreen_smc.h new file mode 100644 index 00000000000..3a03ba37d04 --- /dev/null +++ b/sys/dev/pci/drm/radeon/evergreen_smc.h @@ -0,0 +1,67 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __EVERGREEN_SMC_H__ +#define __EVERGREEN_SMC_H__ + +#include "rv770_smc.h" + +#pragma pack(push, 1) + +#define SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE 16 + +struct SMC_Evergreen_MCRegisterAddress +{ + uint16_t s0; + uint16_t s1; +}; + +typedef struct SMC_Evergreen_MCRegisterAddress SMC_Evergreen_MCRegisterAddress; + + +struct SMC_Evergreen_MCRegisterSet +{ + uint32_t value[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE]; +}; + +typedef struct SMC_Evergreen_MCRegisterSet SMC_Evergreen_MCRegisterSet; + +struct SMC_Evergreen_MCRegisters +{ + uint8_t last; + uint8_t reserved[3]; + SMC_Evergreen_MCRegisterAddress address[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE]; + SMC_Evergreen_MCRegisterSet data[5]; +}; + +typedef struct SMC_Evergreen_MCRegisters SMC_Evergreen_MCRegisters; + +#define EVERGREEN_SMC_FIRMWARE_HEADER_LOCATION 0x100 + +#define EVERGREEN_SMC_FIRMWARE_HEADER_softRegisters 0x8 +#define EVERGREEN_SMC_FIRMWARE_HEADER_stateTable 0xC +#define EVERGREEN_SMC_FIRMWARE_HEADER_mcRegisterTable 0x20 + + +#pragma pack(pop) + +#endif diff --git a/sys/dev/pci/drm/radeon/evergreend.h b/sys/dev/pci/drm/radeon/evergreend.h index 32e4a326b09..13b6029d65c 100644 --- a/sys/dev/pci/drm/radeon/evergreend.h +++ b/sys/dev/pci/drm/radeon/evergreend.h @@ -1,4 +1,3 @@ -/* $OpenBSD: evergreend.h,v 1.5 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * @@ -49,11 +48,367 @@ #define SUMO_GB_ADDR_CONFIG_GOLDEN 0x02010002 #define SUMO2_GB_ADDR_CONFIG_GOLDEN 0x02010002 +/* pm registers */ +#define SMC_MSG 0x20c +#define HOST_SMC_MSG(x) ((x) << 0) +#define HOST_SMC_MSG_MASK (0xff << 0) +#define HOST_SMC_MSG_SHIFT 0 +#define HOST_SMC_RESP(x) ((x) << 8) +#define HOST_SMC_RESP_MASK (0xff << 8) +#define HOST_SMC_RESP_SHIFT 8 +#define SMC_HOST_MSG(x) ((x) << 16) +#define SMC_HOST_MSG_MASK (0xff << 16) +#define SMC_HOST_MSG_SHIFT 16 +#define SMC_HOST_RESP(x) ((x) << 24) +#define SMC_HOST_RESP_MASK (0xff << 24) +#define SMC_HOST_RESP_SHIFT 24 + +#define DCCG_DISP_SLOW_SELECT_REG 0x4fc +#define DCCG_DISP1_SLOW_SELECT(x) ((x) << 0) +#define DCCG_DISP1_SLOW_SELECT_MASK (7 << 0) +#define DCCG_DISP1_SLOW_SELECT_SHIFT 0 +#define DCCG_DISP2_SLOW_SELECT(x) ((x) << 4) +#define DCCG_DISP2_SLOW_SELECT_MASK (7 << 4) +#define DCCG_DISP2_SLOW_SELECT_SHIFT 4 + +#define CG_SPLL_FUNC_CNTL 0x600 +#define SPLL_RESET (1 << 0) +#define SPLL_SLEEP (1 << 1) +#define SPLL_BYPASS_EN (1 << 3) +#define SPLL_REF_DIV(x) ((x) << 4) +#define SPLL_REF_DIV_MASK (0x3f << 4) +#define SPLL_PDIV_A(x) ((x) << 20) +#define SPLL_PDIV_A_MASK (0x7f << 20) +#define CG_SPLL_FUNC_CNTL_2 0x604 +#define SCLK_MUX_SEL(x) ((x) << 0) +#define SCLK_MUX_SEL_MASK (0x1ff << 0) +#define SCLK_MUX_UPDATE (1 << 26) +#define CG_SPLL_FUNC_CNTL_3 0x608 +#define SPLL_FB_DIV(x) ((x) << 0) +#define SPLL_FB_DIV_MASK (0x3ffffff << 0) +#define SPLL_DITHEN (1 << 28) +#define CG_SPLL_STATUS 0x60c +#define SPLL_CHG_STATUS (1 << 1) + +#define MPLL_CNTL_MODE 0x61c +# define MPLL_MCLK_SEL (1 << 11) +# define SS_SSEN (1 << 24) +# define SS_DSMODE_EN (1 << 25) + +#define MPLL_AD_FUNC_CNTL 0x624 +#define CLKF(x) ((x) << 0) +#define CLKF_MASK (0x7f << 0) +#define CLKR(x) ((x) << 7) +#define CLKR_MASK (0x1f << 7) +#define CLKFRAC(x) ((x) << 12) +#define CLKFRAC_MASK (0x1f << 12) +#define YCLK_POST_DIV(x) ((x) << 17) +#define YCLK_POST_DIV_MASK (3 << 17) +#define IBIAS(x) ((x) << 20) +#define IBIAS_MASK (0x3ff << 20) +#define RESET (1 << 30) +#define PDNB (1 << 31) +#define MPLL_AD_FUNC_CNTL_2 0x628 +#define BYPASS (1 << 19) +#define BIAS_GEN_PDNB (1 << 24) +#define RESET_EN (1 << 25) +#define VCO_MODE (1 << 29) +#define MPLL_DQ_FUNC_CNTL 0x62c +#define MPLL_DQ_FUNC_CNTL_2 0x630 + +#define GENERAL_PWRMGT 0x63c +# define GLOBAL_PWRMGT_EN (1 << 0) +# define STATIC_PM_EN (1 << 1) +# define THERMAL_PROTECTION_DIS (1 << 2) +# define THERMAL_PROTECTION_TYPE (1 << 3) +# define ENABLE_GEN2PCIE (1 << 4) +# define ENABLE_GEN2XSP (1 << 5) +# define SW_SMIO_INDEX(x) ((x) << 6) +# define SW_SMIO_INDEX_MASK (3 << 6) +# define SW_SMIO_INDEX_SHIFT 6 +# define LOW_VOLT_D2_ACPI (1 << 8) +# define LOW_VOLT_D3_ACPI (1 << 9) +# define VOLT_PWRMGT_EN (1 << 10) +# define BACKBIAS_PAD_EN (1 << 18) +# define BACKBIAS_VALUE (1 << 19) +# define DYN_SPREAD_SPECTRUM_EN (1 << 23) +# define AC_DC_SW (1 << 24) + +#define SCLK_PWRMGT_CNTL 0x644 +# define SCLK_PWRMGT_OFF (1 << 0) +# define SCLK_LOW_D1 (1 << 1) +# define FIR_RESET (1 << 4) +# define FIR_FORCE_TREND_SEL (1 << 5) +# define FIR_TREND_MODE (1 << 6) +# define DYN_GFX_CLK_OFF_EN (1 << 7) +# define GFX_CLK_FORCE_ON (1 << 8) +# define GFX_CLK_REQUEST_OFF (1 << 9) +# define GFX_CLK_FORCE_OFF (1 << 10) +# define GFX_CLK_OFF_ACPI_D1 (1 << 11) +# define GFX_CLK_OFF_ACPI_D2 (1 << 12) +# define GFX_CLK_OFF_ACPI_D3 (1 << 13) +# define DYN_LIGHT_SLEEP_EN (1 << 14) +#define MCLK_PWRMGT_CNTL 0x648 +# define DLL_SPEED(x) ((x) << 0) +# define DLL_SPEED_MASK (0x1f << 0) +# define MPLL_PWRMGT_OFF (1 << 5) +# define DLL_READY (1 << 6) +# define MC_INT_CNTL (1 << 7) +# define MRDCKA0_PDNB (1 << 8) +# define MRDCKA1_PDNB (1 << 9) +# define MRDCKB0_PDNB (1 << 10) +# define MRDCKB1_PDNB (1 << 11) +# define MRDCKC0_PDNB (1 << 12) +# define MRDCKC1_PDNB (1 << 13) +# define MRDCKD0_PDNB (1 << 14) +# define MRDCKD1_PDNB (1 << 15) +# define MRDCKA0_RESET (1 << 16) +# define MRDCKA1_RESET (1 << 17) +# define MRDCKB0_RESET (1 << 18) +# define MRDCKB1_RESET (1 << 19) +# define MRDCKC0_RESET (1 << 20) +# define MRDCKC1_RESET (1 << 21) +# define MRDCKD0_RESET (1 << 22) +# define MRDCKD1_RESET (1 << 23) +# define DLL_READY_READ (1 << 24) +# define USE_DISPLAY_GAP (1 << 25) +# define USE_DISPLAY_URGENT_NORMAL (1 << 26) +# define MPLL_TURNOFF_D2 (1 << 28) +#define DLL_CNTL 0x64c +# define MRDCKA0_BYPASS (1 << 24) +# define MRDCKA1_BYPASS (1 << 25) +# define MRDCKB0_BYPASS (1 << 26) +# define MRDCKB1_BYPASS (1 << 27) +# define MRDCKC0_BYPASS (1 << 28) +# define MRDCKC1_BYPASS (1 << 29) +# define MRDCKD0_BYPASS (1 << 30) +# define MRDCKD1_BYPASS (1 << 31) + +#define CG_AT 0x6d4 +# define CG_R(x) ((x) << 0) +# define CG_R_MASK (0xffff << 0) +# define CG_L(x) ((x) << 16) +# define CG_L_MASK (0xffff << 16) + +#define CG_DISPLAY_GAP_CNTL 0x714 +# define DISP1_GAP(x) ((x) << 0) +# define DISP1_GAP_MASK (3 << 0) +# define DISP2_GAP(x) ((x) << 2) +# define DISP2_GAP_MASK (3 << 2) +# define VBI_TIMER_COUNT(x) ((x) << 4) +# define VBI_TIMER_COUNT_MASK (0x3fff << 4) +# define VBI_TIMER_UNIT(x) ((x) << 20) +# define VBI_TIMER_UNIT_MASK (7 << 20) +# define DISP1_GAP_MCHG(x) ((x) << 24) +# define DISP1_GAP_MCHG_MASK (3 << 24) +# define DISP2_GAP_MCHG(x) ((x) << 26) +# define DISP2_GAP_MCHG_MASK (3 << 26) + +#define CG_BIF_REQ_AND_RSP 0x7f4 +#define CG_CLIENT_REQ(x) ((x) << 0) +#define CG_CLIENT_REQ_MASK (0xff << 0) +#define CG_CLIENT_REQ_SHIFT 0 +#define CG_CLIENT_RESP(x) ((x) << 8) +#define CG_CLIENT_RESP_MASK (0xff << 8) +#define CG_CLIENT_RESP_SHIFT 8 +#define CLIENT_CG_REQ(x) ((x) << 16) +#define CLIENT_CG_REQ_MASK (0xff << 16) +#define CLIENT_CG_REQ_SHIFT 16 +#define CLIENT_CG_RESP(x) ((x) << 24) +#define CLIENT_CG_RESP_MASK (0xff << 24) +#define CLIENT_CG_RESP_SHIFT 24 + +#define CG_SPLL_SPREAD_SPECTRUM 0x790 +#define SSEN (1 << 0) +#define CG_SPLL_SPREAD_SPECTRUM_2 0x794 + +#define MPLL_SS1 0x85c +#define CLKV(x) ((x) << 0) +#define CLKV_MASK (0x3ffffff << 0) +#define MPLL_SS2 0x860 +#define CLKS(x) ((x) << 0) +#define CLKS_MASK (0xfff << 0) + +#define CG_IND_ADDR 0x8f8 +#define CG_IND_DATA 0x8fc +/* CGIND regs */ +#define CG_CGTT_LOCAL_0 0x00 +#define CG_CGTT_LOCAL_1 0x01 +#define CG_CGTT_LOCAL_2 0x02 +#define CG_CGTT_LOCAL_3 0x03 +#define CG_CGLS_TILE_0 0x20 +#define CG_CGLS_TILE_1 0x21 +#define CG_CGLS_TILE_2 0x22 +#define CG_CGLS_TILE_3 0x23 +#define CG_CGLS_TILE_4 0x24 +#define CG_CGLS_TILE_5 0x25 +#define CG_CGLS_TILE_6 0x26 +#define CG_CGLS_TILE_7 0x27 +#define CG_CGLS_TILE_8 0x28 +#define CG_CGLS_TILE_9 0x29 +#define CG_CGLS_TILE_10 0x2a +#define CG_CGLS_TILE_11 0x2b + +#define VM_L2_CG 0x15c0 + +#define MC_CONFIG 0x2000 + +#define MC_CONFIG_MCD 0x20a0 +#define MC_CG_CONFIG_MCD 0x20a4 +#define MC_RD_ENABLE_MCD(x) ((x) << 8) +#define MC_RD_ENABLE_MCD_MASK (7 << 8) + +#define MC_HUB_MISC_HUB_CG 0x20b8 +#define MC_HUB_MISC_VM_CG 0x20bc +#define MC_HUB_MISC_SIP_CG 0x20c0 + +#define MC_XPB_CLK_GAT 0x2478 + +#define MC_CG_CONFIG 0x25bc +#define MC_RD_ENABLE(x) ((x) << 4) +#define MC_RD_ENABLE_MASK (3 << 4) + +#define MC_CITF_MISC_RD_CG 0x2648 +#define MC_CITF_MISC_WR_CG 0x264c +#define MC_CITF_MISC_VM_CG 0x2650 +# define MEM_LS_ENABLE (1 << 19) + +#define MC_ARB_BURST_TIME 0x2808 +#define STATE0(x) ((x) << 0) +#define STATE0_MASK (0x1f << 0) +#define STATE1(x) ((x) << 5) +#define STATE1_MASK (0x1f << 5) +#define STATE2(x) ((x) << 10) +#define STATE2_MASK (0x1f << 10) +#define STATE3(x) ((x) << 15) +#define STATE3_MASK (0x1f << 15) + +#define MC_SEQ_RAS_TIMING 0x28a0 +#define MC_SEQ_CAS_TIMING 0x28a4 +#define MC_SEQ_MISC_TIMING 0x28a8 +#define MC_SEQ_MISC_TIMING2 0x28ac + +#define MC_SEQ_RD_CTL_D0 0x28b4 +#define MC_SEQ_RD_CTL_D1 0x28b8 +#define MC_SEQ_WR_CTL_D0 0x28bc +#define MC_SEQ_WR_CTL_D1 0x28c0 + +#define MC_SEQ_STATUS_M 0x29f4 +# define PMG_PWRSTATE (1 << 16) + +#define MC_SEQ_MISC1 0x2a04 +#define MC_SEQ_RESERVE_M 0x2a08 +#define MC_PMG_CMD_EMRS 0x2a0c + +#define MC_SEQ_MISC3 0x2a2c + +#define MC_SEQ_MISC5 0x2a54 +#define MC_SEQ_MISC6 0x2a58 + +#define MC_SEQ_MISC7 0x2a64 + +#define MC_SEQ_CG 0x2a68 +#define CG_SEQ_REQ(x) ((x) << 0) +#define CG_SEQ_REQ_MASK (0xff << 0) +#define CG_SEQ_REQ_SHIFT 0 +#define CG_SEQ_RESP(x) ((x) << 8) +#define CG_SEQ_RESP_MASK (0xff << 8) +#define CG_SEQ_RESP_SHIFT 8 +#define SEQ_CG_REQ(x) ((x) << 16) +#define SEQ_CG_REQ_MASK (0xff << 16) +#define SEQ_CG_REQ_SHIFT 16 +#define SEQ_CG_RESP(x) ((x) << 24) +#define SEQ_CG_RESP_MASK (0xff << 24) +#define SEQ_CG_RESP_SHIFT 24 +#define MC_SEQ_RAS_TIMING_LP 0x2a6c +#define MC_SEQ_CAS_TIMING_LP 0x2a70 +#define MC_SEQ_MISC_TIMING_LP 0x2a74 +#define MC_SEQ_MISC_TIMING2_LP 0x2a78 +#define MC_SEQ_WR_CTL_D0_LP 0x2a7c +#define MC_SEQ_WR_CTL_D1_LP 0x2a80 +#define MC_SEQ_PMG_CMD_EMRS_LP 0x2a84 +#define MC_SEQ_PMG_CMD_MRS_LP 0x2a88 + +#define MC_PMG_CMD_MRS 0x2aac + +#define MC_SEQ_RD_CTL_D0_LP 0x2b1c +#define MC_SEQ_RD_CTL_D1_LP 0x2b20 + +#define MC_PMG_CMD_MRS1 0x2b44 +#define MC_SEQ_PMG_CMD_MRS1_LP 0x2b48 + +#define CGTS_SM_CTRL_REG 0x9150 + /* Registers */ #define RCU_IND_INDEX 0x100 #define RCU_IND_DATA 0x104 +/* discrete uvd clocks */ +#define CG_UPLL_FUNC_CNTL 0x718 +# define UPLL_RESET_MASK 0x00000001 +# define UPLL_SLEEP_MASK 0x00000002 +# define UPLL_BYPASS_EN_MASK 0x00000004 +# define UPLL_CTLREQ_MASK 0x00000008 +# define UPLL_REF_DIV_MASK 0x003F0000 +# define UPLL_VCO_MODE_MASK 0x00000200 +# define UPLL_CTLACK_MASK 0x40000000 +# define UPLL_CTLACK2_MASK 0x80000000 +#define CG_UPLL_FUNC_CNTL_2 0x71c +# define UPLL_PDIV_A(x) ((x) << 0) +# define UPLL_PDIV_A_MASK 0x0000007F +# define UPLL_PDIV_B(x) ((x) << 8) +# define UPLL_PDIV_B_MASK 0x00007F00 +# define VCLK_SRC_SEL(x) ((x) << 20) +# define VCLK_SRC_SEL_MASK 0x01F00000 +# define DCLK_SRC_SEL(x) ((x) << 25) +# define DCLK_SRC_SEL_MASK 0x3E000000 +#define CG_UPLL_FUNC_CNTL_3 0x720 +# define UPLL_FB_DIV(x) ((x) << 0) +# define UPLL_FB_DIV_MASK 0x01FFFFFF +#define CG_UPLL_FUNC_CNTL_4 0x854 +# define UPLL_SPARE_ISPARE9 0x00020000 +#define CG_UPLL_SPREAD_SPECTRUM 0x79c +# define SSEN_MASK 0x00000001 + +/* fusion uvd clocks */ +#define CG_DCLK_CNTL 0x610 +# define DCLK_DIVIDER_MASK 0x7f +# define DCLK_DIR_CNTL_EN (1 << 8) +#define CG_DCLK_STATUS 0x614 +# define DCLK_STATUS (1 << 0) +#define CG_VCLK_CNTL 0x618 +#define CG_VCLK_STATUS 0x61c +#define CG_SCRATCH1 0x820 + +#define RLC_CNTL 0x3f00 +# define RLC_ENABLE (1 << 0) +# define GFX_POWER_GATING_ENABLE (1 << 7) +# define GFX_POWER_GATING_SRC (1 << 8) +# define DYN_PER_SIMD_PG_ENABLE (1 << 27) +# define LB_CNT_SPIM_ACTIVE (1 << 30) +# define LOAD_BALANCE_ENABLE (1 << 31) + +#define RLC_HB_BASE 0x3f10 +#define RLC_HB_CNTL 0x3f0c +#define RLC_HB_RPTR 0x3f20 +#define RLC_HB_WPTR 0x3f1c +#define RLC_HB_WPTR_LSB_ADDR 0x3f14 +#define RLC_HB_WPTR_MSB_ADDR 0x3f18 +#define RLC_MC_CNTL 0x3f44 +#define RLC_UCODE_CNTL 0x3f48 +#define RLC_UCODE_ADDR 0x3f2c +#define RLC_UCODE_DATA 0x3f30 + +/* new for TN */ +#define TN_RLC_SAVE_AND_RESTORE_BASE 0x3f10 +#define TN_RLC_LB_CNTR_MAX 0x3f14 +#define TN_RLC_LB_CNTR_INIT 0x3f18 +#define TN_RLC_CLEAR_STATE_RESTORE_BASE 0x3f20 +#define TN_RLC_LB_INIT_SIMD_MASK 0x3fe4 +#define TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK 0x3fe8 +#define TN_RLC_LB_PARAMS 0x3fec + #define GRBM_GFX_INDEX 0x802C #define INSTANCE_INDEX(x) ((x) << 0) #define SE_INDEX(x) ((x) << 16) @@ -146,11 +501,20 @@ #define DCCG_AUDIO_DTO0_MODULE 0x05b4 #define DCCG_AUDIO_DTO0_LOAD 0x05b8 #define DCCG_AUDIO_DTO0_CNTL 0x05bc +# define DCCG_AUDIO_DTO_WALLCLOCK_RATIO(x) (((x) & 7) << 0) +# define DCCG_AUDIO_DTO_WALLCLOCK_RATIO_MASK 7 +# define DCCG_AUDIO_DTO_WALLCLOCK_RATIO_SHIFT 0 #define DCCG_AUDIO_DTO1_PHASE 0x05c0 #define DCCG_AUDIO_DTO1_MODULE 0x05c4 #define DCCG_AUDIO_DTO1_LOAD 0x05c8 #define DCCG_AUDIO_DTO1_CNTL 0x05cc +# define DCCG_AUDIO_DTO1_USE_512FBR_DTO (1 << 3) + +#define DCE41_DENTIST_DISPCLK_CNTL 0x049c +# define DENTIST_DPREFCLK_WDIVIDER(x) (((x) & 0x7f) << 24) +# define DENTIST_DPREFCLK_WDIVIDER_MASK (0x7f << 24) +# define DENTIST_DPREFCLK_WDIVIDER_SHIFT 24 /* DCE 4.0 AFMT */ #define HDMI_CONTROL 0x7030 @@ -159,10 +523,11 @@ # define HDMI_ERROR_ACK (1 << 8) # define HDMI_ERROR_MASK (1 << 9) # define HDMI_DEEP_COLOR_ENABLE (1 << 24) -# define HDMI_DEEP_COLOR_DEPTH (((x) & 3) << 28) +# define HDMI_DEEP_COLOR_DEPTH(x) (((x) & 3) << 28) # define HDMI_24BIT_DEEP_COLOR 0 # define HDMI_30BIT_DEEP_COLOR 1 # define HDMI_36BIT_DEEP_COLOR 2 +# define HDMI_DEEP_COLOR_DEPTH_MASK (3 << 28) #define HDMI_STATUS 0x7034 # define HDMI_ACTIVE_AVMUTE (1 << 0) # define HDMI_AUDIO_PACKET_ERROR (1 << 16) @@ -198,6 +563,7 @@ # define HDMI_MPEG_INFO_CONT (1 << 9) #define HDMI_INFOFRAME_CONTROL1 0x7048 # define HDMI_AVI_INFO_LINE(x) (((x) & 0x3f) << 0) +# define HDMI_AVI_INFO_LINE_MASK (0x3f << 0) # define HDMI_AUDIO_INFO_LINE(x) (((x) & 0x3f) << 8) # define HDMI_MPEG_INFO_LINE(x) (((x) & 0x3f) << 16) #define HDMI_GENERIC_PACKET_CONTROL 0x704c @@ -359,6 +725,13 @@ #define AFMT_GENERIC0_7 0x7138 /* DCE4/5 ELD audio interface */ +#define AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER 0x5f78 +#define SPEAKER_ALLOCATION(x) (((x) & 0x7f) << 0) +#define SPEAKER_ALLOCATION_MASK (0x7f << 0) +#define SPEAKER_ALLOCATION_SHIFT 0 +#define HDMI_CONNECTION (1 << 16) +#define DP_CONNECTION (1 << 17) + #define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR0 0x5f84 /* LPCM */ #define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR1 0x5f88 /* AC3 */ #define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR2 0x5f8c /* MPEG1 */ @@ -388,6 +761,44 @@ * bit6 = 192 kHz */ +#define AZ_CHANNEL_COUNT_CONTROL 0x5fe4 +# define HBR_CHANNEL_COUNT(x) (((x) & 0x7) << 0) +# define COMPRESSED_CHANNEL_COUNT(x) (((x) & 0x7) << 4) +/* HBR_CHANNEL_COUNT, COMPRESSED_CHANNEL_COUNT + * 0 = use stream header + * 1-7 = channel count - 1 + */ +#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_LIPSYNC 0x5fe8 +# define VIDEO_LIPSYNC(x) (((x) & 0xff) << 0) +# define AUDIO_LIPSYNC(x) (((x) & 0xff) << 8) +/* VIDEO_LIPSYNC, AUDIO_LIPSYNC + * 0 = invalid + * x = legal delay value + * 255 = sync not supported + */ +#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_HBR 0x5fec +# define HBR_CAPABLE (1 << 0) /* enabled by default */ + +#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_AV_ASSOCIATION0 0x5ff4 +# define DISPLAY0_TYPE(x) (((x) & 0x3) << 0) +# define DISPLAY_TYPE_NONE 0 +# define DISPLAY_TYPE_HDMI 1 +# define DISPLAY_TYPE_DP 2 +# define DISPLAY0_ID(x) (((x) & 0x3f) << 2) +# define DISPLAY1_TYPE(x) (((x) & 0x3) << 8) +# define DISPLAY1_ID(x) (((x) & 0x3f) << 10) +# define DISPLAY2_TYPE(x) (((x) & 0x3) << 16) +# define DISPLAY2_ID(x) (((x) & 0x3f) << 18) +# define DISPLAY3_TYPE(x) (((x) & 0x3) << 24) +# define DISPLAY3_ID(x) (((x) & 0x3f) << 26) +#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_AV_ASSOCIATION1 0x5ff8 +# define DISPLAY4_TYPE(x) (((x) & 0x3) << 0) +# define DISPLAY4_ID(x) (((x) & 0x3f) << 2) +# define DISPLAY5_TYPE(x) (((x) & 0x3) << 8) +# define DISPLAY5_ID(x) (((x) & 0x3f) << 10) +#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_AV_NUMBER 0x5ffc +# define NUMBER_OF_DISPLAY_ID(x) (((x) & 0x7) << 0) + #define AZ_HOT_PLUG_CONTROL 0x5e78 # define AZ_FORCE_CODEC_WAKE (1 << 0) # define PIN0_JACK_DETECTION_ENABLE (1 << 4) @@ -466,6 +877,30 @@ #define CG_THERMAL_CTRL 0x72c #define TOFFSET_MASK 0x00003FE0 #define TOFFSET_SHIFT 5 +#define DIG_THERM_DPM(x) ((x) << 14) +#define DIG_THERM_DPM_MASK 0x003FC000 +#define DIG_THERM_DPM_SHIFT 14 + +#define CG_THERMAL_INT 0x734 +#define DIG_THERM_INTH(x) ((x) << 8) +#define DIG_THERM_INTH_MASK 0x0000FF00 +#define DIG_THERM_INTH_SHIFT 8 +#define DIG_THERM_INTL(x) ((x) << 16) +#define DIG_THERM_INTL_MASK 0x00FF0000 +#define DIG_THERM_INTL_SHIFT 16 +#define THERM_INT_MASK_HIGH (1 << 24) +#define THERM_INT_MASK_LOW (1 << 25) + +#define TN_CG_THERMAL_INT_CTRL 0x738 +#define TN_DIG_THERM_INTH(x) ((x) << 0) +#define TN_DIG_THERM_INTH_MASK 0x000000FF +#define TN_DIG_THERM_INTH_SHIFT 0 +#define TN_DIG_THERM_INTL(x) ((x) << 8) +#define TN_DIG_THERM_INTL_MASK 0x0000FF00 +#define TN_DIG_THERM_INTL_SHIFT 8 +#define TN_THERM_INT_MASK_HIGH (1 << 24) +#define TN_THERM_INT_MASK_LOW (1 << 25) + #define CG_MULT_THERMAL_STATUS 0x740 #define ASIC_T(x) ((x) << 16) #define ASIC_T_MASK 0x07FF0000 @@ -473,6 +908,7 @@ #define CG_TS0_STATUS 0x760 #define TS0_ADC_DOUT_MASK 0x000003FF #define TS0_ADC_DOUT_SHIFT 0 + /* APU */ #define CG_THERMAL_STATUS 0x678 @@ -730,6 +1166,18 @@ #define WAIT_UNTIL 0x8040 #define SRBM_STATUS 0x0E50 +#define RLC_RQ_PENDING (1 << 3) +#define GRBM_RQ_PENDING (1 << 5) +#define VMC_BUSY (1 << 8) +#define MCB_BUSY (1 << 9) +#define MCB_NON_DISPLAY_BUSY (1 << 10) +#define MCC_BUSY (1 << 11) +#define MCD_BUSY (1 << 12) +#define SEM_BUSY (1 << 14) +#define RLC_BUSY (1 << 15) +#define IH_BUSY (1 << 17) +#define SRBM_STATUS2 0x0EC4 +#define DMA_BUSY (1 << 5) #define SRBM_SOFT_RESET 0x0E60 #define SRBM_SOFT_RESET_ALL_MASK 0x00FEEFA6 #define SOFT_RESET_BIF (1 << 1) @@ -748,6 +1196,10 @@ #define SOFT_RESET_REGBB (1 << 22) #define SOFT_RESET_ORB (1 << 23) +#define SRBM_READ_ERROR 0xE98 +#define SRBM_INT_CNTL 0xEA0 +#define SRBM_INT_ACK 0xEA8 + /* display watermarks */ #define DC_LB_MEMORY_SPLIT 0x6b0c #define PRIORITY_A_CNT 0x6b18 @@ -913,6 +1365,38 @@ # define DC_HPDx_RX_INT_TIMER(x) ((x) << 16) # define DC_HPDx_EN (1 << 28) +/* DCE4/5/6 FMT blocks */ +#define FMT_DYNAMIC_EXP_CNTL 0x6fb4 +# define FMT_DYNAMIC_EXP_EN (1 << 0) +# define FMT_DYNAMIC_EXP_MODE (1 << 4) + /* 0 = 10bit -> 12bit, 1 = 8bit -> 12bit */ +#define FMT_CONTROL 0x6fb8 +# define FMT_PIXEL_ENCODING (1 << 16) + /* 0 = RGB 4:4:4 or YCbCr 4:4:4, 1 = YCbCr 4:2:2 */ +#define FMT_BIT_DEPTH_CONTROL 0x6fc8 +# define FMT_TRUNCATE_EN (1 << 0) +# define FMT_TRUNCATE_DEPTH (1 << 4) +# define FMT_SPATIAL_DITHER_EN (1 << 8) +# define FMT_SPATIAL_DITHER_MODE(x) ((x) << 9) +# define FMT_SPATIAL_DITHER_DEPTH (1 << 12) +# define FMT_FRAME_RANDOM_ENABLE (1 << 13) +# define FMT_RGB_RANDOM_ENABLE (1 << 14) +# define FMT_HIGHPASS_RANDOM_ENABLE (1 << 15) +# define FMT_TEMPORAL_DITHER_EN (1 << 16) +# define FMT_TEMPORAL_DITHER_DEPTH (1 << 20) +# define FMT_TEMPORAL_DITHER_OFFSET(x) ((x) << 21) +# define FMT_TEMPORAL_LEVEL (1 << 24) +# define FMT_TEMPORAL_DITHER_RESET (1 << 25) +# define FMT_25FRC_SEL(x) ((x) << 26) +# define FMT_50FRC_SEL(x) ((x) << 28) +# define FMT_75FRC_SEL(x) ((x) << 30) +#define FMT_CLAMP_CONTROL 0x6fe4 +# define FMT_CLAMP_DATA_EN (1 << 0) +# define FMT_CLAMP_COLOR_FORMAT(x) ((x) << 16) +# define FMT_CLAMP_6BPC 0 +# define FMT_CLAMP_8BPC 1 +# define FMT_CLAMP_10BPC 2 + /* ASYNC DMA */ #define DMA_RB_RPTR 0xd008 #define DMA_RB_WPTR 0xd00c @@ -929,22 +1413,66 @@ #define CAYMAN_DMA1_CNTL 0xd82c /* async DMA packets */ -#define DMA_PACKET(cmd, t, s, n) ((((cmd) & 0xF) << 28) | \ - (((t) & 0x1) << 23) | \ - (((s) & 0x1) << 22) | \ - (((n) & 0xFFFFF) << 0)) +#define DMA_PACKET(cmd, sub_cmd, n) ((((cmd) & 0xF) << 28) | \ + (((sub_cmd) & 0xFF) << 20) |\ + (((n) & 0xFFFFF) << 0)) +#define GET_DMA_CMD(h) (((h) & 0xf0000000) >> 28) +#define GET_DMA_COUNT(h) ((h) & 0x000fffff) +#define GET_DMA_SUB_CMD(h) (((h) & 0x0ff00000) >> 20) + /* async DMA Packet types */ -#define DMA_PACKET_WRITE 0x2 -#define DMA_PACKET_COPY 0x3 -#define DMA_PACKET_INDIRECT_BUFFER 0x4 -#define DMA_PACKET_SEMAPHORE 0x5 -#define DMA_PACKET_FENCE 0x6 -#define DMA_PACKET_TRAP 0x7 -#define DMA_PACKET_SRBM_WRITE 0x9 -#define DMA_PACKET_CONSTANT_FILL 0xd -#define DMA_PACKET_NOP 0xf - -/* PCIE link stuff */ +#define DMA_PACKET_WRITE 0x2 +#define DMA_PACKET_COPY 0x3 +#define DMA_PACKET_INDIRECT_BUFFER 0x4 +#define DMA_PACKET_SEMAPHORE 0x5 +#define DMA_PACKET_FENCE 0x6 +#define DMA_PACKET_TRAP 0x7 +#define DMA_PACKET_SRBM_WRITE 0x9 +#define DMA_PACKET_CONSTANT_FILL 0xd +#define DMA_PACKET_NOP 0xf + +/* PIF PHY0 indirect regs */ +#define PB0_PIF_CNTL 0x10 +# define LS2_EXIT_TIME(x) ((x) << 17) +# define LS2_EXIT_TIME_MASK (0x7 << 17) +# define LS2_EXIT_TIME_SHIFT 17 +#define PB0_PIF_PAIRING 0x11 +# define MULTI_PIF (1 << 25) +#define PB0_PIF_PWRDOWN_0 0x12 +# define PLL_POWER_STATE_IN_TXS2_0(x) ((x) << 7) +# define PLL_POWER_STATE_IN_TXS2_0_MASK (0x7 << 7) +# define PLL_POWER_STATE_IN_TXS2_0_SHIFT 7 +# define PLL_POWER_STATE_IN_OFF_0(x) ((x) << 10) +# define PLL_POWER_STATE_IN_OFF_0_MASK (0x7 << 10) +# define PLL_POWER_STATE_IN_OFF_0_SHIFT 10 +# define PLL_RAMP_UP_TIME_0(x) ((x) << 24) +# define PLL_RAMP_UP_TIME_0_MASK (0x7 << 24) +# define PLL_RAMP_UP_TIME_0_SHIFT 24 +#define PB0_PIF_PWRDOWN_1 0x13 +# define PLL_POWER_STATE_IN_TXS2_1(x) ((x) << 7) +# define PLL_POWER_STATE_IN_TXS2_1_MASK (0x7 << 7) +# define PLL_POWER_STATE_IN_TXS2_1_SHIFT 7 +# define PLL_POWER_STATE_IN_OFF_1(x) ((x) << 10) +# define PLL_POWER_STATE_IN_OFF_1_MASK (0x7 << 10) +# define PLL_POWER_STATE_IN_OFF_1_SHIFT 10 +# define PLL_RAMP_UP_TIME_1(x) ((x) << 24) +# define PLL_RAMP_UP_TIME_1_MASK (0x7 << 24) +# define PLL_RAMP_UP_TIME_1_SHIFT 24 +/* PIF PHY1 indirect regs */ +#define PB1_PIF_CNTL 0x10 +#define PB1_PIF_PAIRING 0x11 +#define PB1_PIF_PWRDOWN_0 0x12 +#define PB1_PIF_PWRDOWN_1 0x13 +/* PCIE PORT indirect regs */ +#define PCIE_LC_CNTL 0xa0 +# define LC_L0S_INACTIVITY(x) ((x) << 8) +# define LC_L0S_INACTIVITY_MASK (0xf << 8) +# define LC_L0S_INACTIVITY_SHIFT 8 +# define LC_L1_INACTIVITY(x) ((x) << 12) +# define LC_L1_INACTIVITY_MASK (0xf << 12) +# define LC_L1_INACTIVITY_SHIFT 12 +# define LC_PMI_TO_L1_DIS (1 << 16) +# define LC_ASPM_TO_L1_DIS (1 << 24) #define PCIE_LC_TRAINING_CNTL 0xa1 /* PCIE_P */ #define PCIE_LC_LINK_WIDTH_CNTL 0xa2 /* PCIE_P */ # define LC_LINK_WIDTH_SHIFT 0 @@ -964,6 +1492,9 @@ # define LC_SHORT_RECONFIG_EN (1 << 11) # define LC_UPCONFIGURE_SUPPORT (1 << 12) # define LC_UPCONFIGURE_DIS (1 << 13) +# define LC_DYN_LANES_PWR_STATE(x) ((x) << 21) +# define LC_DYN_LANES_PWR_STATE_MASK (0x3 << 21) +# define LC_DYN_LANES_PWR_STATE_SHIFT 21 #define PCIE_LC_SPEED_CNTL 0xa4 /* PCIE_P */ # define LC_GEN2_EN_STRAP (1 << 0) # define LC_TARGET_LINK_SPEED_OVERRIDE_EN (1 << 1) @@ -972,6 +1503,9 @@ # define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK (0x3 << 8) # define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT 3 # define LC_CURRENT_DATA_RATE (1 << 11) +# define LC_HW_VOLTAGE_IF_CONTROL(x) ((x) << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_MASK (3 << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_SHIFT 12 # define LC_VOLTAGE_TIMER_SEL_MASK (0xf << 14) # define LC_CLR_FAILED_SPD_CHANGE_CNT (1 << 21) # define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 23) @@ -982,19 +1516,21 @@ # define TARGET_LINK_SPEED_MASK (0xf << 0) # define SELECTABLE_DEEMPHASIS (1 << 6) + +/* + * UVD + */ +#define UVD_UDEC_ADDR_CONFIG 0xef4c +#define UVD_UDEC_DB_ADDR_CONFIG 0xef50 +#define UVD_UDEC_DBW_ADDR_CONFIG 0xef54 +#define UVD_RBC_RB_RPTR 0xf690 +#define UVD_RBC_RB_WPTR 0xf694 +#define UVD_STATUS 0xf6bc + /* * PM4 */ -#define PACKET_TYPE0 0 -#define PACKET_TYPE1 1 -#define PACKET_TYPE2 2 -#define PACKET_TYPE3 3 - -#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) -#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) -#define CP_PACKET0_GET_REG(h) (((h) & 0xFFFF) << 2) -#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) -#define PACKET0(reg, n) ((PACKET_TYPE0 << 30) | \ +#define PACKET0(reg, n) ((RADEON_PACKET_TYPE0 << 30) | \ (((reg) >> 2) & 0xFFFF) | \ ((n) & 0x3FFF) << 16) #define CP_PACKET2 0x80000000 @@ -1003,7 +1539,7 @@ #define PACKET2(v) (CP_PACKET2 | REG_SET(PACKET2_PAD, (v))) -#define PACKET3(op, n) ((PACKET_TYPE3 << 30) | \ +#define PACKET3(op, n) ((RADEON_PACKET_TYPE3 << 30) | \ (((op) & 0xFF) << 8) | \ ((n) & 0x3FFF) << 16) diff --git a/sys/dev/pci/drm/radeon/kv_dpm.c b/sys/dev/pci/drm/radeon/kv_dpm.c new file mode 100644 index 00000000000..b2b77d65076 --- /dev/null +++ b/sys/dev/pci/drm/radeon/kv_dpm.c @@ -0,0 +1,2897 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "radeon.h" +#include "cikd.h" +#include "r600_dpm.h" +#include "kv_dpm.h" +#include "radeon_asic.h" + +#define KV_MAX_DEEPSLEEP_DIVIDER_ID 5 +#define KV_MINIMUM_ENGINE_CLOCK 800 +#define SMC_RAM_END 0x40000 + +static int kv_enable_nb_dpm(struct radeon_device *rdev, + bool enable); +static void kv_init_graphics_levels(struct radeon_device *rdev); +static int kv_calculate_ds_divider(struct radeon_device *rdev); +static int kv_calculate_nbps_level_settings(struct radeon_device *rdev); +static int kv_calculate_dpm_settings(struct radeon_device *rdev); +static void kv_enable_new_levels(struct radeon_device *rdev); +static void kv_program_nbps_index_settings(struct radeon_device *rdev, + struct radeon_ps *new_rps); +static int kv_set_enabled_level(struct radeon_device *rdev, u32 level); +static int kv_set_enabled_levels(struct radeon_device *rdev); +static int kv_force_dpm_highest(struct radeon_device *rdev); +static int kv_force_dpm_lowest(struct radeon_device *rdev); +static void kv_apply_state_adjust_rules(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps); +static int kv_set_thermal_temperature_range(struct radeon_device *rdev, + int min_temp, int max_temp); +static int kv_init_fps_limits(struct radeon_device *rdev); + +void kv_dpm_powergate_uvd(struct radeon_device *rdev, bool gate); +static void kv_dpm_powergate_vce(struct radeon_device *rdev, bool gate); +static void kv_dpm_powergate_samu(struct radeon_device *rdev, bool gate); +static void kv_dpm_powergate_acp(struct radeon_device *rdev, bool gate); + +extern void cik_enter_rlc_safe_mode(struct radeon_device *rdev); +extern void cik_exit_rlc_safe_mode(struct radeon_device *rdev); +extern void cik_update_cg(struct radeon_device *rdev, + u32 block, bool enable); + +static const struct kv_lcac_config_values sx_local_cac_cfg_kv[] = +{ + { 0, 4, 1 }, + { 1, 4, 1 }, + { 2, 5, 1 }, + { 3, 4, 2 }, + { 4, 1, 1 }, + { 5, 5, 2 }, + { 6, 6, 1 }, + { 7, 9, 2 }, + { 0xffffffff } +}; + +static const struct kv_lcac_config_values mc0_local_cac_cfg_kv[] = +{ + { 0, 4, 1 }, + { 0xffffffff } +}; + +static const struct kv_lcac_config_values mc1_local_cac_cfg_kv[] = +{ + { 0, 4, 1 }, + { 0xffffffff } +}; + +static const struct kv_lcac_config_values mc2_local_cac_cfg_kv[] = +{ + { 0, 4, 1 }, + { 0xffffffff } +}; + +static const struct kv_lcac_config_values mc3_local_cac_cfg_kv[] = +{ + { 0, 4, 1 }, + { 0xffffffff } +}; + +static const struct kv_lcac_config_values cpl_local_cac_cfg_kv[] = +{ + { 0, 4, 1 }, + { 1, 4, 1 }, + { 2, 5, 1 }, + { 3, 4, 1 }, + { 4, 1, 1 }, + { 5, 5, 1 }, + { 6, 6, 1 }, + { 7, 9, 1 }, + { 8, 4, 1 }, + { 9, 2, 1 }, + { 10, 3, 1 }, + { 11, 6, 1 }, + { 12, 8, 2 }, + { 13, 1, 1 }, + { 14, 2, 1 }, + { 15, 3, 1 }, + { 16, 1, 1 }, + { 17, 4, 1 }, + { 18, 3, 1 }, + { 19, 1, 1 }, + { 20, 8, 1 }, + { 21, 5, 1 }, + { 22, 1, 1 }, + { 23, 1, 1 }, + { 24, 4, 1 }, + { 27, 6, 1 }, + { 28, 1, 1 }, + { 0xffffffff } +}; + +static const struct kv_lcac_config_reg sx0_cac_config_reg[] = +{ + { 0xc0400d00, 0x003e0000, 17, 0x3fc00000, 22, 0x0001fffe, 1, 0x00000001, 0 } +}; + +static const struct kv_lcac_config_reg mc0_cac_config_reg[] = +{ + { 0xc0400d30, 0x003e0000, 17, 0x3fc00000, 22, 0x0001fffe, 1, 0x00000001, 0 } +}; + +static const struct kv_lcac_config_reg mc1_cac_config_reg[] = +{ + { 0xc0400d3c, 0x003e0000, 17, 0x3fc00000, 22, 0x0001fffe, 1, 0x00000001, 0 } +}; + +static const struct kv_lcac_config_reg mc2_cac_config_reg[] = +{ + { 0xc0400d48, 0x003e0000, 17, 0x3fc00000, 22, 0x0001fffe, 1, 0x00000001, 0 } +}; + +static const struct kv_lcac_config_reg mc3_cac_config_reg[] = +{ + { 0xc0400d54, 0x003e0000, 17, 0x3fc00000, 22, 0x0001fffe, 1, 0x00000001, 0 } +}; + +static const struct kv_lcac_config_reg cpl_cac_config_reg[] = +{ + { 0xc0400d80, 0x003e0000, 17, 0x3fc00000, 22, 0x0001fffe, 1, 0x00000001, 0 } +}; + +static const struct kv_pt_config_reg didt_config_kv[] = +{ + { 0x10, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x10, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x10, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x10, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x11, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x11, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x11, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x11, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x12, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x12, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x12, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x12, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x2, 0x00003fff, 0, 0x4, KV_CONFIGREG_DIDT_IND }, + { 0x2, 0x03ff0000, 16, 0x80, KV_CONFIGREG_DIDT_IND }, + { 0x2, 0x78000000, 27, 0x3, KV_CONFIGREG_DIDT_IND }, + { 0x1, 0x0000ffff, 0, 0x3FFF, KV_CONFIGREG_DIDT_IND }, + { 0x1, 0xffff0000, 16, 0x3FFF, KV_CONFIGREG_DIDT_IND }, + { 0x0, 0x00000001, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x30, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x30, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x30, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x30, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x31, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x31, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x31, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x31, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x32, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x32, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x32, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x32, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x22, 0x00003fff, 0, 0x4, KV_CONFIGREG_DIDT_IND }, + { 0x22, 0x03ff0000, 16, 0x80, KV_CONFIGREG_DIDT_IND }, + { 0x22, 0x78000000, 27, 0x3, KV_CONFIGREG_DIDT_IND }, + { 0x21, 0x0000ffff, 0, 0x3FFF, KV_CONFIGREG_DIDT_IND }, + { 0x21, 0xffff0000, 16, 0x3FFF, KV_CONFIGREG_DIDT_IND }, + { 0x20, 0x00000001, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x50, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x50, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x50, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x50, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x51, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x51, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x51, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x51, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x52, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x52, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x52, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x52, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x42, 0x00003fff, 0, 0x4, KV_CONFIGREG_DIDT_IND }, + { 0x42, 0x03ff0000, 16, 0x80, KV_CONFIGREG_DIDT_IND }, + { 0x42, 0x78000000, 27, 0x3, KV_CONFIGREG_DIDT_IND }, + { 0x41, 0x0000ffff, 0, 0x3FFF, KV_CONFIGREG_DIDT_IND }, + { 0x41, 0xffff0000, 16, 0x3FFF, KV_CONFIGREG_DIDT_IND }, + { 0x40, 0x00000001, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x70, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x70, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x70, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x70, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x71, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x71, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x71, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x71, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x72, 0x000000ff, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x72, 0x0000ff00, 8, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x72, 0x00ff0000, 16, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x72, 0xff000000, 24, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0x62, 0x00003fff, 0, 0x4, KV_CONFIGREG_DIDT_IND }, + { 0x62, 0x03ff0000, 16, 0x80, KV_CONFIGREG_DIDT_IND }, + { 0x62, 0x78000000, 27, 0x3, KV_CONFIGREG_DIDT_IND }, + { 0x61, 0x0000ffff, 0, 0x3FFF, KV_CONFIGREG_DIDT_IND }, + { 0x61, 0xffff0000, 16, 0x3FFF, KV_CONFIGREG_DIDT_IND }, + { 0x60, 0x00000001, 0, 0x0, KV_CONFIGREG_DIDT_IND }, + { 0xFFFFFFFF } +}; + +static struct kv_ps *kv_get_ps(struct radeon_ps *rps) +{ + struct kv_ps *ps = rps->ps_priv; + + return ps; +} + +static struct kv_power_info *kv_get_pi(struct radeon_device *rdev) +{ + struct kv_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +#if 0 +static void kv_program_local_cac_table(struct radeon_device *rdev, + const struct kv_lcac_config_values *local_cac_table, + const struct kv_lcac_config_reg *local_cac_reg) +{ + u32 i, count, data; + const struct kv_lcac_config_values *values = local_cac_table; + + while (values->block_id != 0xffffffff) { + count = values->signal_id; + for (i = 0; i < count; i++) { + data = ((values->block_id << local_cac_reg->block_shift) & + local_cac_reg->block_mask); + data |= ((i << local_cac_reg->signal_shift) & + local_cac_reg->signal_mask); + data |= ((values->t << local_cac_reg->t_shift) & + local_cac_reg->t_mask); + data |= ((1 << local_cac_reg->enable_shift) & + local_cac_reg->enable_mask); + WREG32_SMC(local_cac_reg->cntl, data); + } + values++; + } +} +#endif + +static int kv_program_pt_config_registers(struct radeon_device *rdev, + const struct kv_pt_config_reg *cac_config_regs) +{ + const struct kv_pt_config_reg *config_regs = cac_config_regs; + u32 data; + u32 cache = 0; + + if (config_regs == NULL) + return -EINVAL; + + while (config_regs->offset != 0xFFFFFFFF) { + if (config_regs->type == KV_CONFIGREG_CACHE) { + cache |= ((config_regs->value << config_regs->shift) & config_regs->mask); + } else { + switch (config_regs->type) { + case KV_CONFIGREG_SMC_IND: + data = RREG32_SMC(config_regs->offset); + break; + case KV_CONFIGREG_DIDT_IND: + data = RREG32_DIDT(config_regs->offset); + break; + default: + data = RREG32(config_regs->offset << 2); + break; + } + + data &= ~config_regs->mask; + data |= ((config_regs->value << config_regs->shift) & config_regs->mask); + data |= cache; + cache = 0; + + switch (config_regs->type) { + case KV_CONFIGREG_SMC_IND: + WREG32_SMC(config_regs->offset, data); + break; + case KV_CONFIGREG_DIDT_IND: + WREG32_DIDT(config_regs->offset, data); + break; + default: + WREG32(config_regs->offset << 2, data); + break; + } + } + config_regs++; + } + + return 0; +} + +static void kv_do_enable_didt(struct radeon_device *rdev, bool enable) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 data; + + if (pi->caps_sq_ramping) { + data = RREG32_DIDT(DIDT_SQ_CTRL0); + if (enable) + data |= DIDT_CTRL_EN; + else + data &= ~DIDT_CTRL_EN; + WREG32_DIDT(DIDT_SQ_CTRL0, data); + } + + if (pi->caps_db_ramping) { + data = RREG32_DIDT(DIDT_DB_CTRL0); + if (enable) + data |= DIDT_CTRL_EN; + else + data &= ~DIDT_CTRL_EN; + WREG32_DIDT(DIDT_DB_CTRL0, data); + } + + if (pi->caps_td_ramping) { + data = RREG32_DIDT(DIDT_TD_CTRL0); + if (enable) + data |= DIDT_CTRL_EN; + else + data &= ~DIDT_CTRL_EN; + WREG32_DIDT(DIDT_TD_CTRL0, data); + } + + if (pi->caps_tcp_ramping) { + data = RREG32_DIDT(DIDT_TCP_CTRL0); + if (enable) + data |= DIDT_CTRL_EN; + else + data &= ~DIDT_CTRL_EN; + WREG32_DIDT(DIDT_TCP_CTRL0, data); + } +} + +static int kv_enable_didt(struct radeon_device *rdev, bool enable) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret; + + if (pi->caps_sq_ramping || + pi->caps_db_ramping || + pi->caps_td_ramping || + pi->caps_tcp_ramping) { + cik_enter_rlc_safe_mode(rdev); + + if (enable) { + ret = kv_program_pt_config_registers(rdev, didt_config_kv); + if (ret) { + cik_exit_rlc_safe_mode(rdev); + return ret; + } + } + + kv_do_enable_didt(rdev, enable); + + cik_exit_rlc_safe_mode(rdev); + } + + return 0; +} + +#if 0 +static void kv_initialize_hardware_cac_manager(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + if (pi->caps_cac) { + WREG32_SMC(LCAC_SX0_OVR_SEL, 0); + WREG32_SMC(LCAC_SX0_OVR_VAL, 0); + kv_program_local_cac_table(rdev, sx_local_cac_cfg_kv, sx0_cac_config_reg); + + WREG32_SMC(LCAC_MC0_OVR_SEL, 0); + WREG32_SMC(LCAC_MC0_OVR_VAL, 0); + kv_program_local_cac_table(rdev, mc0_local_cac_cfg_kv, mc0_cac_config_reg); + + WREG32_SMC(LCAC_MC1_OVR_SEL, 0); + WREG32_SMC(LCAC_MC1_OVR_VAL, 0); + kv_program_local_cac_table(rdev, mc1_local_cac_cfg_kv, mc1_cac_config_reg); + + WREG32_SMC(LCAC_MC2_OVR_SEL, 0); + WREG32_SMC(LCAC_MC2_OVR_VAL, 0); + kv_program_local_cac_table(rdev, mc2_local_cac_cfg_kv, mc2_cac_config_reg); + + WREG32_SMC(LCAC_MC3_OVR_SEL, 0); + WREG32_SMC(LCAC_MC3_OVR_VAL, 0); + kv_program_local_cac_table(rdev, mc3_local_cac_cfg_kv, mc3_cac_config_reg); + + WREG32_SMC(LCAC_CPL_OVR_SEL, 0); + WREG32_SMC(LCAC_CPL_OVR_VAL, 0); + kv_program_local_cac_table(rdev, cpl_local_cac_cfg_kv, cpl_cac_config_reg); + } +} +#endif + +static int kv_enable_smc_cac(struct radeon_device *rdev, bool enable) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret = 0; + + if (pi->caps_cac) { + if (enable) { + ret = kv_notify_message_to_smu(rdev, PPSMC_MSG_EnableCac); + if (ret) + pi->cac_enabled = false; + else + pi->cac_enabled = true; + } else if (pi->cac_enabled) { + kv_notify_message_to_smu(rdev, PPSMC_MSG_DisableCac); + pi->cac_enabled = false; + } + } + + return ret; +} + +static int kv_process_firmware_header(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 tmp; + int ret; + + ret = kv_read_smc_sram_dword(rdev, SMU7_FIRMWARE_HEADER_LOCATION + + offsetof(SMU7_Firmware_Header, DpmTable), + &tmp, pi->sram_end); + + if (ret == 0) + pi->dpm_table_start = tmp; + + ret = kv_read_smc_sram_dword(rdev, SMU7_FIRMWARE_HEADER_LOCATION + + offsetof(SMU7_Firmware_Header, SoftRegisters), + &tmp, pi->sram_end); + + if (ret == 0) + pi->soft_regs_start = tmp; + + return ret; +} + +static int kv_enable_dpm_voltage_scaling(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret; + + pi->graphics_voltage_change_enable = 1; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, GraphicsVoltageChangeEnable), + &pi->graphics_voltage_change_enable, + sizeof(u8), pi->sram_end); + + return ret; +} + +static int kv_set_dpm_interval(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret; + + pi->graphics_interval = 1; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, GraphicsInterval), + &pi->graphics_interval, + sizeof(u8), pi->sram_end); + + return ret; +} + +static int kv_set_dpm_boot_state(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, GraphicsBootLevel), + &pi->graphics_boot_level, + sizeof(u8), pi->sram_end); + + return ret; +} + +static void kv_program_vc(struct radeon_device *rdev) +{ + WREG32_SMC(CG_FTV_0, 0x3FFFC100); +} + +static void kv_clear_vc(struct radeon_device *rdev) +{ + WREG32_SMC(CG_FTV_0, 0); +} + +static int kv_set_divider_value(struct radeon_device *rdev, + u32 index, u32 sclk) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct atom_clock_dividers dividers; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + sclk, false, ÷rs); + if (ret) + return ret; + + pi->graphics_level[index].SclkDid = (u8)dividers.post_div; + pi->graphics_level[index].SclkFrequency = cpu_to_be32(sclk); + + return 0; +} + +static u32 kv_convert_vid2_to_vid7(struct radeon_device *rdev, + struct sumo_vid_mapping_table *vid_mapping_table, + u32 vid_2bit) +{ + struct radeon_clock_voltage_dependency_table *vddc_sclk_table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + u32 i; + + if (vddc_sclk_table && vddc_sclk_table->count) { + if (vid_2bit < vddc_sclk_table->count) + return vddc_sclk_table->entries[vid_2bit].v; + else + return vddc_sclk_table->entries[vddc_sclk_table->count - 1].v; + } else { + for (i = 0; i < vid_mapping_table->num_entries; i++) { + if (vid_mapping_table->entries[i].vid_2bit == vid_2bit) + return vid_mapping_table->entries[i].vid_7bit; + } + return vid_mapping_table->entries[vid_mapping_table->num_entries - 1].vid_7bit; + } +} + +static u32 kv_convert_vid7_to_vid2(struct radeon_device *rdev, + struct sumo_vid_mapping_table *vid_mapping_table, + u32 vid_7bit) +{ + struct radeon_clock_voltage_dependency_table *vddc_sclk_table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + u32 i; + + if (vddc_sclk_table && vddc_sclk_table->count) { + for (i = 0; i < vddc_sclk_table->count; i++) { + if (vddc_sclk_table->entries[i].v == vid_7bit) + return i; + } + return vddc_sclk_table->count - 1; + } else { + for (i = 0; i < vid_mapping_table->num_entries; i++) { + if (vid_mapping_table->entries[i].vid_7bit == vid_7bit) + return vid_mapping_table->entries[i].vid_2bit; + } + + return vid_mapping_table->entries[vid_mapping_table->num_entries - 1].vid_2bit; + } +} + +static u16 kv_convert_8bit_index_to_voltage(struct radeon_device *rdev, + u16 voltage) +{ + return 6200 - (voltage * 25); +} + +static u16 kv_convert_2bit_index_to_voltage(struct radeon_device *rdev, + u32 vid_2bit) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 vid_8bit = kv_convert_vid2_to_vid7(rdev, + &pi->sys_info.vid_mapping_table, + vid_2bit); + + return kv_convert_8bit_index_to_voltage(rdev, (u16)vid_8bit); +} + + +static int kv_set_vid(struct radeon_device *rdev, u32 index, u32 vid) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->graphics_level[index].VoltageDownH = (u8)pi->voltage_drop_t; + pi->graphics_level[index].MinVddNb = + cpu_to_be32(kv_convert_2bit_index_to_voltage(rdev, vid)); + + return 0; +} + +static int kv_set_at(struct radeon_device *rdev, u32 index, u32 at) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->graphics_level[index].AT = cpu_to_be16((u16)at); + + return 0; +} + +static void kv_dpm_power_level_enable(struct radeon_device *rdev, + u32 index, bool enable) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->graphics_level[index].EnabledForActivity = enable ? 1 : 0; +} + +static void kv_start_dpm(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(GENERAL_PWRMGT); + + tmp |= GLOBAL_PWRMGT_EN; + WREG32_SMC(GENERAL_PWRMGT, tmp); + + kv_smc_dpm_enable(rdev, true); +} + +static void kv_stop_dpm(struct radeon_device *rdev) +{ + kv_smc_dpm_enable(rdev, false); +} + +static void kv_start_am(struct radeon_device *rdev) +{ + u32 sclk_pwrmgt_cntl = RREG32_SMC(SCLK_PWRMGT_CNTL); + + sclk_pwrmgt_cntl &= ~(RESET_SCLK_CNT | RESET_BUSY_CNT); + sclk_pwrmgt_cntl |= DYNAMIC_PM_EN; + + WREG32_SMC(SCLK_PWRMGT_CNTL, sclk_pwrmgt_cntl); +} + +static void kv_reset_am(struct radeon_device *rdev) +{ + u32 sclk_pwrmgt_cntl = RREG32_SMC(SCLK_PWRMGT_CNTL); + + sclk_pwrmgt_cntl |= (RESET_SCLK_CNT | RESET_BUSY_CNT); + + WREG32_SMC(SCLK_PWRMGT_CNTL, sclk_pwrmgt_cntl); +} + +static int kv_freeze_sclk_dpm(struct radeon_device *rdev, bool freeze) +{ + return kv_notify_message_to_smu(rdev, freeze ? + PPSMC_MSG_SCLKDPM_FreezeLevel : PPSMC_MSG_SCLKDPM_UnfreezeLevel); +} + +static int kv_force_lowest_valid(struct radeon_device *rdev) +{ + return kv_force_dpm_lowest(rdev); +} + +static int kv_unforce_levels(struct radeon_device *rdev) +{ + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) + return kv_notify_message_to_smu(rdev, PPSMC_MSG_NoForcedLevel); + else + return kv_set_enabled_levels(rdev); +} + +static int kv_update_sclk_t(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 low_sclk_interrupt_t = 0; + int ret = 0; + + if (pi->caps_sclk_throttle_low_notification) { + low_sclk_interrupt_t = cpu_to_be32(pi->low_sclk_interrupt_t); + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, LowSclkInterruptT), + (u8 *)&low_sclk_interrupt_t, + sizeof(u32), pi->sram_end); + } + return ret; +} + +static int kv_program_bootup_state(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 i; + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + + if (table && table->count) { + for (i = pi->graphics_dpm_level_count - 1; i > 0; i--) { + if (table->entries[i].clk == pi->boot_pl.sclk) + break; + } + + pi->graphics_boot_level = (u8)i; + kv_dpm_power_level_enable(rdev, i, true); + } else { + struct sumo_sclk_voltage_mapping_table *table = + &pi->sys_info.sclk_voltage_mapping_table; + + if (table->num_max_dpm_entries == 0) + return -EINVAL; + + for (i = pi->graphics_dpm_level_count - 1; i > 0; i--) { + if (table->entries[i].sclk_frequency == pi->boot_pl.sclk) + break; + } + + pi->graphics_boot_level = (u8)i; + kv_dpm_power_level_enable(rdev, i, true); + } + return 0; +} + +static int kv_enable_auto_thermal_throttling(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret; + + pi->graphics_therm_throttle_enable = 1; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, GraphicsThermThrottleEnable), + &pi->graphics_therm_throttle_enable, + sizeof(u8), pi->sram_end); + + return ret; +} + +static int kv_upload_dpm_settings(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, GraphicsLevel), + (u8 *)&pi->graphics_level, + sizeof(SMU7_Fusion_GraphicsLevel) * SMU7_MAX_LEVELS_GRAPHICS, + pi->sram_end); + + if (ret) + return ret; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, GraphicsDpmLevelCount), + &pi->graphics_dpm_level_count, + sizeof(u8), pi->sram_end); + + return ret; +} + +static u32 kv_get_clock_difference(u32 a, u32 b) +{ + return (a >= b) ? a - b : b - a; +} + +static u32 kv_get_clk_bypass(struct radeon_device *rdev, u32 clk) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 value; + + if (pi->caps_enable_dfs_bypass) { + if (kv_get_clock_difference(clk, 40000) < 200) + value = 3; + else if (kv_get_clock_difference(clk, 30000) < 200) + value = 2; + else if (kv_get_clock_difference(clk, 20000) < 200) + value = 7; + else if (kv_get_clock_difference(clk, 15000) < 200) + value = 6; + else if (kv_get_clock_difference(clk, 10000) < 200) + value = 8; + else + value = 0; + } else { + value = 0; + } + + return value; +} + +static int kv_populate_uvd_table(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_uvd_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table; + struct atom_clock_dividers dividers; + int ret; + u32 i; + + if (table == NULL || table->count == 0) + return 0; + + pi->uvd_level_count = 0; + for (i = 0; i < table->count; i++) { + if (pi->high_voltage_t && + (pi->high_voltage_t < table->entries[i].v)) + break; + + pi->uvd_level[i].VclkFrequency = cpu_to_be32(table->entries[i].vclk); + pi->uvd_level[i].DclkFrequency = cpu_to_be32(table->entries[i].dclk); + pi->uvd_level[i].MinVddNb = cpu_to_be16(table->entries[i].v); + + pi->uvd_level[i].VClkBypassCntl = + (u8)kv_get_clk_bypass(rdev, table->entries[i].vclk); + pi->uvd_level[i].DClkBypassCntl = + (u8)kv_get_clk_bypass(rdev, table->entries[i].dclk); + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + table->entries[i].vclk, false, ÷rs); + if (ret) + return ret; + pi->uvd_level[i].VclkDivider = (u8)dividers.post_div; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + table->entries[i].dclk, false, ÷rs); + if (ret) + return ret; + pi->uvd_level[i].DclkDivider = (u8)dividers.post_div; + + pi->uvd_level_count++; + } + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, UvdLevelCount), + (u8 *)&pi->uvd_level_count, + sizeof(u8), pi->sram_end); + if (ret) + return ret; + + pi->uvd_interval = 1; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, UVDInterval), + &pi->uvd_interval, + sizeof(u8), pi->sram_end); + if (ret) + return ret; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, UvdLevel), + (u8 *)&pi->uvd_level, + sizeof(SMU7_Fusion_UvdLevel) * SMU7_MAX_LEVELS_UVD, + pi->sram_end); + + return ret; + +} + +static int kv_populate_vce_table(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret; + u32 i; + struct radeon_vce_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table; + struct atom_clock_dividers dividers; + + if (table == NULL || table->count == 0) + return 0; + + pi->vce_level_count = 0; + for (i = 0; i < table->count; i++) { + if (pi->high_voltage_t && + pi->high_voltage_t < table->entries[i].v) + break; + + pi->vce_level[i].Frequency = cpu_to_be32(table->entries[i].evclk); + pi->vce_level[i].MinVoltage = cpu_to_be16(table->entries[i].v); + + pi->vce_level[i].ClkBypassCntl = + (u8)kv_get_clk_bypass(rdev, table->entries[i].evclk); + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + table->entries[i].evclk, false, ÷rs); + if (ret) + return ret; + pi->vce_level[i].Divider = (u8)dividers.post_div; + + pi->vce_level_count++; + } + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, VceLevelCount), + (u8 *)&pi->vce_level_count, + sizeof(u8), + pi->sram_end); + if (ret) + return ret; + + pi->vce_interval = 1; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, VCEInterval), + (u8 *)&pi->vce_interval, + sizeof(u8), + pi->sram_end); + if (ret) + return ret; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, VceLevel), + (u8 *)&pi->vce_level, + sizeof(SMU7_Fusion_ExtClkLevel) * SMU7_MAX_LEVELS_VCE, + pi->sram_end); + + return ret; +} + +static int kv_populate_samu_table(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table; + struct atom_clock_dividers dividers; + int ret; + u32 i; + + if (table == NULL || table->count == 0) + return 0; + + pi->samu_level_count = 0; + for (i = 0; i < table->count; i++) { + if (pi->high_voltage_t && + pi->high_voltage_t < table->entries[i].v) + break; + + pi->samu_level[i].Frequency = cpu_to_be32(table->entries[i].clk); + pi->samu_level[i].MinVoltage = cpu_to_be16(table->entries[i].v); + + pi->samu_level[i].ClkBypassCntl = + (u8)kv_get_clk_bypass(rdev, table->entries[i].clk); + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + table->entries[i].clk, false, ÷rs); + if (ret) + return ret; + pi->samu_level[i].Divider = (u8)dividers.post_div; + + pi->samu_level_count++; + } + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, SamuLevelCount), + (u8 *)&pi->samu_level_count, + sizeof(u8), + pi->sram_end); + if (ret) + return ret; + + pi->samu_interval = 1; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, SAMUInterval), + (u8 *)&pi->samu_interval, + sizeof(u8), + pi->sram_end); + if (ret) + return ret; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, SamuLevel), + (u8 *)&pi->samu_level, + sizeof(SMU7_Fusion_ExtClkLevel) * SMU7_MAX_LEVELS_SAMU, + pi->sram_end); + if (ret) + return ret; + + return ret; +} + + +static int kv_populate_acp_table(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table; + struct atom_clock_dividers dividers; + int ret; + u32 i; + + if (table == NULL || table->count == 0) + return 0; + + pi->acp_level_count = 0; + for (i = 0; i < table->count; i++) { + pi->acp_level[i].Frequency = cpu_to_be32(table->entries[i].clk); + pi->acp_level[i].MinVoltage = cpu_to_be16(table->entries[i].v); + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + table->entries[i].clk, false, ÷rs); + if (ret) + return ret; + pi->acp_level[i].Divider = (u8)dividers.post_div; + + pi->acp_level_count++; + } + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, AcpLevelCount), + (u8 *)&pi->acp_level_count, + sizeof(u8), + pi->sram_end); + if (ret) + return ret; + + pi->acp_interval = 1; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, ACPInterval), + (u8 *)&pi->acp_interval, + sizeof(u8), + pi->sram_end); + if (ret) + return ret; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, AcpLevel), + (u8 *)&pi->acp_level, + sizeof(SMU7_Fusion_ExtClkLevel) * SMU7_MAX_LEVELS_ACP, + pi->sram_end); + if (ret) + return ret; + + return ret; +} + +static void kv_calculate_dfs_bypass_settings(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 i; + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + + if (table && table->count) { + for (i = 0; i < pi->graphics_dpm_level_count; i++) { + if (pi->caps_enable_dfs_bypass) { + if (kv_get_clock_difference(table->entries[i].clk, 40000) < 200) + pi->graphics_level[i].ClkBypassCntl = 3; + else if (kv_get_clock_difference(table->entries[i].clk, 30000) < 200) + pi->graphics_level[i].ClkBypassCntl = 2; + else if (kv_get_clock_difference(table->entries[i].clk, 26600) < 200) + pi->graphics_level[i].ClkBypassCntl = 7; + else if (kv_get_clock_difference(table->entries[i].clk , 20000) < 200) + pi->graphics_level[i].ClkBypassCntl = 6; + else if (kv_get_clock_difference(table->entries[i].clk , 10000) < 200) + pi->graphics_level[i].ClkBypassCntl = 8; + else + pi->graphics_level[i].ClkBypassCntl = 0; + } else { + pi->graphics_level[i].ClkBypassCntl = 0; + } + } + } else { + struct sumo_sclk_voltage_mapping_table *table = + &pi->sys_info.sclk_voltage_mapping_table; + for (i = 0; i < pi->graphics_dpm_level_count; i++) { + if (pi->caps_enable_dfs_bypass) { + if (kv_get_clock_difference(table->entries[i].sclk_frequency, 40000) < 200) + pi->graphics_level[i].ClkBypassCntl = 3; + else if (kv_get_clock_difference(table->entries[i].sclk_frequency, 30000) < 200) + pi->graphics_level[i].ClkBypassCntl = 2; + else if (kv_get_clock_difference(table->entries[i].sclk_frequency, 26600) < 200) + pi->graphics_level[i].ClkBypassCntl = 7; + else if (kv_get_clock_difference(table->entries[i].sclk_frequency, 20000) < 200) + pi->graphics_level[i].ClkBypassCntl = 6; + else if (kv_get_clock_difference(table->entries[i].sclk_frequency, 10000) < 200) + pi->graphics_level[i].ClkBypassCntl = 8; + else + pi->graphics_level[i].ClkBypassCntl = 0; + } else { + pi->graphics_level[i].ClkBypassCntl = 0; + } + } + } +} + +static int kv_enable_ulv(struct radeon_device *rdev, bool enable) +{ + return kv_notify_message_to_smu(rdev, enable ? + PPSMC_MSG_EnableULV : PPSMC_MSG_DisableULV); +} + +static void kv_reset_acp_boot_level(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->acp_boot_level = 0xff; +} + +static void kv_update_current_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct kv_ps *new_ps = kv_get_ps(rps); + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->current_rps = *rps; + pi->current_ps = *new_ps; + pi->current_rps.ps_priv = &pi->current_ps; +} + +static void kv_update_requested_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct kv_ps *new_ps = kv_get_ps(rps); + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->requested_rps = *rps; + pi->requested_ps = *new_ps; + pi->requested_rps.ps_priv = &pi->requested_ps; +} + +void kv_dpm_enable_bapm(struct radeon_device *rdev, bool enable) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret; + + if (pi->bapm_enable) { + ret = kv_smc_bapm_enable(rdev, enable); + if (ret) + DRM_ERROR("kv_smc_bapm_enable failed\n"); + } +} + +static void kv_enable_thermal_int(struct radeon_device *rdev, bool enable) +{ + u32 thermal_int; + + thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL); + if (enable) + thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK; + else + thermal_int &= ~(THERM_INTH_MASK | THERM_INTL_MASK); + WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int); + +} + +int kv_dpm_enable(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret; + + ret = kv_process_firmware_header(rdev); + if (ret) { + DRM_ERROR("kv_process_firmware_header failed\n"); + return ret; + } + kv_init_fps_limits(rdev); + kv_init_graphics_levels(rdev); + ret = kv_program_bootup_state(rdev); + if (ret) { + DRM_ERROR("kv_program_bootup_state failed\n"); + return ret; + } + kv_calculate_dfs_bypass_settings(rdev); + ret = kv_upload_dpm_settings(rdev); + if (ret) { + DRM_ERROR("kv_upload_dpm_settings failed\n"); + return ret; + } + ret = kv_populate_uvd_table(rdev); + if (ret) { + DRM_ERROR("kv_populate_uvd_table failed\n"); + return ret; + } + ret = kv_populate_vce_table(rdev); + if (ret) { + DRM_ERROR("kv_populate_vce_table failed\n"); + return ret; + } + ret = kv_populate_samu_table(rdev); + if (ret) { + DRM_ERROR("kv_populate_samu_table failed\n"); + return ret; + } + ret = kv_populate_acp_table(rdev); + if (ret) { + DRM_ERROR("kv_populate_acp_table failed\n"); + return ret; + } + kv_program_vc(rdev); +#if 0 + kv_initialize_hardware_cac_manager(rdev); +#endif + kv_start_am(rdev); + if (pi->enable_auto_thermal_throttling) { + ret = kv_enable_auto_thermal_throttling(rdev); + if (ret) { + DRM_ERROR("kv_enable_auto_thermal_throttling failed\n"); + return ret; + } + } + ret = kv_enable_dpm_voltage_scaling(rdev); + if (ret) { + DRM_ERROR("kv_enable_dpm_voltage_scaling failed\n"); + return ret; + } + ret = kv_set_dpm_interval(rdev); + if (ret) { + DRM_ERROR("kv_set_dpm_interval failed\n"); + return ret; + } + ret = kv_set_dpm_boot_state(rdev); + if (ret) { + DRM_ERROR("kv_set_dpm_boot_state failed\n"); + return ret; + } + ret = kv_enable_ulv(rdev, true); + if (ret) { + DRM_ERROR("kv_enable_ulv failed\n"); + return ret; + } + kv_start_dpm(rdev); + ret = kv_enable_didt(rdev, true); + if (ret) { + DRM_ERROR("kv_enable_didt failed\n"); + return ret; + } + ret = kv_enable_smc_cac(rdev, true); + if (ret) { + DRM_ERROR("kv_enable_smc_cac failed\n"); + return ret; + } + + kv_reset_acp_boot_level(rdev); + + ret = kv_smc_bapm_enable(rdev, false); + if (ret) { + DRM_ERROR("kv_smc_bapm_enable failed\n"); + return ret; + } + + kv_update_current_ps(rdev, rdev->pm.dpm.boot_ps); + + return ret; +} + +int kv_dpm_late_enable(struct radeon_device *rdev) +{ + int ret = 0; + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + ret = kv_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) { + DRM_ERROR("kv_set_thermal_temperature_range failed\n"); + return ret; + } + kv_enable_thermal_int(rdev, true); + } + + /* powerdown unused blocks for now */ + kv_dpm_powergate_acp(rdev, true); + kv_dpm_powergate_samu(rdev, true); + kv_dpm_powergate_vce(rdev, true); + kv_dpm_powergate_uvd(rdev, true); + + return ret; +} + +void kv_dpm_disable(struct radeon_device *rdev) +{ + kv_smc_bapm_enable(rdev, false); + + if (rdev->family == CHIP_MULLINS) + kv_enable_nb_dpm(rdev, false); + + /* powerup blocks */ + kv_dpm_powergate_acp(rdev, false); + kv_dpm_powergate_samu(rdev, false); + kv_dpm_powergate_vce(rdev, false); + kv_dpm_powergate_uvd(rdev, false); + + kv_enable_smc_cac(rdev, false); + kv_enable_didt(rdev, false); + kv_clear_vc(rdev); + kv_stop_dpm(rdev); + kv_enable_ulv(rdev, false); + kv_reset_am(rdev); + kv_enable_thermal_int(rdev, false); + + kv_update_current_ps(rdev, rdev->pm.dpm.boot_ps); +} + +#if 0 +static int kv_write_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 value) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + return kv_copy_bytes_to_smc(rdev, pi->soft_regs_start + reg_offset, + (u8 *)&value, sizeof(u16), pi->sram_end); +} + +static int kv_read_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 *value) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + return kv_read_smc_sram_dword(rdev, pi->soft_regs_start + reg_offset, + value, pi->sram_end); +} +#endif + +static void kv_init_sclk_t(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->low_sclk_interrupt_t = 0; +} + +static int kv_init_fps_limits(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret = 0; + + if (pi->caps_fps) { + u16 tmp; + + tmp = 45; + pi->fps_high_t = cpu_to_be16(tmp); + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, FpsHighT), + (u8 *)&pi->fps_high_t, + sizeof(u16), pi->sram_end); + + tmp = 30; + pi->fps_low_t = cpu_to_be16(tmp); + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, FpsLowT), + (u8 *)&pi->fps_low_t, + sizeof(u16), pi->sram_end); + + } + return ret; +} + +static void kv_init_powergate_state(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->uvd_power_gated = false; + pi->vce_power_gated = false; + pi->samu_power_gated = false; + pi->acp_power_gated = false; + +} + +static int kv_enable_uvd_dpm(struct radeon_device *rdev, bool enable) +{ + return kv_notify_message_to_smu(rdev, enable ? + PPSMC_MSG_UVDDPM_Enable : PPSMC_MSG_UVDDPM_Disable); +} + +static int kv_enable_vce_dpm(struct radeon_device *rdev, bool enable) +{ + return kv_notify_message_to_smu(rdev, enable ? + PPSMC_MSG_VCEDPM_Enable : PPSMC_MSG_VCEDPM_Disable); +} + +static int kv_enable_samu_dpm(struct radeon_device *rdev, bool enable) +{ + return kv_notify_message_to_smu(rdev, enable ? + PPSMC_MSG_SAMUDPM_Enable : PPSMC_MSG_SAMUDPM_Disable); +} + +static int kv_enable_acp_dpm(struct radeon_device *rdev, bool enable) +{ + return kv_notify_message_to_smu(rdev, enable ? + PPSMC_MSG_ACPDPM_Enable : PPSMC_MSG_ACPDPM_Disable); +} + +static int kv_update_uvd_dpm(struct radeon_device *rdev, bool gate) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_uvd_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table; + int ret; + u32 mask; + + if (!gate) { + if (table->count) + pi->uvd_boot_level = table->count - 1; + else + pi->uvd_boot_level = 0; + + if (!pi->caps_uvd_dpm || pi->caps_stable_p_state) { + mask = 1 << pi->uvd_boot_level; + } else { + mask = 0x1f; + } + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, UvdBootLevel), + (uint8_t *)&pi->uvd_boot_level, + sizeof(u8), pi->sram_end); + if (ret) + return ret; + + kv_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_UVDDPM_SetEnabledMask, + mask); + } + + return kv_enable_uvd_dpm(rdev, !gate); +} + +static u8 kv_get_vce_boot_level(struct radeon_device *rdev, u32 evclk) +{ + u8 i; + struct radeon_vce_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table; + + for (i = 0; i < table->count; i++) { + if (table->entries[i].evclk >= evclk) + break; + } + + return i; +} + +static int kv_update_vce_dpm(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_vce_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table; + int ret; + + if (radeon_new_state->evclk > 0 && radeon_current_state->evclk == 0) { + kv_dpm_powergate_vce(rdev, false); + /* turn the clocks on when encoding */ + cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, false); + if (pi->caps_stable_p_state) + pi->vce_boot_level = table->count - 1; + else + pi->vce_boot_level = kv_get_vce_boot_level(rdev, radeon_new_state->evclk); + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, VceBootLevel), + (u8 *)&pi->vce_boot_level, + sizeof(u8), + pi->sram_end); + if (ret) + return ret; + + if (pi->caps_stable_p_state) + kv_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_VCEDPM_SetEnabledMask, + (1 << pi->vce_boot_level)); + + kv_enable_vce_dpm(rdev, true); + } else if (radeon_new_state->evclk == 0 && radeon_current_state->evclk > 0) { + kv_enable_vce_dpm(rdev, false); + /* turn the clocks off when not encoding */ + cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, true); + kv_dpm_powergate_vce(rdev, true); + } + + return 0; +} + +static int kv_update_samu_dpm(struct radeon_device *rdev, bool gate) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table; + int ret; + + if (!gate) { + if (pi->caps_stable_p_state) + pi->samu_boot_level = table->count - 1; + else + pi->samu_boot_level = 0; + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, SamuBootLevel), + (u8 *)&pi->samu_boot_level, + sizeof(u8), + pi->sram_end); + if (ret) + return ret; + + if (pi->caps_stable_p_state) + kv_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_SAMUDPM_SetEnabledMask, + (1 << pi->samu_boot_level)); + } + + return kv_enable_samu_dpm(rdev, !gate); +} + +static u8 kv_get_acp_boot_level(struct radeon_device *rdev) +{ + u8 i; + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table; + + for (i = 0; i < table->count; i++) { + if (table->entries[i].clk >= 0) /* XXX */ + break; + } + + if (i >= table->count) + i = table->count - 1; + + return i; +} + +static void kv_update_acp_boot_level(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u8 acp_boot_level; + + if (!pi->caps_stable_p_state) { + acp_boot_level = kv_get_acp_boot_level(rdev); + if (acp_boot_level != pi->acp_boot_level) { + pi->acp_boot_level = acp_boot_level; + kv_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_ACPDPM_SetEnabledMask, + (1 << pi->acp_boot_level)); + } + } +} + +static int kv_update_acp_dpm(struct radeon_device *rdev, bool gate) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table; + int ret; + + if (!gate) { + if (pi->caps_stable_p_state) + pi->acp_boot_level = table->count - 1; + else + pi->acp_boot_level = kv_get_acp_boot_level(rdev); + + ret = kv_copy_bytes_to_smc(rdev, + pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, AcpBootLevel), + (u8 *)&pi->acp_boot_level, + sizeof(u8), + pi->sram_end); + if (ret) + return ret; + + if (pi->caps_stable_p_state) + kv_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_ACPDPM_SetEnabledMask, + (1 << pi->acp_boot_level)); + } + + return kv_enable_acp_dpm(rdev, !gate); +} + +void kv_dpm_powergate_uvd(struct radeon_device *rdev, bool gate) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + if (pi->uvd_power_gated == gate) + return; + + pi->uvd_power_gated = gate; + + if (gate) { + if (pi->caps_uvd_pg) { + uvd_v1_0_stop(rdev); + cik_update_cg(rdev, RADEON_CG_BLOCK_UVD, false); + } + kv_update_uvd_dpm(rdev, gate); + if (pi->caps_uvd_pg) + kv_notify_message_to_smu(rdev, PPSMC_MSG_UVDPowerOFF); + } else { + if (pi->caps_uvd_pg) { + kv_notify_message_to_smu(rdev, PPSMC_MSG_UVDPowerON); + uvd_v4_2_resume(rdev); + uvd_v1_0_start(rdev); + cik_update_cg(rdev, RADEON_CG_BLOCK_UVD, true); + } + kv_update_uvd_dpm(rdev, gate); + } +} + +static void kv_dpm_powergate_vce(struct radeon_device *rdev, bool gate) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + if (pi->vce_power_gated == gate) + return; + + pi->vce_power_gated = gate; + + if (gate) { + if (pi->caps_vce_pg) { + /* XXX do we need a vce_v1_0_stop() ? */ + kv_notify_message_to_smu(rdev, PPSMC_MSG_VCEPowerOFF); + } + } else { + if (pi->caps_vce_pg) { + kv_notify_message_to_smu(rdev, PPSMC_MSG_VCEPowerON); + vce_v2_0_resume(rdev); + vce_v1_0_start(rdev); + } + } +} + +static void kv_dpm_powergate_samu(struct radeon_device *rdev, bool gate) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + if (pi->samu_power_gated == gate) + return; + + pi->samu_power_gated = gate; + + if (gate) { + kv_update_samu_dpm(rdev, true); + if (pi->caps_samu_pg) + kv_notify_message_to_smu(rdev, PPSMC_MSG_SAMPowerOFF); + } else { + if (pi->caps_samu_pg) + kv_notify_message_to_smu(rdev, PPSMC_MSG_SAMPowerON); + kv_update_samu_dpm(rdev, false); + } +} + +static void kv_dpm_powergate_acp(struct radeon_device *rdev, bool gate) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + if (pi->acp_power_gated == gate) + return; + + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) + return; + + pi->acp_power_gated = gate; + + if (gate) { + kv_update_acp_dpm(rdev, true); + if (pi->caps_acp_pg) + kv_notify_message_to_smu(rdev, PPSMC_MSG_ACPPowerOFF); + } else { + if (pi->caps_acp_pg) + kv_notify_message_to_smu(rdev, PPSMC_MSG_ACPPowerON); + kv_update_acp_dpm(rdev, false); + } +} + +static void kv_set_valid_clock_range(struct radeon_device *rdev, + struct radeon_ps *new_rps) +{ + struct kv_ps *new_ps = kv_get_ps(new_rps); + struct kv_power_info *pi = kv_get_pi(rdev); + u32 i; + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + + if (table && table->count) { + for (i = 0; i < pi->graphics_dpm_level_count; i++) { + if ((table->entries[i].clk >= new_ps->levels[0].sclk) || + (i == (pi->graphics_dpm_level_count - 1))) { + pi->lowest_valid = i; + break; + } + } + + for (i = pi->graphics_dpm_level_count - 1; i > 0; i--) { + if (table->entries[i].clk <= new_ps->levels[new_ps->num_levels - 1].sclk) + break; + } + pi->highest_valid = i; + + if (pi->lowest_valid > pi->highest_valid) { + if ((new_ps->levels[0].sclk - table->entries[pi->highest_valid].clk) > + (table->entries[pi->lowest_valid].clk - new_ps->levels[new_ps->num_levels - 1].sclk)) + pi->highest_valid = pi->lowest_valid; + else + pi->lowest_valid = pi->highest_valid; + } + } else { + struct sumo_sclk_voltage_mapping_table *table = + &pi->sys_info.sclk_voltage_mapping_table; + + for (i = 0; i < (int)pi->graphics_dpm_level_count; i++) { + if (table->entries[i].sclk_frequency >= new_ps->levels[0].sclk || + i == (int)(pi->graphics_dpm_level_count - 1)) { + pi->lowest_valid = i; + break; + } + } + + for (i = pi->graphics_dpm_level_count - 1; i > 0; i--) { + if (table->entries[i].sclk_frequency <= + new_ps->levels[new_ps->num_levels - 1].sclk) + break; + } + pi->highest_valid = i; + + if (pi->lowest_valid > pi->highest_valid) { + if ((new_ps->levels[0].sclk - + table->entries[pi->highest_valid].sclk_frequency) > + (table->entries[pi->lowest_valid].sclk_frequency - + new_ps->levels[new_ps->num_levels -1].sclk)) + pi->highest_valid = pi->lowest_valid; + else + pi->lowest_valid = pi->highest_valid; + } + } +} + +static int kv_update_dfs_bypass_settings(struct radeon_device *rdev, + struct radeon_ps *new_rps) +{ + struct kv_ps *new_ps = kv_get_ps(new_rps); + struct kv_power_info *pi = kv_get_pi(rdev); + int ret = 0; + u8 clk_bypass_cntl; + + if (pi->caps_enable_dfs_bypass) { + clk_bypass_cntl = new_ps->need_dfs_bypass ? + pi->graphics_level[pi->graphics_boot_level].ClkBypassCntl : 0; + ret = kv_copy_bytes_to_smc(rdev, + (pi->dpm_table_start + + offsetof(SMU7_Fusion_DpmTable, GraphicsLevel) + + (pi->graphics_boot_level * sizeof(SMU7_Fusion_GraphicsLevel)) + + offsetof(SMU7_Fusion_GraphicsLevel, ClkBypassCntl)), + &clk_bypass_cntl, + sizeof(u8), pi->sram_end); + } + + return ret; +} + +static int kv_enable_nb_dpm(struct radeon_device *rdev, + bool enable) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + int ret = 0; + + if (enable) { + if (pi->enable_nb_dpm && !pi->nb_dpm_enabled) { + ret = kv_notify_message_to_smu(rdev, PPSMC_MSG_NBDPM_Enable); + if (ret == 0) + pi->nb_dpm_enabled = true; + } + } else { + if (pi->enable_nb_dpm && pi->nb_dpm_enabled) { + ret = kv_notify_message_to_smu(rdev, PPSMC_MSG_NBDPM_Disable); + if (ret == 0) + pi->nb_dpm_enabled = false; + } + } + + return ret; +} + +int kv_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level) +{ + int ret; + + if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + ret = kv_force_dpm_highest(rdev); + if (ret) + return ret; + } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { + ret = kv_force_dpm_lowest(rdev); + if (ret) + return ret; + } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) { + ret = kv_unforce_levels(rdev); + if (ret) + return ret; + } + + rdev->pm.dpm.forced_level = level; + + return 0; +} + +int kv_dpm_pre_set_power_state(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps; + struct radeon_ps *new_ps = &requested_ps; + + kv_update_requested_ps(rdev, new_ps); + + kv_apply_state_adjust_rules(rdev, + &pi->requested_rps, + &pi->current_rps); + + return 0; +} + +int kv_dpm_set_power_state(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_ps *new_ps = &pi->requested_rps; + struct radeon_ps *old_ps = &pi->current_rps; + int ret; + + if (pi->bapm_enable) { + ret = kv_smc_bapm_enable(rdev, rdev->pm.dpm.ac_power); + if (ret) { + DRM_ERROR("kv_smc_bapm_enable failed\n"); + return ret; + } + } + + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) { + if (pi->enable_dpm) { + kv_set_valid_clock_range(rdev, new_ps); + kv_update_dfs_bypass_settings(rdev, new_ps); + ret = kv_calculate_ds_divider(rdev); + if (ret) { + DRM_ERROR("kv_calculate_ds_divider failed\n"); + return ret; + } + kv_calculate_nbps_level_settings(rdev); + kv_calculate_dpm_settings(rdev); + kv_force_lowest_valid(rdev); + kv_enable_new_levels(rdev); + kv_upload_dpm_settings(rdev); + kv_program_nbps_index_settings(rdev, new_ps); + kv_unforce_levels(rdev); + kv_set_enabled_levels(rdev); + kv_force_lowest_valid(rdev); + kv_unforce_levels(rdev); + + ret = kv_update_vce_dpm(rdev, new_ps, old_ps); + if (ret) { + DRM_ERROR("kv_update_vce_dpm failed\n"); + return ret; + } + kv_update_sclk_t(rdev); + if (rdev->family == CHIP_MULLINS) + kv_enable_nb_dpm(rdev, true); + } + } else { + if (pi->enable_dpm) { + kv_set_valid_clock_range(rdev, new_ps); + kv_update_dfs_bypass_settings(rdev, new_ps); + ret = kv_calculate_ds_divider(rdev); + if (ret) { + DRM_ERROR("kv_calculate_ds_divider failed\n"); + return ret; + } + kv_calculate_nbps_level_settings(rdev); + kv_calculate_dpm_settings(rdev); + kv_freeze_sclk_dpm(rdev, true); + kv_upload_dpm_settings(rdev); + kv_program_nbps_index_settings(rdev, new_ps); + kv_freeze_sclk_dpm(rdev, false); + kv_set_enabled_levels(rdev); + ret = kv_update_vce_dpm(rdev, new_ps, old_ps); + if (ret) { + DRM_ERROR("kv_update_vce_dpm failed\n"); + return ret; + } + kv_update_acp_boot_level(rdev); + kv_update_sclk_t(rdev); + kv_enable_nb_dpm(rdev, true); + } + } + + return 0; +} + +void kv_dpm_post_set_power_state(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_ps *new_ps = &pi->requested_rps; + + kv_update_current_ps(rdev, new_ps); +} + +void kv_dpm_setup_asic(struct radeon_device *rdev) +{ + sumo_take_smu_control(rdev, true); + kv_init_powergate_state(rdev); + kv_init_sclk_t(rdev); +} + +#if 0 +void kv_dpm_reset_asic(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) { + kv_force_lowest_valid(rdev); + kv_init_graphics_levels(rdev); + kv_program_bootup_state(rdev); + kv_upload_dpm_settings(rdev); + kv_force_lowest_valid(rdev); + kv_unforce_levels(rdev); + } else { + kv_init_graphics_levels(rdev); + kv_program_bootup_state(rdev); + kv_freeze_sclk_dpm(rdev, true); + kv_upload_dpm_settings(rdev); + kv_freeze_sclk_dpm(rdev, false); + kv_set_enabled_level(rdev, pi->graphics_boot_level); + } +} +#endif + +//XXX use sumo_dpm_display_configuration_changed + +static void kv_construct_max_power_limits_table(struct radeon_device *rdev, + struct radeon_clock_and_voltage_limits *table) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + if (pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries > 0) { + int idx = pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries - 1; + table->sclk = + pi->sys_info.sclk_voltage_mapping_table.entries[idx].sclk_frequency; + table->vddc = + kv_convert_2bit_index_to_voltage(rdev, + pi->sys_info.sclk_voltage_mapping_table.entries[idx].vid_2bit); + } + + table->mclk = pi->sys_info.nbp_memory_clock[0]; +} + +static void kv_patch_voltage_values(struct radeon_device *rdev) +{ + int i; + struct radeon_uvd_clock_voltage_dependency_table *uvd_table = + &rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table; + struct radeon_vce_clock_voltage_dependency_table *vce_table = + &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table; + struct radeon_clock_voltage_dependency_table *samu_table = + &rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table; + struct radeon_clock_voltage_dependency_table *acp_table = + &rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table; + + if (uvd_table->count) { + for (i = 0; i < uvd_table->count; i++) + uvd_table->entries[i].v = + kv_convert_8bit_index_to_voltage(rdev, + uvd_table->entries[i].v); + } + + if (vce_table->count) { + for (i = 0; i < vce_table->count; i++) + vce_table->entries[i].v = + kv_convert_8bit_index_to_voltage(rdev, + vce_table->entries[i].v); + } + + if (samu_table->count) { + for (i = 0; i < samu_table->count; i++) + samu_table->entries[i].v = + kv_convert_8bit_index_to_voltage(rdev, + samu_table->entries[i].v); + } + + if (acp_table->count) { + for (i = 0; i < acp_table->count; i++) + acp_table->entries[i].v = + kv_convert_8bit_index_to_voltage(rdev, + acp_table->entries[i].v); + } + +} + +static void kv_construct_boot_state(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->boot_pl.sclk = pi->sys_info.bootup_sclk; + pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index; + pi->boot_pl.ds_divider_index = 0; + pi->boot_pl.ss_divider_index = 0; + pi->boot_pl.allow_gnb_slow = 1; + pi->boot_pl.force_nbp_state = 0; + pi->boot_pl.display_wm = 0; + pi->boot_pl.vce_wm = 0; +} + +static int kv_force_dpm_highest(struct radeon_device *rdev) +{ + int ret; + u32 enable_mask, i; + + ret = kv_dpm_get_enable_mask(rdev, &enable_mask); + if (ret) + return ret; + + for (i = SMU7_MAX_LEVELS_GRAPHICS - 1; i > 0; i--) { + if (enable_mask & (1 << i)) + break; + } + + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) + return kv_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_DPM_ForceState, i); + else + return kv_set_enabled_level(rdev, i); +} + +static int kv_force_dpm_lowest(struct radeon_device *rdev) +{ + int ret; + u32 enable_mask, i; + + ret = kv_dpm_get_enable_mask(rdev, &enable_mask); + if (ret) + return ret; + + for (i = 0; i < SMU7_MAX_LEVELS_GRAPHICS; i++) { + if (enable_mask & (1 << i)) + break; + } + + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) + return kv_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_DPM_ForceState, i); + else + return kv_set_enabled_level(rdev, i); +} + +static u8 kv_get_sleep_divider_id_from_clock(struct radeon_device *rdev, + u32 sclk, u32 min_sclk_in_sr) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 i; + u32 temp; + u32 min = (min_sclk_in_sr > KV_MINIMUM_ENGINE_CLOCK) ? + min_sclk_in_sr : KV_MINIMUM_ENGINE_CLOCK; + + if (sclk < min) + return 0; + + if (!pi->caps_sclk_ds) + return 0; + + for (i = KV_MAX_DEEPSLEEP_DIVIDER_ID; i > 0; i--) { + temp = sclk / sumo_get_sleep_divider_from_id(i); + if (temp >= min) + break; + } + + return (u8)i; +} + +static int kv_get_high_voltage_limit(struct radeon_device *rdev, int *limit) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + int i; + + if (table && table->count) { + for (i = table->count - 1; i >= 0; i--) { + if (pi->high_voltage_t && + (kv_convert_8bit_index_to_voltage(rdev, table->entries[i].v) <= + pi->high_voltage_t)) { + *limit = i; + return 0; + } + } + } else { + struct sumo_sclk_voltage_mapping_table *table = + &pi->sys_info.sclk_voltage_mapping_table; + + for (i = table->num_max_dpm_entries - 1; i >= 0; i--) { + if (pi->high_voltage_t && + (kv_convert_2bit_index_to_voltage(rdev, table->entries[i].vid_2bit) <= + pi->high_voltage_t)) { + *limit = i; + return 0; + } + } + } + + *limit = 0; + return 0; +} + +static void kv_apply_state_adjust_rules(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct kv_ps *ps = kv_get_ps(new_rps); + struct kv_power_info *pi = kv_get_pi(rdev); + u32 min_sclk = 10000; /* ??? */ + u32 sclk, mclk = 0; + int i, limit; + bool force_high; + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + u32 stable_p_state_sclk = 0; + struct radeon_clock_and_voltage_limits *max_limits = + &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + + if (new_rps->vce_active) { + new_rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk; + new_rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk; + } else { + new_rps->evclk = 0; + new_rps->ecclk = 0; + } + + mclk = max_limits->mclk; + sclk = min_sclk; + + if (pi->caps_stable_p_state) { + stable_p_state_sclk = (max_limits->sclk * 75) / 100; + + for (i = table->count - 1; i >= 0; i++) { + if (stable_p_state_sclk >= table->entries[i].clk) { + stable_p_state_sclk = table->entries[i].clk; + break; + } + } + + if (i > 0) + stable_p_state_sclk = table->entries[0].clk; + + sclk = stable_p_state_sclk; + } + + if (new_rps->vce_active) { + if (sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk) + sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk; + } + + ps->need_dfs_bypass = true; + + for (i = 0; i < ps->num_levels; i++) { + if (ps->levels[i].sclk < sclk) + ps->levels[i].sclk = sclk; + } + + if (table && table->count) { + for (i = 0; i < ps->num_levels; i++) { + if (pi->high_voltage_t && + (pi->high_voltage_t < + kv_convert_8bit_index_to_voltage(rdev, ps->levels[i].vddc_index))) { + kv_get_high_voltage_limit(rdev, &limit); + ps->levels[i].sclk = table->entries[limit].clk; + } + } + } else { + struct sumo_sclk_voltage_mapping_table *table = + &pi->sys_info.sclk_voltage_mapping_table; + + for (i = 0; i < ps->num_levels; i++) { + if (pi->high_voltage_t && + (pi->high_voltage_t < + kv_convert_8bit_index_to_voltage(rdev, ps->levels[i].vddc_index))) { + kv_get_high_voltage_limit(rdev, &limit); + ps->levels[i].sclk = table->entries[limit].sclk_frequency; + } + } + } + + if (pi->caps_stable_p_state) { + for (i = 0; i < ps->num_levels; i++) { + ps->levels[i].sclk = stable_p_state_sclk; + } + } + + pi->video_start = new_rps->dclk || new_rps->vclk || + new_rps->evclk || new_rps->ecclk; + + if ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == + ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) + pi->battery_state = true; + else + pi->battery_state = false; + + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) { + ps->dpm0_pg_nb_ps_lo = 0x1; + ps->dpm0_pg_nb_ps_hi = 0x0; + ps->dpmx_nb_ps_lo = 0x1; + ps->dpmx_nb_ps_hi = 0x0; + } else { + ps->dpm0_pg_nb_ps_lo = 0x3; + ps->dpm0_pg_nb_ps_hi = 0x0; + ps->dpmx_nb_ps_lo = 0x3; + ps->dpmx_nb_ps_hi = 0x0; + + if (pi->sys_info.nb_dpm_enable) { + force_high = (mclk >= pi->sys_info.nbp_memory_clock[3]) || + pi->video_start || (rdev->pm.dpm.new_active_crtc_count >= 3) || + pi->disable_nb_ps3_in_battery; + ps->dpm0_pg_nb_ps_lo = force_high ? 0x2 : 0x3; + ps->dpm0_pg_nb_ps_hi = 0x2; + ps->dpmx_nb_ps_lo = force_high ? 0x2 : 0x3; + ps->dpmx_nb_ps_hi = 0x2; + } + } +} + +static void kv_dpm_power_level_enabled_for_throttle(struct radeon_device *rdev, + u32 index, bool enable) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + pi->graphics_level[index].EnabledForThrottle = enable ? 1 : 0; +} + +static int kv_calculate_ds_divider(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 sclk_in_sr = 10000; /* ??? */ + u32 i; + + if (pi->lowest_valid > pi->highest_valid) + return -EINVAL; + + for (i = pi->lowest_valid; i <= pi->highest_valid; i++) { + pi->graphics_level[i].DeepSleepDivId = + kv_get_sleep_divider_id_from_clock(rdev, + be32_to_cpu(pi->graphics_level[i].SclkFrequency), + sclk_in_sr); + } + return 0; +} + +static int kv_calculate_nbps_level_settings(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 i; + bool force_high; + struct radeon_clock_and_voltage_limits *max_limits = + &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + u32 mclk = max_limits->mclk; + + if (pi->lowest_valid > pi->highest_valid) + return -EINVAL; + + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) { + for (i = pi->lowest_valid; i <= pi->highest_valid; i++) { + pi->graphics_level[i].GnbSlow = 1; + pi->graphics_level[i].ForceNbPs1 = 0; + pi->graphics_level[i].UpH = 0; + } + + if (!pi->sys_info.nb_dpm_enable) + return 0; + + force_high = ((mclk >= pi->sys_info.nbp_memory_clock[3]) || + (rdev->pm.dpm.new_active_crtc_count >= 3) || pi->video_start); + + if (force_high) { + for (i = pi->lowest_valid; i <= pi->highest_valid; i++) + pi->graphics_level[i].GnbSlow = 0; + } else { + if (pi->battery_state) + pi->graphics_level[0].ForceNbPs1 = 1; + + pi->graphics_level[1].GnbSlow = 0; + pi->graphics_level[2].GnbSlow = 0; + pi->graphics_level[3].GnbSlow = 0; + pi->graphics_level[4].GnbSlow = 0; + } + } else { + for (i = pi->lowest_valid; i <= pi->highest_valid; i++) { + pi->graphics_level[i].GnbSlow = 1; + pi->graphics_level[i].ForceNbPs1 = 0; + pi->graphics_level[i].UpH = 0; + } + + if (pi->sys_info.nb_dpm_enable && pi->battery_state) { + pi->graphics_level[pi->lowest_valid].UpH = 0x28; + pi->graphics_level[pi->lowest_valid].GnbSlow = 0; + if (pi->lowest_valid != pi->highest_valid) + pi->graphics_level[pi->lowest_valid].ForceNbPs1 = 1; + } + } + return 0; +} + +static int kv_calculate_dpm_settings(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 i; + + if (pi->lowest_valid > pi->highest_valid) + return -EINVAL; + + for (i = pi->lowest_valid; i <= pi->highest_valid; i++) + pi->graphics_level[i].DisplayWatermark = (i == pi->highest_valid) ? 1 : 0; + + return 0; +} + +static void kv_init_graphics_levels(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 i; + struct radeon_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk; + + if (table && table->count) { + u32 vid_2bit; + + pi->graphics_dpm_level_count = 0; + for (i = 0; i < table->count; i++) { + if (pi->high_voltage_t && + (pi->high_voltage_t < + kv_convert_8bit_index_to_voltage(rdev, table->entries[i].v))) + break; + + kv_set_divider_value(rdev, i, table->entries[i].clk); + vid_2bit = kv_convert_vid7_to_vid2(rdev, + &pi->sys_info.vid_mapping_table, + table->entries[i].v); + kv_set_vid(rdev, i, vid_2bit); + kv_set_at(rdev, i, pi->at[i]); + kv_dpm_power_level_enabled_for_throttle(rdev, i, true); + pi->graphics_dpm_level_count++; + } + } else { + struct sumo_sclk_voltage_mapping_table *table = + &pi->sys_info.sclk_voltage_mapping_table; + + pi->graphics_dpm_level_count = 0; + for (i = 0; i < table->num_max_dpm_entries; i++) { + if (pi->high_voltage_t && + pi->high_voltage_t < + kv_convert_2bit_index_to_voltage(rdev, table->entries[i].vid_2bit)) + break; + + kv_set_divider_value(rdev, i, table->entries[i].sclk_frequency); + kv_set_vid(rdev, i, table->entries[i].vid_2bit); + kv_set_at(rdev, i, pi->at[i]); + kv_dpm_power_level_enabled_for_throttle(rdev, i, true); + pi->graphics_dpm_level_count++; + } + } + + for (i = 0; i < SMU7_MAX_LEVELS_GRAPHICS; i++) + kv_dpm_power_level_enable(rdev, i, false); +} + +static void kv_enable_new_levels(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 i; + + for (i = 0; i < SMU7_MAX_LEVELS_GRAPHICS; i++) { + if (i >= pi->lowest_valid && i <= pi->highest_valid) + kv_dpm_power_level_enable(rdev, i, true); + } +} + +static int kv_set_enabled_level(struct radeon_device *rdev, u32 level) +{ + u32 new_mask = (1 << level); + + return kv_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_SCLKDPM_SetEnabledMask, + new_mask); +} + +static int kv_set_enabled_levels(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 i, new_mask = 0; + + for (i = pi->lowest_valid; i <= pi->highest_valid; i++) + new_mask |= (1 << i); + + return kv_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_SCLKDPM_SetEnabledMask, + new_mask); +} + +static void kv_program_nbps_index_settings(struct radeon_device *rdev, + struct radeon_ps *new_rps) +{ + struct kv_ps *new_ps = kv_get_ps(new_rps); + struct kv_power_info *pi = kv_get_pi(rdev); + u32 nbdpmconfig1; + + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) + return; + + if (pi->sys_info.nb_dpm_enable) { + nbdpmconfig1 = RREG32_SMC(NB_DPM_CONFIG_1); + nbdpmconfig1 &= ~(Dpm0PgNbPsLo_MASK | Dpm0PgNbPsHi_MASK | + DpmXNbPsLo_MASK | DpmXNbPsHi_MASK); + nbdpmconfig1 |= (Dpm0PgNbPsLo(new_ps->dpm0_pg_nb_ps_lo) | + Dpm0PgNbPsHi(new_ps->dpm0_pg_nb_ps_hi) | + DpmXNbPsLo(new_ps->dpmx_nb_ps_lo) | + DpmXNbPsHi(new_ps->dpmx_nb_ps_hi)); + WREG32_SMC(NB_DPM_CONFIG_1, nbdpmconfig1); + } +} + +static int kv_set_thermal_temperature_range(struct radeon_device *rdev, + int min_temp, int max_temp) +{ + int low_temp = 0 * 1000; + int high_temp = 255 * 1000; + u32 tmp; + + if (low_temp < min_temp) + low_temp = min_temp; + if (high_temp > max_temp) + high_temp = max_temp; + if (high_temp < low_temp) { + DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp); + return -EINVAL; + } + + tmp = RREG32_SMC(CG_THERMAL_INT_CTRL); + tmp &= ~(DIG_THERM_INTH_MASK | DIG_THERM_INTL_MASK); + tmp |= (DIG_THERM_INTH(49 + (high_temp / 1000)) | + DIG_THERM_INTL(49 + (low_temp / 1000))); + WREG32_SMC(CG_THERMAL_INT_CTRL, tmp); + + rdev->pm.dpm.thermal.min_temp = low_temp; + rdev->pm.dpm.thermal.max_temp = high_temp; + + return 0; +} + +union igp_info { + struct _ATOM_INTEGRATED_SYSTEM_INFO info; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_8 info_8; +}; + +static int kv_parse_sys_info_table(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct radeon_mode_info *mode_info = &rdev->mode_info; + int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo); + union igp_info *igp_info; + u8 frev, crev; + u16 data_offset; + int i; + + if (atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + igp_info = (union igp_info *)(mode_info->atom_context->bios + + data_offset); + + if (crev != 8) { + DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev); + return -EINVAL; + } + pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_8.ulBootUpEngineClock); + pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_8.ulBootUpUMAClock); + pi->sys_info.bootup_nb_voltage_index = + le16_to_cpu(igp_info->info_8.usBootUpNBVoltage); + if (igp_info->info_8.ucHtcTmpLmt == 0) + pi->sys_info.htc_tmp_lmt = 203; + else + pi->sys_info.htc_tmp_lmt = igp_info->info_8.ucHtcTmpLmt; + if (igp_info->info_8.ucHtcHystLmt == 0) + pi->sys_info.htc_hyst_lmt = 5; + else + pi->sys_info.htc_hyst_lmt = igp_info->info_8.ucHtcHystLmt; + if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) { + DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n"); + } + + if (le32_to_cpu(igp_info->info_8.ulSystemConfig) & (1 << 3)) + pi->sys_info.nb_dpm_enable = true; + else + pi->sys_info.nb_dpm_enable = false; + + for (i = 0; i < KV_NUM_NBPSTATES; i++) { + pi->sys_info.nbp_memory_clock[i] = + le32_to_cpu(igp_info->info_8.ulNbpStateMemclkFreq[i]); + pi->sys_info.nbp_n_clock[i] = + le32_to_cpu(igp_info->info_8.ulNbpStateNClkFreq[i]); + } + if (le32_to_cpu(igp_info->info_8.ulGPUCapInfo) & + SYS_INFO_GPUCAPS__ENABEL_DFS_BYPASS) + pi->caps_enable_dfs_bypass = true; + + sumo_construct_sclk_voltage_mapping_table(rdev, + &pi->sys_info.sclk_voltage_mapping_table, + igp_info->info_8.sAvail_SCLK); + + sumo_construct_vid_mapping_table(rdev, + &pi->sys_info.vid_mapping_table, + igp_info->info_8.sAvail_SCLK); + + kv_construct_max_power_limits_table(rdev, + &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac); + } + return 0; +} + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; +}; + +union pplib_clock_info { + struct _ATOM_PPLIB_R600_CLOCK_INFO r600; + struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; + struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; + struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; +}; + +union pplib_power_state { + struct _ATOM_PPLIB_STATE v1; + struct _ATOM_PPLIB_STATE_V2 v2; +}; + +static void kv_patch_boot_state(struct radeon_device *rdev, + struct kv_ps *ps) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + ps->num_levels = 1; + ps->levels[0] = pi->boot_pl; +} + +static void kv_parse_pplib_non_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info, + u8 table_rev) +{ + struct kv_ps *ps = kv_get_ps(rps); + + rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); + rps->class = le16_to_cpu(non_clock_info->usClassification); + rps->class2 = le16_to_cpu(non_clock_info->usClassification2); + + if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) { + rps->vclk = le32_to_cpu(non_clock_info->ulVCLK); + rps->dclk = le32_to_cpu(non_clock_info->ulDCLK); + } else { + rps->vclk = 0; + rps->dclk = 0; + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { + rdev->pm.dpm.boot_ps = rps; + kv_patch_boot_state(rdev, ps); + } + if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + rdev->pm.dpm.uvd_ps = rps; +} + +static void kv_parse_pplib_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, int index, + union pplib_clock_info *clock_info) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct kv_ps *ps = kv_get_ps(rps); + struct kv_pl *pl = &ps->levels[index]; + u32 sclk; + + sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow); + sclk |= clock_info->sumo.ucEngineClockHigh << 16; + pl->sclk = sclk; + pl->vddc_index = clock_info->sumo.vddcIndex; + + ps->num_levels = index + 1; + + if (pi->caps_sclk_ds) { + pl->ds_divider_index = 5; + pl->ss_divider_index = 5; + } +} + +static int kv_parse_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; + union pplib_power_state *power_state; + int i, j, k, non_clock_array_index, clock_array_index; + union pplib_clock_info *clock_info; + struct _StateArray *state_array; + struct _ClockInfoArray *clock_info_array; + struct _NonClockInfoArray *non_clock_info_array; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + u8 *power_state_offset; + struct kv_ps *ps; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + state_array = (struct _StateArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usStateArrayOffset)); + clock_info_array = (struct _ClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usClockInfoArrayOffset)); + non_clock_info_array = (struct _NonClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset)); + + rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) * + state_array->ucNumEntries, GFP_KERNEL); + if (!rdev->pm.dpm.ps) + return -ENOMEM; + power_state_offset = (u8 *)state_array->states; + for (i = 0; i < state_array->ucNumEntries; i++) { + u8 *idx; + power_state = (union pplib_power_state *)power_state_offset; + non_clock_array_index = power_state->v2.nonClockInfoIndex; + non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) + &non_clock_info_array->nonClockInfo[non_clock_array_index]; + if (!rdev->pm.power_state[i].clock_info) + return -EINVAL; + ps = kzalloc(sizeof(struct kv_ps), GFP_KERNEL); + if (ps == NULL) { + kfree(rdev->pm.dpm.ps); + return -ENOMEM; + } + rdev->pm.dpm.ps[i].ps_priv = ps; + k = 0; + idx = (u8 *)&power_state->v2.clockInfoIndex[0]; + for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) { + clock_array_index = idx[j]; + if (clock_array_index >= clock_info_array->ucNumEntries) + continue; + if (k >= SUMO_MAX_HARDWARE_POWERLEVELS) + break; + clock_info = (union pplib_clock_info *) + ((u8 *)&clock_info_array->clockInfo[0] + + (clock_array_index * clock_info_array->ucEntrySize)); + kv_parse_pplib_clock_info(rdev, + &rdev->pm.dpm.ps[i], k, + clock_info); + k++; + } + kv_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i], + non_clock_info, + non_clock_info_array->ucEntrySize); + power_state_offset += 2 + power_state->v2.ucNumDPMLevels; + } + rdev->pm.dpm.num_ps = state_array->ucNumEntries; + + /* fill in the vce power states */ + for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) { + u32 sclk; + clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx; + clock_info = (union pplib_clock_info *) + &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize]; + sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow); + sclk |= clock_info->sumo.ucEngineClockHigh << 16; + rdev->pm.dpm.vce_states[i].sclk = sclk; + rdev->pm.dpm.vce_states[i].mclk = 0; + } + + return 0; +} + +int kv_dpm_init(struct radeon_device *rdev) +{ + struct kv_power_info *pi; + int ret, i; + + pi = kzalloc(sizeof(struct kv_power_info), GFP_KERNEL); + if (pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = pi; + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = r600_parse_extended_power_table(rdev); + if (ret) + return ret; + + for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++) + pi->at[i] = TRINITY_AT_DFLT; + + pi->sram_end = SMC_RAM_END; + + /* Enabling nb dpm on an asrock system prevents dpm from working */ + if (rdev->pdev->subsystem_vendor == 0x1849) + pi->enable_nb_dpm = false; + else + pi->enable_nb_dpm = true; + + pi->caps_power_containment = true; + pi->caps_cac = true; + pi->enable_didt = false; + if (pi->enable_didt) { + pi->caps_sq_ramping = true; + pi->caps_db_ramping = true; + pi->caps_td_ramping = true; + pi->caps_tcp_ramping = true; + } + + pi->caps_sclk_ds = true; + pi->enable_auto_thermal_throttling = true; + pi->disable_nb_ps3_in_battery = false; + if (radeon_bapm == -1) { + /* only enable bapm on KB, ML by default */ + if (rdev->family == CHIP_KABINI || rdev->family == CHIP_MULLINS) + pi->bapm_enable = true; + else + pi->bapm_enable = false; + } else if (radeon_bapm == 0) { + pi->bapm_enable = false; + } else { + pi->bapm_enable = true; + } + pi->voltage_drop_t = 0; + pi->caps_sclk_throttle_low_notification = false; + pi->caps_fps = false; /* true? */ + pi->caps_uvd_pg = true; + pi->caps_uvd_dpm = true; + pi->caps_vce_pg = false; /* XXX true */ + pi->caps_samu_pg = false; + pi->caps_acp_pg = false; + pi->caps_stable_p_state = false; + + ret = kv_parse_sys_info_table(rdev); + if (ret) + return ret; + + kv_patch_voltage_values(rdev); + kv_construct_boot_state(rdev); + + ret = kv_parse_power_table(rdev); + if (ret) + return ret; + + pi->enable_dpm = true; + + return 0; +} + +void kv_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 current_index = + (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) & CURR_SCLK_INDEX_MASK) >> + CURR_SCLK_INDEX_SHIFT; + u32 sclk, tmp; + u16 vddc; + + if (current_index >= SMU__NUM_SCLK_DPM_STATE) { + seq_printf(m, "invalid dpm profile %d\n", current_index); + } else { + sclk = be32_to_cpu(pi->graphics_level[current_index].SclkFrequency); + tmp = (RREG32_SMC(SMU_VOLTAGE_STATUS) & SMU_VOLTAGE_CURRENT_LEVEL_MASK) >> + SMU_VOLTAGE_CURRENT_LEVEL_SHIFT; + vddc = kv_convert_8bit_index_to_voltage(rdev, (u16)tmp); + seq_printf(m, "uvd %sabled\n", pi->uvd_power_gated ? "dis" : "en"); + seq_printf(m, "vce %sabled\n", pi->vce_power_gated ? "dis" : "en"); + seq_printf(m, "power level %d sclk: %u vddc: %u\n", + current_index, sclk, vddc); + } +} + +u32 kv_dpm_get_current_sclk(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + u32 current_index = + (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) & CURR_SCLK_INDEX_MASK) >> + CURR_SCLK_INDEX_SHIFT; + u32 sclk; + + if (current_index >= SMU__NUM_SCLK_DPM_STATE) { + return 0; + } else { + sclk = be32_to_cpu(pi->graphics_level[current_index].SclkFrequency); + return sclk; + } +} + +u32 kv_dpm_get_current_mclk(struct radeon_device *rdev) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + return pi->sys_info.bootup_uma_clk; +} + +void kv_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + int i; + struct kv_ps *ps = kv_get_ps(rps); + + r600_dpm_print_class_info(rps->class, rps->class2); + r600_dpm_print_cap_info(rps->caps); + printk("\tuvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + for (i = 0; i < ps->num_levels; i++) { + struct kv_pl *pl = &ps->levels[i]; + printk("\t\tpower level %d sclk: %u vddc: %u\n", + i, pl->sclk, + kv_convert_8bit_index_to_voltage(rdev, pl->vddc_index)); + } + r600_dpm_print_ps_status(rdev, rps); +} + +void kv_dpm_fini(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); + r600_free_extended_power_table(rdev); +} + +void kv_dpm_display_configuration_changed(struct radeon_device *rdev) +{ + +} + +u32 kv_dpm_get_sclk(struct radeon_device *rdev, bool low) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + struct kv_ps *requested_state = kv_get_ps(&pi->requested_rps); + + if (low) + return requested_state->levels[0].sclk; + else + return requested_state->levels[requested_state->num_levels - 1].sclk; +} + +u32 kv_dpm_get_mclk(struct radeon_device *rdev, bool low) +{ + struct kv_power_info *pi = kv_get_pi(rdev); + + return pi->sys_info.bootup_uma_clk; +} + diff --git a/sys/dev/pci/drm/radeon/kv_dpm.h b/sys/dev/pci/drm/radeon/kv_dpm.h new file mode 100644 index 00000000000..8cef7525d7a --- /dev/null +++ b/sys/dev/pci/drm/radeon/kv_dpm.h @@ -0,0 +1,200 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __KV_DPM_H__ +#define __KV_DPM_H__ + +#define SMU__NUM_SCLK_DPM_STATE 8 +#define SMU__NUM_MCLK_DPM_LEVELS 4 +#define SMU__NUM_LCLK_DPM_LEVELS 8 +#define SMU__NUM_PCIE_DPM_LEVELS 0 /* ??? */ +#include "smu7_fusion.h" +#include "trinity_dpm.h" +#include "ppsmc.h" + +#define KV_NUM_NBPSTATES 4 + +enum kv_pt_config_reg_type { + KV_CONFIGREG_MMR = 0, + KV_CONFIGREG_SMC_IND, + KV_CONFIGREG_DIDT_IND, + KV_CONFIGREG_CACHE, + KV_CONFIGREG_MAX +}; + +struct kv_pt_config_reg { + u32 offset; + u32 mask; + u32 shift; + u32 value; + enum kv_pt_config_reg_type type; +}; + +struct kv_lcac_config_values { + u32 block_id; + u32 signal_id; + u32 t; +}; + +struct kv_lcac_config_reg { + u32 cntl; + u32 block_mask; + u32 block_shift; + u32 signal_mask; + u32 signal_shift; + u32 t_mask; + u32 t_shift; + u32 enable_mask; + u32 enable_shift; +}; + +struct kv_pl { + u32 sclk; + u8 vddc_index; + u8 ds_divider_index; + u8 ss_divider_index; + u8 allow_gnb_slow; + u8 force_nbp_state; + u8 display_wm; + u8 vce_wm; +}; + +struct kv_ps { + struct kv_pl levels[SUMO_MAX_HARDWARE_POWERLEVELS]; + u32 num_levels; + bool need_dfs_bypass; + u8 dpm0_pg_nb_ps_lo; + u8 dpm0_pg_nb_ps_hi; + u8 dpmx_nb_ps_lo; + u8 dpmx_nb_ps_hi; +}; + +struct kv_sys_info { + u32 bootup_uma_clk; + u32 bootup_sclk; + u32 dentist_vco_freq; + u32 nb_dpm_enable; + u32 nbp_memory_clock[KV_NUM_NBPSTATES]; + u32 nbp_n_clock[KV_NUM_NBPSTATES]; + u16 bootup_nb_voltage_index; + u8 htc_tmp_lmt; + u8 htc_hyst_lmt; + struct sumo_sclk_voltage_mapping_table sclk_voltage_mapping_table; + struct sumo_vid_mapping_table vid_mapping_table; + u32 uma_channel_number; +}; + +struct kv_power_info { + u32 at[SUMO_MAX_HARDWARE_POWERLEVELS]; + u32 voltage_drop_t; + struct kv_sys_info sys_info; + struct kv_pl boot_pl; + bool enable_nb_ps_policy; + bool disable_nb_ps3_in_battery; + bool video_start; + bool battery_state; + u32 lowest_valid; + u32 highest_valid; + u16 high_voltage_t; + bool cac_enabled; + bool bapm_enable; + /* smc offsets */ + u32 sram_end; + u32 dpm_table_start; + u32 soft_regs_start; + /* dpm SMU tables */ + u8 graphics_dpm_level_count; + u8 uvd_level_count; + u8 vce_level_count; + u8 acp_level_count; + u8 samu_level_count; + u16 fps_high_t; + SMU7_Fusion_GraphicsLevel graphics_level[SMU__NUM_SCLK_DPM_STATE]; + SMU7_Fusion_ACPILevel acpi_level; + SMU7_Fusion_UvdLevel uvd_level[SMU7_MAX_LEVELS_UVD]; + SMU7_Fusion_ExtClkLevel vce_level[SMU7_MAX_LEVELS_VCE]; + SMU7_Fusion_ExtClkLevel acp_level[SMU7_MAX_LEVELS_ACP]; + SMU7_Fusion_ExtClkLevel samu_level[SMU7_MAX_LEVELS_SAMU]; + u8 uvd_boot_level; + u8 vce_boot_level; + u8 acp_boot_level; + u8 samu_boot_level; + u8 uvd_interval; + u8 vce_interval; + u8 acp_interval; + u8 samu_interval; + u8 graphics_boot_level; + u8 graphics_interval; + u8 graphics_therm_throttle_enable; + u8 graphics_voltage_change_enable; + u8 graphics_clk_slow_enable; + u8 graphics_clk_slow_divider; + u8 fps_low_t; + u32 low_sclk_interrupt_t; + bool uvd_power_gated; + bool vce_power_gated; + bool acp_power_gated; + bool samu_power_gated; + bool nb_dpm_enabled; + /* flags */ + bool enable_didt; + bool enable_dpm; + bool enable_auto_thermal_throttling; + bool enable_nb_dpm; + /* caps */ + bool caps_cac; + bool caps_power_containment; + bool caps_sq_ramping; + bool caps_db_ramping; + bool caps_td_ramping; + bool caps_tcp_ramping; + bool caps_sclk_throttle_low_notification; + bool caps_fps; + bool caps_uvd_dpm; + bool caps_uvd_pg; + bool caps_vce_pg; + bool caps_samu_pg; + bool caps_acp_pg; + bool caps_stable_p_state; + bool caps_enable_dfs_bypass; + bool caps_sclk_ds; + struct radeon_ps current_rps; + struct kv_ps current_ps; + struct radeon_ps requested_rps; + struct kv_ps requested_ps; +}; + + +/* kv_smc.c */ +int kv_notify_message_to_smu(struct radeon_device *rdev, u32 id); +int kv_dpm_get_enable_mask(struct radeon_device *rdev, u32 *enable_mask); +int kv_send_msg_to_smc_with_parameter(struct radeon_device *rdev, + PPSMC_Msg msg, u32 parameter); +int kv_read_smc_sram_dword(struct radeon_device *rdev, u32 smc_address, + u32 *value, u32 limit); +int kv_smc_dpm_enable(struct radeon_device *rdev, bool enable); +int kv_smc_bapm_enable(struct radeon_device *rdev, bool enable); +int kv_copy_bytes_to_smc(struct radeon_device *rdev, + u32 smc_start_address, + const u8 *src, u32 byte_count, u32 limit); + +#endif diff --git a/sys/dev/pci/drm/radeon/kv_smc.c b/sys/dev/pci/drm/radeon/kv_smc.c new file mode 100644 index 00000000000..6077d426913 --- /dev/null +++ b/sys/dev/pci/drm/radeon/kv_smc.c @@ -0,0 +1,215 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "cikd.h" +#include "kv_dpm.h" + +int kv_notify_message_to_smu(struct radeon_device *rdev, u32 id) +{ + u32 i; + u32 tmp = 0; + + WREG32(SMC_MESSAGE_0, id & SMC_MSG_MASK); + + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(SMC_RESP_0) & SMC_RESP_MASK) != 0) + break; + udelay(1); + } + tmp = RREG32(SMC_RESP_0) & SMC_RESP_MASK; + + if (tmp != 1) { + if (tmp == 0xFF) + return -EINVAL; + else if (tmp == 0xFE) + return -EINVAL; + } + + return 0; +} + +int kv_dpm_get_enable_mask(struct radeon_device *rdev, u32 *enable_mask) +{ + int ret; + + ret = kv_notify_message_to_smu(rdev, PPSMC_MSG_SCLKDPM_GetEnabledMask); + + if (ret == 0) + *enable_mask = RREG32_SMC(SMC_SYSCON_MSG_ARG_0); + + return ret; +} + +int kv_send_msg_to_smc_with_parameter(struct radeon_device *rdev, + PPSMC_Msg msg, u32 parameter) +{ + + WREG32(SMC_MSG_ARG_0, parameter); + + return kv_notify_message_to_smu(rdev, msg); +} + +static int kv_set_smc_sram_address(struct radeon_device *rdev, + u32 smc_address, u32 limit) +{ + if (smc_address & 3) + return -EINVAL; + if ((smc_address + 3) > limit) + return -EINVAL; + + WREG32(SMC_IND_INDEX_0, smc_address); + WREG32_P(SMC_IND_ACCESS_CNTL, 0, ~AUTO_INCREMENT_IND_0); + + return 0; +} + +int kv_read_smc_sram_dword(struct radeon_device *rdev, u32 smc_address, + u32 *value, u32 limit) +{ + int ret; + + ret = kv_set_smc_sram_address(rdev, smc_address, limit); + if (ret) + return ret; + + *value = RREG32(SMC_IND_DATA_0); + return 0; +} + +int kv_smc_dpm_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + return kv_notify_message_to_smu(rdev, PPSMC_MSG_DPM_Enable); + else + return kv_notify_message_to_smu(rdev, PPSMC_MSG_DPM_Disable); +} + +int kv_smc_bapm_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + return kv_notify_message_to_smu(rdev, PPSMC_MSG_EnableBAPM); + else + return kv_notify_message_to_smu(rdev, PPSMC_MSG_DisableBAPM); +} + +int kv_copy_bytes_to_smc(struct radeon_device *rdev, + u32 smc_start_address, + const u8 *src, u32 byte_count, u32 limit) +{ + int ret; + u32 data, original_data, addr, extra_shift, t_byte, count, mask; + + if ((smc_start_address + byte_count) > limit) + return -EINVAL; + + addr = smc_start_address; + t_byte = addr & 3; + + /* RMW for the initial bytes */ + if (t_byte != 0) { + addr -= t_byte; + + ret = kv_set_smc_sram_address(rdev, addr, limit); + if (ret) + return ret; + + original_data = RREG32(SMC_IND_DATA_0); + + data = 0; + mask = 0; + count = 4; + while (count > 0) { + if (t_byte > 0) { + mask = (mask << 8) | 0xff; + t_byte--; + } else if (byte_count > 0) { + data = (data << 8) + *src++; + byte_count--; + mask <<= 8; + } else { + data <<= 8; + mask = (mask << 8) | 0xff; + } + count--; + } + + data |= original_data & mask; + + ret = kv_set_smc_sram_address(rdev, addr, limit); + if (ret) + return ret; + + WREG32(SMC_IND_DATA_0, data); + + addr += 4; + } + + while (byte_count >= 4) { + /* SMC address space is BE */ + data = (src[0] << 24) + (src[1] << 16) + (src[2] << 8) + src[3]; + + ret = kv_set_smc_sram_address(rdev, addr, limit); + if (ret) + return ret; + + WREG32(SMC_IND_DATA_0, data); + + src += 4; + byte_count -= 4; + addr += 4; + } + + /* RMW for the final bytes */ + if (byte_count > 0) { + data = 0; + + ret = kv_set_smc_sram_address(rdev, addr, limit); + if (ret) + return ret; + + original_data= RREG32(SMC_IND_DATA_0); + + extra_shift = 8 * (4 - byte_count); + + while (byte_count > 0) { + /* SMC address space is BE */ + data = (data << 8) + *src++; + byte_count--; + } + + data <<= extra_shift; + + data |= (original_data & ~((~0UL) << extra_shift)); + + ret = kv_set_smc_sram_address(rdev, addr, limit); + if (ret) + return ret; + + WREG32(SMC_IND_DATA_0, data); + } + return 0; +} + diff --git a/sys/dev/pci/drm/radeon/mkregtable.c b/sys/dev/pci/drm/radeon/mkregtable.c index de1d79eda18..b928c17bdee 100644 --- a/sys/dev/pci/drm/radeon/mkregtable.c +++ b/sys/dev/pci/drm/radeon/mkregtable.c @@ -1,4 +1,3 @@ -/* $OpenBSD: mkregtable.c,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* utility to create the register check tables * this includes inlined list.h safe for userspace. * @@ -348,7 +347,7 @@ static inline void list_splice_tail_init(struct list_head *list, * list_entry - get the struct for this entry * @ptr: the &struct list_head pointer. * @type: the type of the struct this is embedded in. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. */ #define list_entry(ptr, type, member) \ container_of(ptr, type, member) @@ -357,7 +356,7 @@ static inline void list_splice_tail_init(struct list_head *list, * list_first_entry - get the first element from a list * @ptr: the list head to take the element from. * @type: the type of the struct this is embedded in. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. * * Note, that list is expected to be not empty. */ @@ -373,19 +372,6 @@ static inline void list_splice_tail_init(struct list_head *list, for (pos = (head)->next; prefetch(pos->next), pos != (head); \ pos = pos->next) -/** - * __list_for_each - iterate over a list - * @pos: the &struct list_head to use as a loop cursor. - * @head: the head for your list. - * - * This variant differs from list_for_each() in that it's the - * simplest possible list iteration code, no prefetching is done. - * Use this for code that knows the list to be very short (empty - * or 1 entry) most of the time. - */ -#define __list_for_each(pos, head) \ - for (pos = (head)->next; pos != (head); pos = pos->next) - /** * list_for_each_prev - iterate over a list backwards * @pos: the &struct list_head to use as a loop cursor. @@ -420,7 +406,7 @@ static inline void list_splice_tail_init(struct list_head *list, * list_for_each_entry - iterate over list of given type * @pos: the type * to use as a loop cursor. * @head: the head for your list. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. */ #define list_for_each_entry(pos, head, member) \ for (pos = list_entry((head)->next, typeof(*pos), member); \ @@ -431,7 +417,7 @@ static inline void list_splice_tail_init(struct list_head *list, * list_for_each_entry_reverse - iterate backwards over list of given type. * @pos: the type * to use as a loop cursor. * @head: the head for your list. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. */ #define list_for_each_entry_reverse(pos, head, member) \ for (pos = list_entry((head)->prev, typeof(*pos), member); \ @@ -442,7 +428,7 @@ static inline void list_splice_tail_init(struct list_head *list, * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue() * @pos: the type * to use as a start point * @head: the head of the list - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. * * Prepares a pos entry for use as a start point in list_for_each_entry_continue(). */ @@ -453,7 +439,7 @@ static inline void list_splice_tail_init(struct list_head *list, * list_for_each_entry_continue - continue iteration over list of given type * @pos: the type * to use as a loop cursor. * @head: the head for your list. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. * * Continue to iterate over list of given type, continuing after * the current position. @@ -467,7 +453,7 @@ static inline void list_splice_tail_init(struct list_head *list, * list_for_each_entry_continue_reverse - iterate backwards from the given point * @pos: the type * to use as a loop cursor. * @head: the head for your list. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. * * Start to iterate over list of given type backwards, continuing after * the current position. @@ -481,7 +467,7 @@ static inline void list_splice_tail_init(struct list_head *list, * list_for_each_entry_from - iterate over list of given type from the current point * @pos: the type * to use as a loop cursor. * @head: the head for your list. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. * * Iterate over list of given type, continuing from current position. */ @@ -494,7 +480,7 @@ static inline void list_splice_tail_init(struct list_head *list, * @pos: the type * to use as a loop cursor. * @n: another type * to use as temporary storage * @head: the head for your list. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. */ #define list_for_each_entry_safe(pos, n, head, member) \ for (pos = list_entry((head)->next, typeof(*pos), member), \ @@ -507,7 +493,7 @@ static inline void list_splice_tail_init(struct list_head *list, * @pos: the type * to use as a loop cursor. * @n: another type * to use as temporary storage * @head: the head for your list. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. * * Iterate over list of given type, continuing after current point, * safe against removal of list entry. @@ -523,7 +509,7 @@ static inline void list_splice_tail_init(struct list_head *list, * @pos: the type * to use as a loop cursor. * @n: another type * to use as temporary storage * @head: the head for your list. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. * * Iterate over list of given type from current point, safe against * removal of list entry. @@ -538,7 +524,7 @@ static inline void list_splice_tail_init(struct list_head *list, * @pos: the type * to use as a loop cursor. * @n: another type * to use as temporary storage * @head: the head for your list. - * @member: the name of the list_struct within the struct. + * @member: the name of the list_head within the struct. * * Iterate backwards over list of given type, safe against removal * of list entry. @@ -669,7 +655,7 @@ static int parser_auth(struct table *t, const char *filename) /* first line will contain the last register * and gpu name */ - sscanf(buf, "%s %s", gpu_name, last_reg_s); + sscanf(buf, "%9s %9s", gpu_name, last_reg_s); t->gpu_prefix = gpu_name; last_reg = strtol(last_reg_s, NULL, 16); diff --git a/sys/dev/pci/drm/radeon/ni.c b/sys/dev/pci/drm/radeon/ni.c index eda1ed330f6..af18279f912 100644 --- a/sys/dev/pci/drm/radeon/ni.c +++ b/sys/dev/pci/drm/radeon/ni.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ni.c,v 1.12 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * @@ -25,12 +24,168 @@ #include #include "radeon.h" #include "radeon_asic.h" +#include "radeon_audio.h" #include #include "nid.h" #include "atom.h" #include "ni_reg.h" #include "cayman_blit_shaders.h" +#include "radeon_ucode.h" +#include "clearstate_cayman.h" +/* + * Indirect registers accessor + */ +u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + WREG32(TN_SMC_IND_INDEX_0, (reg)); + r = RREG32(TN_SMC_IND_DATA_0); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + return r; +} + +void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + WREG32(TN_SMC_IND_INDEX_0, (reg)); + WREG32(TN_SMC_IND_DATA_0, (v)); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); +} + +static const u32 tn_rlc_save_restore_register_list[] = +{ + 0x98fc, + 0x98f0, + 0x9834, + 0x9838, + 0x9870, + 0x9874, + 0x8a14, + 0x8b24, + 0x8bcc, + 0x8b10, + 0x8c30, + 0x8d00, + 0x8d04, + 0x8c00, + 0x8c04, + 0x8c10, + 0x8c14, + 0x8d8c, + 0x8cf0, + 0x8e38, + 0x9508, + 0x9688, + 0x9608, + 0x960c, + 0x9610, + 0x9614, + 0x88c4, + 0x8978, + 0x88d4, + 0x900c, + 0x9100, + 0x913c, + 0x90e8, + 0x9354, + 0xa008, + 0x98f8, + 0x9148, + 0x914c, + 0x3f94, + 0x98f4, + 0x9b7c, + 0x3f8c, + 0x8950, + 0x8954, + 0x8a18, + 0x8b28, + 0x9144, + 0x3f90, + 0x915c, + 0x9160, + 0x9178, + 0x917c, + 0x9180, + 0x918c, + 0x9190, + 0x9194, + 0x9198, + 0x919c, + 0x91a8, + 0x91ac, + 0x91b0, + 0x91b4, + 0x91b8, + 0x91c4, + 0x91c8, + 0x91cc, + 0x91d0, + 0x91d4, + 0x91e0, + 0x91e4, + 0x91ec, + 0x91f0, + 0x91f4, + 0x9200, + 0x9204, + 0x929c, + 0x8030, + 0x9150, + 0x9a60, + 0x920c, + 0x9210, + 0x9228, + 0x922c, + 0x9244, + 0x9248, + 0x91e8, + 0x9294, + 0x9208, + 0x9224, + 0x9240, + 0x9220, + 0x923c, + 0x9258, + 0x9744, + 0xa200, + 0xa204, + 0xa208, + 0xa20c, + 0x8d58, + 0x9030, + 0x9034, + 0x9038, + 0x903c, + 0x9040, + 0x9654, + 0x897c, + 0xa210, + 0xa214, + 0x9868, + 0xa02c, + 0x9664, + 0x9698, + 0x949c, + 0x8e10, + 0x8e18, + 0x8c50, + 0x8c58, + 0x8c60, + 0x8c68, + 0x89b4, + 0x9830, + 0x802c, +}; + +extern bool evergreen_is_display_hung(struct radeon_device *rdev); +extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev); extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev); @@ -39,40 +194,310 @@ extern void evergreen_irq_suspend(struct radeon_device *rdev); extern int evergreen_mc_init(struct radeon_device *rdev); extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev); extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev); -extern void si_rlc_fini(struct radeon_device *rdev); -extern int si_rlc_init(struct radeon_device *rdev); - -#define EVERGREEN_PFP_UCODE_SIZE 1120 -#define EVERGREEN_PM4_UCODE_SIZE 1376 -#define EVERGREEN_RLC_UCODE_SIZE 768 -#define BTC_MC_UCODE_SIZE 6024 - -#define CAYMAN_PFP_UCODE_SIZE 2176 -#define CAYMAN_PM4_UCODE_SIZE 2176 -#define CAYMAN_RLC_UCODE_SIZE 1024 -#define CAYMAN_MC_UCODE_SIZE 6037 - -#define ARUBA_RLC_UCODE_SIZE 1536 +extern void evergreen_program_aspm(struct radeon_device *rdev); +extern void sumo_rlc_fini(struct radeon_device *rdev); +extern int sumo_rlc_init(struct radeon_device *rdev); +extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev); /* Firmware Names */ MODULE_FIRMWARE("radeon/BARTS_pfp.bin"); MODULE_FIRMWARE("radeon/BARTS_me.bin"); MODULE_FIRMWARE("radeon/BARTS_mc.bin"); +MODULE_FIRMWARE("radeon/BARTS_smc.bin"); MODULE_FIRMWARE("radeon/BTC_rlc.bin"); MODULE_FIRMWARE("radeon/TURKS_pfp.bin"); MODULE_FIRMWARE("radeon/TURKS_me.bin"); MODULE_FIRMWARE("radeon/TURKS_mc.bin"); +MODULE_FIRMWARE("radeon/TURKS_smc.bin"); MODULE_FIRMWARE("radeon/CAICOS_pfp.bin"); MODULE_FIRMWARE("radeon/CAICOS_me.bin"); MODULE_FIRMWARE("radeon/CAICOS_mc.bin"); +MODULE_FIRMWARE("radeon/CAICOS_smc.bin"); MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin"); MODULE_FIRMWARE("radeon/CAYMAN_me.bin"); MODULE_FIRMWARE("radeon/CAYMAN_mc.bin"); MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin"); +MODULE_FIRMWARE("radeon/CAYMAN_smc.bin"); MODULE_FIRMWARE("radeon/ARUBA_pfp.bin"); MODULE_FIRMWARE("radeon/ARUBA_me.bin"); MODULE_FIRMWARE("radeon/ARUBA_rlc.bin"); + +static const u32 cayman_golden_registers2[] = +{ + 0x3e5c, 0xffffffff, 0x00000000, + 0x3e48, 0xffffffff, 0x00000000, + 0x3e4c, 0xffffffff, 0x00000000, + 0x3e64, 0xffffffff, 0x00000000, + 0x3e50, 0xffffffff, 0x00000000, + 0x3e60, 0xffffffff, 0x00000000 +}; + +static const u32 cayman_golden_registers[] = +{ + 0x5eb4, 0xffffffff, 0x00000002, + 0x5e78, 0x8f311ff1, 0x001000f0, + 0x3f90, 0xffff0000, 0xff000000, + 0x9148, 0xffff0000, 0xff000000, + 0x3f94, 0xffff0000, 0xff000000, + 0x914c, 0xffff0000, 0xff000000, + 0xc78, 0x00000080, 0x00000080, + 0xbd4, 0x70073777, 0x00011003, + 0xd02c, 0xbfffff1f, 0x08421000, + 0xd0b8, 0x73773777, 0x02011003, + 0x5bc0, 0x00200000, 0x50100000, + 0x98f8, 0x33773777, 0x02011003, + 0x98fc, 0xffffffff, 0x76541032, + 0x7030, 0x31000311, 0x00000011, + 0x2f48, 0x33773777, 0x42010001, + 0x6b28, 0x00000010, 0x00000012, + 0x7728, 0x00000010, 0x00000012, + 0x10328, 0x00000010, 0x00000012, + 0x10f28, 0x00000010, 0x00000012, + 0x11b28, 0x00000010, 0x00000012, + 0x12728, 0x00000010, 0x00000012, + 0x240c, 0x000007ff, 0x00000000, + 0x8a14, 0xf000001f, 0x00000007, + 0x8b24, 0x3fff3fff, 0x00ff0fff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x10c, 0x00000001, 0x00010003, + 0xa02c, 0xffffffff, 0x0000009b, + 0x913c, 0x0000010f, 0x01000100, + 0x8c04, 0xf8ff00ff, 0x40600060, + 0x28350, 0x00000f01, 0x00000000, + 0x9508, 0x3700001f, 0x00000002, + 0x960c, 0xffffffff, 0x54763210, + 0x88c4, 0x001f3ae3, 0x00000082, + 0x88d0, 0xffffffff, 0x0f40df40, + 0x88d4, 0x0000001f, 0x00000010, + 0x8974, 0xffffffff, 0x00000000 +}; + +static const u32 dvst_golden_registers2[] = +{ + 0x8f8, 0xffffffff, 0, + 0x8fc, 0x00380000, 0, + 0x8f8, 0xffffffff, 1, + 0x8fc, 0x0e000000, 0 +}; + +static const u32 dvst_golden_registers[] = +{ + 0x690, 0x3fff3fff, 0x20c00033, + 0x918c, 0x0fff0fff, 0x00010006, + 0x91a8, 0x0fff0fff, 0x00010006, + 0x9150, 0xffffdfff, 0x6e944040, + 0x917c, 0x0fff0fff, 0x00030002, + 0x9198, 0x0fff0fff, 0x00030002, + 0x915c, 0x0fff0fff, 0x00010000, + 0x3f90, 0xffff0001, 0xff000000, + 0x9178, 0x0fff0fff, 0x00070000, + 0x9194, 0x0fff0fff, 0x00070000, + 0x9148, 0xffff0001, 0xff000000, + 0x9190, 0x0fff0fff, 0x00090008, + 0x91ac, 0x0fff0fff, 0x00090008, + 0x3f94, 0xffff0000, 0xff000000, + 0x914c, 0xffff0000, 0xff000000, + 0x929c, 0x00000fff, 0x00000001, + 0x55e4, 0xff607fff, 0xfc000100, + 0x8a18, 0xff000fff, 0x00000100, + 0x8b28, 0xff000fff, 0x00000100, + 0x9144, 0xfffc0fff, 0x00000100, + 0x6ed8, 0x00010101, 0x00010000, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0xfffffffe, 0x00000000, + 0xd0c0, 0xff000fff, 0x00000100, + 0xd02c, 0xbfffff1f, 0x08421000, + 0xd0b8, 0x73773777, 0x12010001, + 0x5bb0, 0x000000f0, 0x00000070, + 0x98f8, 0x73773777, 0x12010001, + 0x98fc, 0xffffffff, 0x00000010, + 0x9b7c, 0x00ff0000, 0x00fc0000, + 0x8030, 0x00001f0f, 0x0000100a, + 0x2f48, 0x73773777, 0x12010001, + 0x2408, 0x00030000, 0x000c007f, + 0x8a14, 0xf000003f, 0x00000007, + 0x8b24, 0x3fff3fff, 0x00ff0fff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x4d8, 0x00000fff, 0x00000100, + 0xa008, 0xffffffff, 0x00010000, + 0x913c, 0xffff03ff, 0x01000100, + 0x8c00, 0x000000ff, 0x00000003, + 0x8c04, 0xf8ff00ff, 0x40600060, + 0x8cf0, 0x1fff1fff, 0x08e00410, + 0x28350, 0x00000f01, 0x00000000, + 0x9508, 0xf700071f, 0x00000002, + 0x960c, 0xffffffff, 0x54763210, + 0x20ef8, 0x01ff01ff, 0x00000002, + 0x20e98, 0xfffffbff, 0x00200000, + 0x2015c, 0xffffffff, 0x00000f40, + 0x88c4, 0x001f3ae3, 0x00000082, + 0x8978, 0x3fffffff, 0x04050140, + 0x88d4, 0x0000001f, 0x00000010, + 0x8974, 0xffffffff, 0x00000000 +}; + +static const u32 scrapper_golden_registers[] = +{ + 0x690, 0x3fff3fff, 0x20c00033, + 0x918c, 0x0fff0fff, 0x00010006, + 0x918c, 0x0fff0fff, 0x00010006, + 0x91a8, 0x0fff0fff, 0x00010006, + 0x91a8, 0x0fff0fff, 0x00010006, + 0x9150, 0xffffdfff, 0x6e944040, + 0x9150, 0xffffdfff, 0x6e944040, + 0x917c, 0x0fff0fff, 0x00030002, + 0x917c, 0x0fff0fff, 0x00030002, + 0x9198, 0x0fff0fff, 0x00030002, + 0x9198, 0x0fff0fff, 0x00030002, + 0x915c, 0x0fff0fff, 0x00010000, + 0x915c, 0x0fff0fff, 0x00010000, + 0x3f90, 0xffff0001, 0xff000000, + 0x3f90, 0xffff0001, 0xff000000, + 0x9178, 0x0fff0fff, 0x00070000, + 0x9178, 0x0fff0fff, 0x00070000, + 0x9194, 0x0fff0fff, 0x00070000, + 0x9194, 0x0fff0fff, 0x00070000, + 0x9148, 0xffff0001, 0xff000000, + 0x9148, 0xffff0001, 0xff000000, + 0x9190, 0x0fff0fff, 0x00090008, + 0x9190, 0x0fff0fff, 0x00090008, + 0x91ac, 0x0fff0fff, 0x00090008, + 0x91ac, 0x0fff0fff, 0x00090008, + 0x3f94, 0xffff0000, 0xff000000, + 0x3f94, 0xffff0000, 0xff000000, + 0x914c, 0xffff0000, 0xff000000, + 0x914c, 0xffff0000, 0xff000000, + 0x929c, 0x00000fff, 0x00000001, + 0x929c, 0x00000fff, 0x00000001, + 0x55e4, 0xff607fff, 0xfc000100, + 0x8a18, 0xff000fff, 0x00000100, + 0x8a18, 0xff000fff, 0x00000100, + 0x8b28, 0xff000fff, 0x00000100, + 0x8b28, 0xff000fff, 0x00000100, + 0x9144, 0xfffc0fff, 0x00000100, + 0x9144, 0xfffc0fff, 0x00000100, + 0x6ed8, 0x00010101, 0x00010000, + 0x9830, 0xffffffff, 0x00000000, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0xfffffffe, 0x00000000, + 0x9838, 0xfffffffe, 0x00000000, + 0xd0c0, 0xff000fff, 0x00000100, + 0xd02c, 0xbfffff1f, 0x08421000, + 0xd02c, 0xbfffff1f, 0x08421000, + 0xd0b8, 0x73773777, 0x12010001, + 0xd0b8, 0x73773777, 0x12010001, + 0x5bb0, 0x000000f0, 0x00000070, + 0x98f8, 0x73773777, 0x12010001, + 0x98f8, 0x73773777, 0x12010001, + 0x98fc, 0xffffffff, 0x00000010, + 0x98fc, 0xffffffff, 0x00000010, + 0x9b7c, 0x00ff0000, 0x00fc0000, + 0x9b7c, 0x00ff0000, 0x00fc0000, + 0x8030, 0x00001f0f, 0x0000100a, + 0x8030, 0x00001f0f, 0x0000100a, + 0x2f48, 0x73773777, 0x12010001, + 0x2f48, 0x73773777, 0x12010001, + 0x2408, 0x00030000, 0x000c007f, + 0x8a14, 0xf000003f, 0x00000007, + 0x8a14, 0xf000003f, 0x00000007, + 0x8b24, 0x3fff3fff, 0x00ff0fff, + 0x8b24, 0x3fff3fff, 0x00ff0fff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x28a4c, 0x07ffffff, 0x06000000, + 0x4d8, 0x00000fff, 0x00000100, + 0x4d8, 0x00000fff, 0x00000100, + 0xa008, 0xffffffff, 0x00010000, + 0xa008, 0xffffffff, 0x00010000, + 0x913c, 0xffff03ff, 0x01000100, + 0x913c, 0xffff03ff, 0x01000100, + 0x90e8, 0x001fffff, 0x010400c0, + 0x8c00, 0x000000ff, 0x00000003, + 0x8c00, 0x000000ff, 0x00000003, + 0x8c04, 0xf8ff00ff, 0x40600060, + 0x8c04, 0xf8ff00ff, 0x40600060, + 0x8c30, 0x0000000f, 0x00040005, + 0x8cf0, 0x1fff1fff, 0x08e00410, + 0x8cf0, 0x1fff1fff, 0x08e00410, + 0x900c, 0x00ffffff, 0x0017071f, + 0x28350, 0x00000f01, 0x00000000, + 0x28350, 0x00000f01, 0x00000000, + 0x9508, 0xf700071f, 0x00000002, + 0x9508, 0xf700071f, 0x00000002, + 0x9688, 0x00300000, 0x0017000f, + 0x960c, 0xffffffff, 0x54763210, + 0x960c, 0xffffffff, 0x54763210, + 0x20ef8, 0x01ff01ff, 0x00000002, + 0x20e98, 0xfffffbff, 0x00200000, + 0x2015c, 0xffffffff, 0x00000f40, + 0x88c4, 0x001f3ae3, 0x00000082, + 0x88c4, 0x001f3ae3, 0x00000082, + 0x8978, 0x3fffffff, 0x04050140, + 0x8978, 0x3fffffff, 0x04050140, + 0x88d4, 0x0000001f, 0x00000010, + 0x88d4, 0x0000001f, 0x00000010, + 0x8974, 0xffffffff, 0x00000000, + 0x8974, 0xffffffff, 0x00000000 +}; + +static void ni_init_golden_registers(struct radeon_device *rdev) +{ + switch (rdev->family) { + case CHIP_CAYMAN: + radeon_program_register_sequence(rdev, + cayman_golden_registers, + (const u32)ARRAY_SIZE(cayman_golden_registers)); + radeon_program_register_sequence(rdev, + cayman_golden_registers2, + (const u32)ARRAY_SIZE(cayman_golden_registers2)); + break; + case CHIP_ARUBA: + if ((rdev->pdev->device == 0x9900) || + (rdev->pdev->device == 0x9901) || + (rdev->pdev->device == 0x9903) || + (rdev->pdev->device == 0x9904) || + (rdev->pdev->device == 0x9905) || + (rdev->pdev->device == 0x9906) || + (rdev->pdev->device == 0x9907) || + (rdev->pdev->device == 0x9908) || + (rdev->pdev->device == 0x9909) || + (rdev->pdev->device == 0x990A) || + (rdev->pdev->device == 0x990B) || + (rdev->pdev->device == 0x990C) || + (rdev->pdev->device == 0x990D) || + (rdev->pdev->device == 0x990E) || + (rdev->pdev->device == 0x990F) || + (rdev->pdev->device == 0x9910) || + (rdev->pdev->device == 0x9913) || + (rdev->pdev->device == 0x9917) || + (rdev->pdev->device == 0x9918)) { + radeon_program_register_sequence(rdev, + dvst_golden_registers, + (const u32)ARRAY_SIZE(dvst_golden_registers)); + radeon_program_register_sequence(rdev, + dvst_golden_registers2, + (const u32)ARRAY_SIZE(dvst_golden_registers2)); + } else { + radeon_program_register_sequence(rdev, + scrapper_golden_registers, + (const u32)ARRAY_SIZE(scrapper_golden_registers)); + radeon_program_register_sequence(rdev, + dvst_golden_registers2, + (const u32)ARRAY_SIZE(dvst_golden_registers2)); + } + break; + default: + break; + } +} + #define BTC_IO_MC_REGS_SIZE 29 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = { @@ -256,7 +681,7 @@ int ni_mc_load_microcode(struct radeon_device *rdev) WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); } /* load the MC ucode */ - fw_data = (const __be32 *)rdev->mc_fw; + fw_data = (const __be32 *)rdev->mc_fw->data; for (i = 0; i < ucode_size; i++) WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); @@ -284,6 +709,7 @@ int ni_init_microcode(struct radeon_device *rdev) const char *chip_name; const char *rlc_chip_name; size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size; + size_t smc_req_size = 0; char fw_name[30]; int err; @@ -291,40 +717,44 @@ int ni_init_microcode(struct radeon_device *rdev) switch (rdev->family) { case CHIP_BARTS: - chip_name = "barts"; - rlc_chip_name = "btc"; + chip_name = "BARTS"; + rlc_chip_name = "BTC"; pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4; me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4; rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4; mc_req_size = BTC_MC_UCODE_SIZE * 4; + smc_req_size = roundup2(BARTS_SMC_UCODE_SIZE, 4); break; case CHIP_TURKS: - chip_name = "turks"; - rlc_chip_name = "btc"; + chip_name = "TURKS"; + rlc_chip_name = "BTC"; pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4; me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4; rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4; mc_req_size = BTC_MC_UCODE_SIZE * 4; + smc_req_size = roundup2(TURKS_SMC_UCODE_SIZE, 4); break; case CHIP_CAICOS: - chip_name = "caicos"; - rlc_chip_name = "btc"; + chip_name = "CAICOS"; + rlc_chip_name = "BTC"; pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4; me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4; rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4; mc_req_size = BTC_MC_UCODE_SIZE * 4; + smc_req_size = roundup2(CAICOS_SMC_UCODE_SIZE, 4); break; case CHIP_CAYMAN: - chip_name = "cayman"; - rlc_chip_name = "cayman"; + chip_name = "CAYMAN"; + rlc_chip_name = "CAYMAN"; pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4; me_req_size = CAYMAN_PM4_UCODE_SIZE * 4; rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4; mc_req_size = CAYMAN_MC_UCODE_SIZE * 4; + smc_req_size = roundup2(CAYMAN_SMC_UCODE_SIZE, 4); break; case CHIP_ARUBA: - chip_name = "aruba"; - rlc_chip_name = "aruba"; + chip_name = "ARUBA"; + rlc_chip_name = "ARUBA"; /* pfp/me same size as CAYMAN */ pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4; me_req_size = CAYMAN_PM4_UCODE_SIZE * 4; @@ -336,79 +766,127 @@ int ni_init_microcode(struct radeon_device *rdev) DRM_INFO("Loading %s Microcode\n", chip_name); - snprintf(fw_name, sizeof(fw_name), "radeon-%s_pfp", chip_name); - err = loadfirmware(fw_name, &rdev->pfp_fw, &rdev->pfp_fw_size); + snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); + err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); if (err) goto out; - if (rdev->pfp_fw_size != pfp_req_size) { - DRM_ERROR( + if (rdev->pfp_fw->size != pfp_req_size) { + printk(KERN_ERR "ni_cp: Bogus length %zu in firmware \"%s\"\n", - rdev->pfp_fw_size, fw_name); + rdev->pfp_fw->size, fw_name); err = -EINVAL; goto out; } - snprintf(fw_name, sizeof(fw_name), "radeon-%s_me", chip_name); - err = loadfirmware(fw_name, &rdev->me_fw, &rdev->me_fw_size); + snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); + err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); if (err) goto out; - if (rdev->me_fw_size != me_req_size) { - DRM_ERROR( + if (rdev->me_fw->size != me_req_size) { + printk(KERN_ERR "ni_cp: Bogus length %zu in firmware \"%s\"\n", - rdev->me_fw_size, fw_name); + rdev->me_fw->size, fw_name); err = -EINVAL; } - snprintf(fw_name, sizeof(fw_name), "radeon-%s_rlc", rlc_chip_name); - err = loadfirmware(fw_name, &rdev->rlc_fw, &rdev->rlc_fw_size); + snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name); + err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); if (err) goto out; - if (rdev->rlc_fw_size != rlc_req_size) { - DRM_ERROR( + if (rdev->rlc_fw->size != rlc_req_size) { + printk(KERN_ERR "ni_rlc: Bogus length %zu in firmware \"%s\"\n", - rdev->rlc_fw_size, fw_name); + rdev->rlc_fw->size, fw_name); err = -EINVAL; } /* no MC ucode on TN */ if (!(rdev->flags & RADEON_IS_IGP)) { - snprintf(fw_name, sizeof(fw_name), "radeon-%s_mc", chip_name); - err = loadfirmware(fw_name, &rdev->mc_fw, &rdev->mc_fw_size); + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); + err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); if (err) goto out; - if (rdev->mc_fw_size != mc_req_size) { - DRM_ERROR( + if (rdev->mc_fw->size != mc_req_size) { + printk(KERN_ERR + "ni_mc: Bogus length %zu in firmware \"%s\"\n", + rdev->mc_fw->size, fw_name); + err = -EINVAL; + } + } + + if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); + err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); + if (err) { + printk(KERN_ERR + "smc: error loading firmware \"%s\"\n", + fw_name); + release_firmware(rdev->smc_fw); + rdev->smc_fw = NULL; + err = 0; + } else if (rdev->smc_fw->size != smc_req_size) { + printk(KERN_ERR "ni_mc: Bogus length %zu in firmware \"%s\"\n", - rdev->mc_fw_size, fw_name); + rdev->mc_fw->size, fw_name); err = -EINVAL; } } + out: if (err) { if (err != -EINVAL) - DRM_ERROR( + printk(KERN_ERR "ni_cp: Failed to load firmware \"%s\"\n", fw_name); - if (rdev->pfp_fw) { - free(rdev->pfp_fw, M_DEVBUF, 0); - rdev->pfp_fw = NULL; - } - if (rdev->me_fw) { - free(rdev->me_fw, M_DEVBUF, 0); - rdev->me_fw = NULL; - } - if (rdev->rlc_fw) { - free(rdev->rlc_fw, M_DEVBUF, 0); - rdev->rlc_fw = NULL; - } - if (rdev->mc_fw) { - free(rdev->mc_fw, M_DEVBUF, 0); - rdev->mc_fw = NULL; - } + release_firmware(rdev->pfp_fw); + rdev->pfp_fw = NULL; + release_firmware(rdev->me_fw); + rdev->me_fw = NULL; + release_firmware(rdev->rlc_fw); + rdev->rlc_fw = NULL; + release_firmware(rdev->mc_fw); + rdev->mc_fw = NULL; } return err; } +/** + * cayman_get_allowed_info_register - fetch the register for the info ioctl + * + * @rdev: radeon_device pointer + * @reg: register offset in bytes + * @val: register value + * + * Returns 0 for success or -EINVAL for an invalid register + * + */ +int cayman_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val) +{ + switch (reg) { + case GRBM_STATUS: + case GRBM_STATUS_SE0: + case GRBM_STATUS_SE1: + case SRBM_STATUS: + case SRBM_STATUS2: + case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET): + case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET): + case UVD_STATUS: + *val = RREG32(reg); + return 0; + default: + return -EINVAL; + } +} + +int tn_get_temp(struct radeon_device *rdev) +{ + u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff; + int actual_temp = (temp / 8) - 49; + + return actual_temp * 1000; +} + /* * Core functions */ @@ -535,6 +1013,8 @@ static void cayman_gpu_init(struct radeon_device *rdev) } WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); + WREG32(SRBM_INT_CNTL, 0x1); + WREG32(SRBM_INT_ACK, 0x1); evergreen_fix_pci_max_read_req_size(rdev); @@ -622,6 +1102,26 @@ static void cayman_gpu_init(struct radeon_device *rdev) } /* enabled rb are just the one not disabled :) */ disabled_rb_mask = tmp; + tmp = 0; + for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++) + tmp |= (1 << i); + /* if all the backends are disabled, fix it up here */ + if ((disabled_rb_mask & tmp) == tmp) { + for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++) + disabled_rb_mask &= ~(1 << i); + } + + for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) { + u32 simd_disable_bitmap; + + WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); + WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); + simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; + simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se; + tmp <<= 16; + tmp |= simd_disable_bitmap; + } + rdev->config.cayman.active_simds = hweight32(~tmp); WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); @@ -633,15 +1133,18 @@ static void cayman_gpu_init(struct radeon_device *rdev) WREG32(HDP_ADDR_CONFIG, gb_addr_config); WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config); WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config); + WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); + WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); + WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); if ((rdev->config.cayman.max_backends_per_se == 1) && (rdev->flags & RADEON_IS_IGP)) { - if ((disabled_rb_mask & 3) == 1) { - /* RB0 disabled, RB1 enabled */ - tmp = 0x11111111; - } else { + if ((disabled_rb_mask & 3) == 2) { /* RB1 disabled, RB0 enabled */ tmp = 0x00000000; + } else { + /* RB0 disabled, RB1 enabled */ + tmp = 0x11111111; } } else { tmp = gb_addr_config & NUM_PIPES_MASK; @@ -745,6 +1248,16 @@ static void cayman_gpu_init(struct radeon_device *rdev) WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); udelay(50); + + /* set clockgating golden values on TN */ + if (rdev->family == CHIP_ARUBA) { + tmp = RREG32_CG(CG_CGTT_LOCAL_0); + tmp &= ~0x00380000; + WREG32_CG(CG_CGTT_LOCAL_0, tmp); + tmp = RREG32_CG(CG_CGTT_LOCAL_1); + tmp &= ~0x0e000000; + WREG32_CG(CG_CGTT_LOCAL_1, tmp); + } } /* @@ -770,7 +1283,6 @@ static int cayman_pcie_gart_enable(struct radeon_device *rdev) r = radeon_gart_table_vram_pin(rdev); if (r) return r; - radeon_gart_restore(rdev); /* Setup TLB control */ WREG32(MC_VM_MX_L1_TLB_CNTL, (0xA << 7) | @@ -781,12 +1293,14 @@ static int cayman_pcie_gart_enable(struct radeon_device *rdev) SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); /* Setup L2 cache */ WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | + ENABLE_L2_FRAGMENT_PROCESSING | ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | EFFECTIVE_L2_QUEUE_SIZE(7) | CONTEXT1_IDENTITY_ACCESS_MODE(1)); WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | + BANK_SELECT(6) | L2_CACHE_BIGK_FRAGMENT_SIZE(6)); /* setup context0 */ WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); @@ -809,9 +1323,10 @@ static int cayman_pcie_gart_enable(struct radeon_device *rdev) */ for (i = 1; i < 8; i++) { WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0); - WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn); + WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), + rdev->vm_manager.max_pfn - 1); WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), - rdev->gart.table_addr >> 12); + rdev->vm_manager.saved_table_addr[i]); } /* enable context1-7 */ @@ -819,6 +1334,7 @@ static int cayman_pcie_gart_enable(struct radeon_device *rdev) (u32)(rdev->dummy_page.addr >> 12)); WREG32(VM_CONTEXT1_CNTL2, 4); WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | + PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | @@ -842,6 +1358,13 @@ static int cayman_pcie_gart_enable(struct radeon_device *rdev) static void cayman_pcie_gart_disable(struct radeon_device *rdev) { + unsigned i; + + for (i = 1; i < 8; ++i) { + rdev->vm_manager.saved_table_addr[i] = RREG32( + VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2)); + } + /* Disable all tables */ WREG32(VM_CONTEXT0_CNTL, 0); WREG32(VM_CONTEXT1_CNTL, 0); @@ -870,9 +1393,7 @@ static void cayman_pcie_gart_fini(struct radeon_device *rdev) void cayman_cp_int_cntl_setup(struct radeon_device *rdev, int ring, u32 cp_int_cntl) { - u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3; - - WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3)); + WREG32(SRBM_GFX_CNTL, RINGID(ring)); WREG32(CP_INT_CNTL, cp_int_cntl); } @@ -896,7 +1417,7 @@ void cayman_fence_ring_emit(struct radeon_device *rdev, /* EVENT_WRITE_EOP - flush caches, send int */ radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5)); - radeon_ring_write(ring, addr & 0xffffffff); + radeon_ring_write(ring, lower_32_bits(addr)); radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); radeon_ring_write(ring, fence->seq); radeon_ring_write(ring, 0); @@ -905,6 +1426,7 @@ void cayman_fence_ring_emit(struct radeon_device *rdev, void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) { struct radeon_ring *ring = &rdev->ring[ib->ring]; + unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA; @@ -927,15 +1449,14 @@ void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) #endif (ib->gpu_addr & 0xFFFFFFFC)); radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); - radeon_ring_write(ring, ib->length_dw | - (ib->vm ? (ib->vm->id << 24) : 0)); + radeon_ring_write(ring, ib->length_dw | (vm_id << 24)); /* flush read cache over gart for this vmid */ radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl); radeon_ring_write(ring, 0xFFFFFFFF); radeon_ring_write(ring, 0); - radeon_ring_write(ring, ((ib->vm ? ib->vm->id : 0) << 24) | 10); /* poll interval */ + radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */ } static void cayman_cp_enable(struct radeon_device *rdev, bool enable) @@ -943,13 +1464,63 @@ static void cayman_cp_enable(struct radeon_device *rdev, bool enable) if (enable) WREG32(CP_ME_CNTL, 0); else { - radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT)); WREG32(SCRATCH_UMSK, 0); rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; } } +u32 cayman_gfx_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 rptr; + + if (rdev->wb.enabled) + rptr = rdev->wb.wb[ring->rptr_offs/4]; + else { + if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) + rptr = RREG32(CP_RB0_RPTR); + else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) + rptr = RREG32(CP_RB1_RPTR); + else + rptr = RREG32(CP_RB2_RPTR); + } + + return rptr; +} + +u32 cayman_gfx_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 wptr; + + if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) + wptr = RREG32(CP_RB0_WPTR); + else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) + wptr = RREG32(CP_RB1_WPTR); + else + wptr = RREG32(CP_RB2_WPTR); + + return wptr; +} + +void cayman_gfx_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) { + WREG32(CP_RB0_WPTR, ring->wptr); + (void)RREG32(CP_RB0_WPTR); + } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) { + WREG32(CP_RB1_WPTR, ring->wptr); + (void)RREG32(CP_RB1_WPTR); + } else { + WREG32(CP_RB2_WPTR, ring->wptr); + (void)RREG32(CP_RB2_WPTR); + } +} + static int cayman_cp_load_microcode(struct radeon_device *rdev) { const __be32 *fw_data; @@ -960,13 +1531,13 @@ static int cayman_cp_load_microcode(struct radeon_device *rdev) cayman_cp_enable(rdev, false); - fw_data = (const __be32 *)rdev->pfp_fw; + fw_data = (const __be32 *)rdev->pfp_fw->data; WREG32(CP_PFP_UCODE_ADDR, 0); for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++) WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); WREG32(CP_PFP_UCODE_ADDR, 0); - fw_data = (const __be32 *)rdev->me_fw; + fw_data = (const __be32 *)rdev->me_fw->data; WREG32(CP_ME_RAM_WADDR, 0); for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++) WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); @@ -994,7 +1565,7 @@ static int cayman_cp_start(struct radeon_device *rdev) radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); radeon_ring_write(ring, 0); radeon_ring_write(ring, 0); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); cayman_cp_enable(rdev, true); @@ -1036,7 +1607,7 @@ static int cayman_cp_start(struct radeon_device *rdev) radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ radeon_ring_write(ring, 0x00000010); /* */ - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); /* XXX init other rings */ @@ -1078,6 +1649,16 @@ static int cayman_cp_resume(struct radeon_device *rdev) CP_RB1_BASE, CP_RB2_BASE }; + static const unsigned cp_rb_rptr[] = { + CP_RB0_RPTR, + CP_RB1_RPTR, + CP_RB2_RPTR + }; + static const unsigned cp_rb_wptr[] = { + CP_RB0_WPTR, + CP_RB1_WPTR, + CP_RB2_WPTR + }; struct radeon_ring *ring; int i, r; @@ -1111,8 +1692,8 @@ static int cayman_cp_resume(struct radeon_device *rdev) /* Set ring buffer size */ ring = &rdev->ring[ridx[i]]; - rb_cntl = drm_order(ring->ring_size / 8); - rb_cntl |= drm_order(RADEON_GPU_PAGE_SIZE/8) << 8; + rb_cntl = order_base_2(ring->ring_size / 8); + rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8; #ifdef __BIG_ENDIAN rb_cntl |= BUF_SWAP_32BIT; #endif @@ -1135,9 +1716,9 @@ static int cayman_cp_resume(struct radeon_device *rdev) ring = &rdev->ring[ridx[i]]; WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA); - ring->rptr = ring->wptr = 0; - WREG32(ring->rptr_reg, ring->rptr); - WREG32(ring->wptr_reg, ring->wptr); + ring->wptr = 0; + WREG32(cp_rb_rptr[i], 0); + WREG32(cp_rb_wptr[i], ring->wptr); mdelay(1); WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA); @@ -1157,359 +1738,262 @@ static int cayman_cp_resume(struct radeon_device *rdev) return r; } + if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); + return 0; } -/* - * DMA - * Starting with R600, the GPU has an asynchronous - * DMA engine. The programming model is very similar - * to the 3D engine (ring buffer, IBs, etc.), but the - * DMA controller has it's own packet format that is - * different form the PM4 format used by the 3D engine. - * It supports copying data, writing embedded data, - * solid fills, and a number of other things. It also - * has support for tiling/detiling of buffers. - * Cayman and newer support two asynchronous DMA engines. - */ -/** - * cayman_dma_ring_ib_execute - Schedule an IB on the DMA engine - * - * @rdev: radeon_device pointer - * @ib: IB object to schedule - * - * Schedule an IB in the DMA ring (cayman-SI). - */ -void cayman_dma_ring_ib_execute(struct radeon_device *rdev, - struct radeon_ib *ib) +u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev) { - struct radeon_ring *ring = &rdev->ring[ib->ring]; + u32 reset_mask = 0; + u32 tmp; - if (rdev->wb.enabled) { - u32 next_rptr = ring->wptr + 4; - while ((next_rptr & 7) != 5) - next_rptr++; - next_rptr += 3; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1)); - radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff); - radeon_ring_write(ring, next_rptr); + /* GRBM_STATUS */ + tmp = RREG32(GRBM_STATUS); + if (tmp & (PA_BUSY | SC_BUSY | + SH_BUSY | SX_BUSY | + TA_BUSY | VGT_BUSY | + DB_BUSY | CB_BUSY | + GDS_BUSY | SPI_BUSY | + IA_BUSY | IA_BUSY_NO_DMA)) + reset_mask |= RADEON_RESET_GFX; + + if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | + CP_BUSY | CP_COHERENCY_BUSY)) + reset_mask |= RADEON_RESET_CP; + + if (tmp & GRBM_EE_BUSY) + reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; + + /* DMA_STATUS_REG 0 */ + tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET); + if (!(tmp & DMA_IDLE)) + reset_mask |= RADEON_RESET_DMA; + + /* DMA_STATUS_REG 1 */ + tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET); + if (!(tmp & DMA_IDLE)) + reset_mask |= RADEON_RESET_DMA1; + + /* SRBM_STATUS2 */ + tmp = RREG32(SRBM_STATUS2); + if (tmp & DMA_BUSY) + reset_mask |= RADEON_RESET_DMA; + + if (tmp & DMA1_BUSY) + reset_mask |= RADEON_RESET_DMA1; + + /* SRBM_STATUS */ + tmp = RREG32(SRBM_STATUS); + if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) + reset_mask |= RADEON_RESET_RLC; + + if (tmp & IH_BUSY) + reset_mask |= RADEON_RESET_IH; + + if (tmp & SEM_BUSY) + reset_mask |= RADEON_RESET_SEM; + + if (tmp & GRBM_RQ_PENDING) + reset_mask |= RADEON_RESET_GRBM; + + if (tmp & VMC_BUSY) + reset_mask |= RADEON_RESET_VMC; + + if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | + MCC_BUSY | MCD_BUSY)) + reset_mask |= RADEON_RESET_MC; + + if (evergreen_is_display_hung(rdev)) + reset_mask |= RADEON_RESET_DISPLAY; + + /* VM_L2_STATUS */ + tmp = RREG32(VM_L2_STATUS); + if (tmp & L2_BUSY) + reset_mask |= RADEON_RESET_VMC; + + /* Skip MC reset as it's mostly likely not hung, just busy */ + if (reset_mask & RADEON_RESET_MC) { + DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); + reset_mask &= ~RADEON_RESET_MC; } - /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. - * Pad as necessary with NOPs. - */ - while ((ring->wptr & 7) != 5) - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); - radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, ib->vm ? ib->vm->id : 0, 0)); - radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); - radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); - + return reset_mask; } -/** - * cayman_dma_stop - stop the async dma engines - * - * @rdev: radeon_device pointer - * - * Stop the async dma engines (cayman-SI). - */ -void cayman_dma_stop(struct radeon_device *rdev) +static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) { - u32 rb_cntl; + struct evergreen_mc_save save; + u32 grbm_soft_reset = 0, srbm_soft_reset = 0; + u32 tmp; - radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + if (reset_mask == 0) + return; - /* dma0 */ - rb_cntl = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); - rb_cntl &= ~DMA_RB_ENABLE; - WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, rb_cntl); + dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); - /* dma1 */ - rb_cntl = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); - rb_cntl &= ~DMA_RB_ENABLE; - WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, rb_cntl); + evergreen_print_gpu_status_regs(rdev); + dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_ADDR 0x%08X\n", + RREG32(0x14F8)); + dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n", + RREG32(0x14D8)); + dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", + RREG32(0x14FC)); + dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", + RREG32(0x14DC)); - rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false; - rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false; -} + /* Disable CP parsing/prefetching */ + WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); -/** - * cayman_dma_resume - setup and start the async dma engines - * - * @rdev: radeon_device pointer - * - * Set up the DMA ring buffers and enable them. (cayman-SI). - * Returns 0 for success, error for failure. - */ -int cayman_dma_resume(struct radeon_device *rdev) -{ - struct radeon_ring *ring; - u32 rb_cntl, dma_cntl, ib_cntl; - u32 rb_bufsz; - u32 reg_offset, wb_offset; - int i, r; + if (reset_mask & RADEON_RESET_DMA) { + /* dma0 */ + tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); + tmp &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); + } - /* Reset dma */ - WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1); - RREG32(SRBM_SOFT_RESET); - udelay(50); - WREG32(SRBM_SOFT_RESET, 0); + if (reset_mask & RADEON_RESET_DMA1) { + /* dma1 */ + tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); + tmp &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); + } - for (i = 0; i < 2; i++) { - if (i == 0) { - ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; - reg_offset = DMA0_REGISTER_OFFSET; - wb_offset = R600_WB_DMA_RPTR_OFFSET; - } else { - ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; - reg_offset = DMA1_REGISTER_OFFSET; - wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET; - } + udelay(50); - WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0); - WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0); + evergreen_mc_stop(rdev, &save); + if (evergreen_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); + } - /* Set ring buffer size in dwords */ - rb_bufsz = drm_order(ring->ring_size / 4); - rb_cntl = rb_bufsz << 1; -#ifdef __BIG_ENDIAN - rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE; -#endif - WREG32(DMA_RB_CNTL + reg_offset, rb_cntl); + if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { + grbm_soft_reset = SOFT_RESET_CB | + SOFT_RESET_DB | + SOFT_RESET_GDS | + SOFT_RESET_PA | + SOFT_RESET_SC | + SOFT_RESET_SPI | + SOFT_RESET_SH | + SOFT_RESET_SX | + SOFT_RESET_TC | + SOFT_RESET_TA | + SOFT_RESET_VGT | + SOFT_RESET_IA; + } - /* Initialize the ring buffer's read and write pointers */ - WREG32(DMA_RB_RPTR + reg_offset, 0); - WREG32(DMA_RB_WPTR + reg_offset, 0); + if (reset_mask & RADEON_RESET_CP) { + grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT; - /* set the wb address whether it's enabled or not */ - WREG32(DMA_RB_RPTR_ADDR_HI + reg_offset, - upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFF); - WREG32(DMA_RB_RPTR_ADDR_LO + reg_offset, - ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC)); + srbm_soft_reset |= SOFT_RESET_GRBM; + } - if (rdev->wb.enabled) - rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE; + if (reset_mask & RADEON_RESET_DMA) + srbm_soft_reset |= SOFT_RESET_DMA; - WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8); + if (reset_mask & RADEON_RESET_DMA1) + srbm_soft_reset |= SOFT_RESET_DMA1; - /* enable DMA IBs */ - ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE; -#ifdef __BIG_ENDIAN - ib_cntl |= DMA_IB_SWAP_ENABLE; -#endif - WREG32(DMA_IB_CNTL + reg_offset, ib_cntl); + if (reset_mask & RADEON_RESET_DISPLAY) + srbm_soft_reset |= SOFT_RESET_DC; - dma_cntl = RREG32(DMA_CNTL + reg_offset); - dma_cntl &= ~CTXEMPTY_INT_ENABLE; - WREG32(DMA_CNTL + reg_offset, dma_cntl); + if (reset_mask & RADEON_RESET_RLC) + srbm_soft_reset |= SOFT_RESET_RLC; - ring->wptr = 0; - WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2); + if (reset_mask & RADEON_RESET_SEM) + srbm_soft_reset |= SOFT_RESET_SEM; - ring->rptr = RREG32(DMA_RB_RPTR + reg_offset) >> 2; + if (reset_mask & RADEON_RESET_IH) + srbm_soft_reset |= SOFT_RESET_IH; - WREG32(DMA_RB_CNTL + reg_offset, rb_cntl | DMA_RB_ENABLE); + if (reset_mask & RADEON_RESET_GRBM) + srbm_soft_reset |= SOFT_RESET_GRBM; - ring->ready = true; + if (reset_mask & RADEON_RESET_VMC) + srbm_soft_reset |= SOFT_RESET_VMC; - r = radeon_ring_test(rdev, ring->idx, ring); - if (r) { - ring->ready = false; - return r; - } + if (!(rdev->flags & RADEON_IS_IGP)) { + if (reset_mask & RADEON_RESET_MC) + srbm_soft_reset |= SOFT_RESET_MC; } - radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); - - return 0; -} + if (grbm_soft_reset) { + tmp = RREG32(GRBM_SOFT_RESET); + tmp |= grbm_soft_reset; + dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(GRBM_SOFT_RESET, tmp); + tmp = RREG32(GRBM_SOFT_RESET); -/** - * cayman_dma_fini - tear down the async dma engines - * - * @rdev: radeon_device pointer - * - * Stop the async dma engines and free the rings (cayman-SI). - */ -void cayman_dma_fini(struct radeon_device *rdev) -{ - cayman_dma_stop(rdev); - radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]); - radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]); -} + udelay(50); -static void cayman_gpu_soft_reset_gfx(struct radeon_device *rdev) -{ - u32 grbm_reset = 0; + tmp &= ~grbm_soft_reset; + WREG32(GRBM_SOFT_RESET, tmp); + tmp = RREG32(GRBM_SOFT_RESET); + } - if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) - return; + if (srbm_soft_reset) { + tmp = RREG32(SRBM_SOFT_RESET); + tmp |= srbm_soft_reset; + dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); - dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", - RREG32(GRBM_STATUS)); - dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", - RREG32(GRBM_STATUS_SE0)); - dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", - RREG32(GRBM_STATUS_SE1)); - dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", - RREG32(SRBM_STATUS)); - dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", - RREG32(CP_STALLED_STAT1)); - dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", - RREG32(CP_STALLED_STAT2)); - dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", - RREG32(CP_BUSY_STAT)); - dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", - RREG32(CP_STAT)); + udelay(50); - /* Disable CP parsing/prefetching */ - WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); + tmp &= ~srbm_soft_reset; + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); + } - /* reset all the gfx blocks */ - grbm_reset = (SOFT_RESET_CP | - SOFT_RESET_CB | - SOFT_RESET_DB | - SOFT_RESET_GDS | - SOFT_RESET_PA | - SOFT_RESET_SC | - SOFT_RESET_SPI | - SOFT_RESET_SH | - SOFT_RESET_SX | - SOFT_RESET_TC | - SOFT_RESET_TA | - SOFT_RESET_VGT | - SOFT_RESET_IA); - - dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset); - WREG32(GRBM_SOFT_RESET, grbm_reset); - (void)RREG32(GRBM_SOFT_RESET); + /* Wait a little for things to settle down */ udelay(50); - WREG32(GRBM_SOFT_RESET, 0); - (void)RREG32(GRBM_SOFT_RESET); - - dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", - RREG32(GRBM_STATUS)); - dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", - RREG32(GRBM_STATUS_SE0)); - dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", - RREG32(GRBM_STATUS_SE1)); - dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", - RREG32(SRBM_STATUS)); - dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", - RREG32(CP_STALLED_STAT1)); - dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", - RREG32(CP_STALLED_STAT2)); - dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", - RREG32(CP_BUSY_STAT)); - dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", - RREG32(CP_STAT)); -} - -static void cayman_gpu_soft_reset_dma(struct radeon_device *rdev) -{ - u32 tmp; - - if (RREG32(DMA_STATUS_REG) & DMA_IDLE) - return; - - dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", - RREG32(DMA_STATUS_REG)); - - /* dma0 */ - tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); - tmp &= ~DMA_RB_ENABLE; - WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); - - /* dma1 */ - tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); - tmp &= ~DMA_RB_ENABLE; - WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); - - /* Reset dma */ - WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1); - RREG32(SRBM_SOFT_RESET); + evergreen_mc_resume(rdev, &save); udelay(50); - WREG32(SRBM_SOFT_RESET, 0); - - dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", - RREG32(DMA_STATUS_REG)); + evergreen_print_gpu_status_regs(rdev); } -static int cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) +int cayman_asic_reset(struct radeon_device *rdev) { - struct evergreen_mc_save save; - - if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) - reset_mask &= ~(RADEON_RESET_GFX | RADEON_RESET_COMPUTE); + u32 reset_mask; - if (RREG32(DMA_STATUS_REG) & DMA_IDLE) - reset_mask &= ~RADEON_RESET_DMA; - - if (reset_mask == 0) - return 0; + reset_mask = cayman_gpu_check_soft_reset(rdev); - dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); - - dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_ADDR 0x%08X\n", - RREG32(0x14F8)); - dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n", - RREG32(0x14D8)); - dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", - RREG32(0x14FC)); - dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", - RREG32(0x14DC)); + if (reset_mask) + r600_set_bios_scratch_engine_hung(rdev, true); - evergreen_mc_stop(rdev, &save); - if (evergreen_mc_wait_for_idle(rdev)) { - dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); - } + cayman_gpu_soft_reset(rdev, reset_mask); - if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) - cayman_gpu_soft_reset_gfx(rdev); + reset_mask = cayman_gpu_check_soft_reset(rdev); - if (reset_mask & RADEON_RESET_DMA) - cayman_gpu_soft_reset_dma(rdev); + if (reset_mask) + evergreen_gpu_pci_config_reset(rdev); - /* Wait a little for things to settle down */ - udelay(50); + r600_set_bios_scratch_engine_hung(rdev, false); - evergreen_mc_resume(rdev, &save); return 0; } -int cayman_asic_reset(struct radeon_device *rdev) -{ - return cayman_gpu_soft_reset(rdev, (RADEON_RESET_GFX | - RADEON_RESET_COMPUTE | - RADEON_RESET_DMA)); -} - /** - * cayman_dma_is_lockup - Check if the DMA engine is locked up + * cayman_gfx_is_lockup - Check if the GFX engine is locked up * * @rdev: radeon_device pointer * @ring: radeon_ring structure holding ring information * - * Check if the async DMA engine is locked up (cayman-SI). + * Check if the GFX engine is locked up. * Returns true if the engine appears to be locked up, false if not. */ -bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) { - u32 dma_status_reg; + u32 reset_mask = cayman_gpu_check_soft_reset(rdev); - if (ring->idx == R600_RING_TYPE_DMA_INDEX) - dma_status_reg = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET); - else - dma_status_reg = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET); - if (dma_status_reg & DMA_IDLE) { - radeon_ring_lockup_update(ring); + if (!(reset_mask & (RADEON_RESET_GFX | + RADEON_RESET_COMPUTE | + RADEON_RESET_CP))) { + radeon_ring_lockup_update(rdev, ring); return false; } - /* force ring activities */ - radeon_ring_force_activity(rdev, ring); return radeon_ring_test_lockup(rdev, ring); } @@ -1520,26 +2004,17 @@ static int cayman_startup(struct radeon_device *rdev) /* enable pcie gen2 link */ evergreen_pcie_gen2_enable(rdev); + /* enable aspm */ + evergreen_program_aspm(rdev); - evergreen_mc_program(rdev); + /* scratch needs to be initialized before MC */ + r = r600_vram_scratch_init(rdev); + if (r) + return r; - if (rdev->flags & RADEON_IS_IGP) { - if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { - r = ni_init_microcode(rdev); - if (r) { - DRM_ERROR("Failed to load firmware!\n"); - return r; - } - } - } else { - if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { - r = ni_init_microcode(rdev); - if (r) { - DRM_ERROR("Failed to load firmware!\n"); - return r; - } - } + evergreen_mc_program(rdev); + if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { r = ni_mc_load_microcode(rdev); if (r) { DRM_ERROR("Failed to load MC firmware!\n"); @@ -1547,25 +2022,18 @@ static int cayman_startup(struct radeon_device *rdev) } } - r = r600_vram_scratch_init(rdev); - if (r) - return r; - r = cayman_pcie_gart_enable(rdev); if (r) return r; cayman_gpu_init(rdev); - r = evergreen_blit_init(rdev); - if (r) { - r600_blit_fini(rdev); - rdev->asic->copy.copy = NULL; - dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); - } - /* allocate rlc buffers */ if (rdev->flags & RADEON_IS_IGP) { - r = si_rlc_init(rdev); + rdev->rlc.reg_list = tn_rlc_save_restore_register_list; + rdev->rlc.reg_list_size = + (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list); + rdev->rlc.cs_data = cayman_cs_data; + r = sumo_rlc_init(rdev); if (r) { DRM_ERROR("Failed to init rlc BOs!\n"); return r; @@ -1583,6 +2051,35 @@ static int cayman_startup(struct radeon_device *rdev) return r; } + r = uvd_v2_2_resume(rdev); + if (!r) { + r = radeon_fence_driver_start_ring(rdev, + R600_RING_TYPE_UVD_INDEX); + if (r) + dev_err(rdev->dev, "UVD fences init error (%d).\n", r); + } + if (r) + rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; + + if (rdev->family == CHIP_ARUBA) { + r = radeon_vce_resume(rdev); + if (!r) + r = vce_v1_0_resume(rdev); + + if (!r) + r = radeon_fence_driver_start_ring(rdev, + TN_RING_TYPE_VCE1_INDEX); + if (!r) + r = radeon_fence_driver_start_ring(rdev, + TN_RING_TYPE_VCE2_INDEX); + + if (r) { + dev_err(rdev->dev, "VCE init error (%d).\n", r); + rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; + rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; + } + } + r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); if (r) { dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); @@ -1623,24 +2120,19 @@ static int cayman_startup(struct radeon_device *rdev) evergreen_irq_set(rdev); r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, - CP_RB0_RPTR, CP_RB0_WPTR, - 0, 0xfffff, RADEON_CP_PACKET2); + RADEON_CP_PACKET2); if (r) return r; ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, - DMA_RB_RPTR + DMA0_REGISTER_OFFSET, - DMA_RB_WPTR + DMA0_REGISTER_OFFSET, - 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); + DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); if (r) return r; ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, - DMA_RB_RPTR + DMA1_REGISTER_OFFSET, - DMA_RB_WPTR + DMA1_REGISTER_OFFSET, - 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); + DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); if (r) return r; @@ -1655,6 +2147,31 @@ static int cayman_startup(struct radeon_device *rdev) if (r) return r; + ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + if (ring->ring_size) { + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + RADEON_CP_PACKET2); + if (!r) + r = uvd_v1_0_init(rdev); + if (r) + DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); + } + + if (rdev->family == CHIP_ARUBA) { + ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; + if (ring->ring_size) + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0); + + ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; + if (ring->ring_size) + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0); + + if (!r) + r = vce_v1_0_init(rdev); + if (r) + DRM_ERROR("radeon: failed initializing VCE (%d).\n", r); + } + r = radeon_ib_pool_init(rdev); if (r) { dev_err(rdev->dev, "IB initialization failed (%d).\n", r); @@ -1667,7 +2184,7 @@ static int cayman_startup(struct radeon_device *rdev) return r; } - r = r600_audio_init(rdev); + r = radeon_audio_init(rdev); if (r) return r; @@ -1685,6 +2202,12 @@ int cayman_resume(struct radeon_device *rdev) /* post card */ atom_asic_init(rdev->mode_info.atom_context); + /* init golden registers */ + ni_init_golden_registers(rdev); + + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_resume(rdev); + rdev->accel_working = true; r = cayman_startup(rdev); if (r) { @@ -1697,10 +2220,13 @@ int cayman_resume(struct radeon_device *rdev) int cayman_suspend(struct radeon_device *rdev) { - r600_audio_fini(rdev); + radeon_pm_suspend(rdev); + radeon_audio_fini(rdev); radeon_vm_manager_fini(rdev); cayman_cp_enable(rdev, false); cayman_dma_stop(rdev); + uvd_v1_0_fini(rdev); + radeon_uvd_suspend(rdev); evergreen_irq_suspend(rdev); radeon_wb_disable(rdev); cayman_pcie_gart_disable(rdev); @@ -1741,6 +2267,8 @@ int cayman_init(struct radeon_device *rdev) DRM_INFO("GPU not posted. posting now...\n"); atom_asic_init(rdev->mode_info.atom_context); } + /* init golden registers */ + ni_init_golden_registers(rdev); /* Initialize scratch registers */ r600_scratch_init(rdev); /* Initialize surface registers */ @@ -1760,6 +2288,27 @@ int cayman_init(struct radeon_device *rdev) if (r) return r; + if (rdev->flags & RADEON_IS_IGP) { + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { + r = ni_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + } else { + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { + r = ni_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + } + + /* Initialize power management */ + radeon_pm_init(rdev); + ring->ring_obj = NULL; r600_ring_init(rdev, ring, 1024 * 1024); @@ -1771,6 +2320,26 @@ int cayman_init(struct radeon_device *rdev) ring->ring_obj = NULL; r600_ring_init(rdev, ring, 64 * 1024); + r = radeon_uvd_init(rdev); + if (!r) { + ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 4096); + } + + if (rdev->family == CHIP_ARUBA) { + r = radeon_vce_init(rdev); + if (!r) { + ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 4096); + + ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 4096); + } + } + rdev->ih.ring_obj = NULL; r600_ih_ring_init(rdev, 64 * 1024); @@ -1786,7 +2355,7 @@ int cayman_init(struct radeon_device *rdev) cayman_dma_fini(rdev); r600_irq_fini(rdev); if (rdev->flags & RADEON_IS_IGP) - si_rlc_fini(rdev); + sumo_rlc_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); radeon_vm_manager_fini(rdev); @@ -1812,16 +2381,20 @@ int cayman_init(struct radeon_device *rdev) void cayman_fini(struct radeon_device *rdev) { - r600_blit_fini(rdev); + radeon_pm_fini(rdev); cayman_cp_fini(rdev); cayman_dma_fini(rdev); r600_irq_fini(rdev); if (rdev->flags & RADEON_IS_IGP) - si_rlc_fini(rdev); + sumo_rlc_fini(rdev); radeon_wb_fini(rdev); radeon_vm_manager_fini(rdev); radeon_ib_pool_fini(rdev); radeon_irq_kms_fini(rdev); + uvd_v1_0_fini(rdev); + radeon_uvd_fini(rdev); + if (rdev->family == CHIP_ARUBA) + radeon_vce_fini(rdev); cayman_pcie_gart_fini(rdev); r600_vram_scratch_fini(rdev); radeon_gem_fini(rdev); @@ -1853,96 +2426,165 @@ void cayman_vm_fini(struct radeon_device *rdev) { } -#define R600_ENTRY_VALID (1 << 0) -#define R600_PTE_SYSTEM (1 << 1) -#define R600_PTE_SNOOPED (1 << 2) -#define R600_PTE_READABLE (1 << 5) -#define R600_PTE_WRITEABLE (1 << 6) - -uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags) -{ - uint32_t r600_flags = 0; - r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_ENTRY_VALID : 0; - r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0; - r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0; - if (flags & RADEON_VM_PAGE_SYSTEM) { - r600_flags |= R600_PTE_SYSTEM; - r600_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0; - } - return r600_flags; -} - /** - * cayman_vm_set_page - update the page tables using the CP + * cayman_vm_decode_fault - print human readable fault info * * @rdev: radeon_device pointer - * @pe: addr of the page entry - * @addr: dst addr to write into pe - * @count: number of page entries to update - * @incr: increase next addr by incr bytes - * @flags: access flags + * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value + * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value * - * Update the page tables using the CP (cayman-si). + * Print human readable fault information (cayman/TN). */ -void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe, - uint64_t addr, unsigned count, - uint32_t incr, uint32_t flags) +void cayman_vm_decode_fault(struct radeon_device *rdev, + u32 status, u32 addr) { - struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index]; - uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); - uint64_t value; - unsigned ndw; - - if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) { - while (count) { - ndw = 1 + count * 2; - if (ndw > 0x3FFF) - ndw = 0x3FFF; - - radeon_ring_write(ring, PACKET3(PACKET3_ME_WRITE, ndw)); - radeon_ring_write(ring, pe); - radeon_ring_write(ring, upper_32_bits(pe) & 0xff); - for (; ndw > 1; ndw -= 2, --count, pe += 8) { - if (flags & RADEON_VM_PAGE_SYSTEM) { - value = radeon_vm_map_gart(rdev, addr); - value &= 0xFFFFFFFFFFFFF000ULL; - } else if (flags & RADEON_VM_PAGE_VALID) { - value = addr; - } else { - value = 0; - } - addr += incr; - value |= r600_flags; - radeon_ring_write(ring, value); - radeon_ring_write(ring, upper_32_bits(value)); - } - } - } else { - while (count) { - ndw = count * 2; - if (ndw > 0xFFFFE) - ndw = 0xFFFFE; - - /* for non-physically contiguous pages (system) */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw)); - radeon_ring_write(ring, pe); - radeon_ring_write(ring, upper_32_bits(pe) & 0xff); - for (; ndw > 0; ndw -= 2, --count, pe += 8) { - if (flags & RADEON_VM_PAGE_SYSTEM) { - value = radeon_vm_map_gart(rdev, addr); - value &= 0xFFFFFFFFFFFFF000ULL; - } else if (flags & RADEON_VM_PAGE_VALID) { - value = addr; - } else { - value = 0; - } - addr += incr; - value |= r600_flags; - radeon_ring_write(ring, value); - radeon_ring_write(ring, upper_32_bits(value)); - } - } + u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; + u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; + u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; + char *block; + + switch (mc_id) { + case 32: + case 16: + case 96: + case 80: + case 160: + case 144: + case 224: + case 208: + block = "CB"; + break; + case 33: + case 17: + case 97: + case 81: + case 161: + case 145: + case 225: + case 209: + block = "CB_FMASK"; + break; + case 34: + case 18: + case 98: + case 82: + case 162: + case 146: + case 226: + case 210: + block = "CB_CMASK"; + break; + case 35: + case 19: + case 99: + case 83: + case 163: + case 147: + case 227: + case 211: + block = "CB_IMMED"; + break; + case 36: + case 20: + case 100: + case 84: + case 164: + case 148: + case 228: + case 212: + block = "DB"; + break; + case 37: + case 21: + case 101: + case 85: + case 165: + case 149: + case 229: + case 213: + block = "DB_HTILE"; + break; + case 38: + case 22: + case 102: + case 86: + case 166: + case 150: + case 230: + case 214: + block = "SX"; + break; + case 39: + case 23: + case 103: + case 87: + case 167: + case 151: + case 231: + case 215: + block = "DB_STEN"; + break; + case 40: + case 24: + case 104: + case 88: + case 232: + case 216: + case 168: + case 152: + block = "TC_TFETCH"; + break; + case 41: + case 25: + case 105: + case 89: + case 233: + case 217: + case 169: + case 153: + block = "TC_VFETCH"; + break; + case 42: + case 26: + case 106: + case 90: + case 234: + case 218: + case 170: + case 154: + block = "VC"; + break; + case 112: + block = "CP"; + break; + case 113: + case 114: + block = "SH"; + break; + case 115: + block = "VGT"; + break; + case 178: + block = "IH"; + break; + case 51: + block = "RLC"; + break; + case 55: + block = "DMA"; + break; + case 56: + block = "HDP"; + break; + default: + block = "unknown"; + break; } + + printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n", + protections, vmid, addr, + (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", + block, mc_id); } /** @@ -1953,15 +2595,11 @@ void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe, * Update the page table base and flush the VM TLB * using the CP (cayman-si). */ -void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) { - struct radeon_ring *ring = &rdev->ring[ridx]; - - if (vm == NULL) - return; - - radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0)); - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0)); + radeon_ring_write(ring, pd_addr >> 12); /* flush hdp cache */ radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0)); @@ -1969,32 +2607,50 @@ void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) /* bits 0-7 are the VM contexts0-7 */ radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0)); - radeon_ring_write(ring, 1 << vm->id); + radeon_ring_write(ring, 1 << vm_id); + + /* wait for the invalidate to complete */ + radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); + radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) | /* always */ + WAIT_REG_MEM_ENGINE(0))); /* me */ + radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, 0); /* ref */ + radeon_ring_write(ring, 0); /* mask */ + radeon_ring_write(ring, 0x20); /* poll interval */ /* sync PFP to ME, otherwise we might get invalid PFP reads */ radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); radeon_ring_write(ring, 0x0); } -void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) { - struct radeon_ring *ring = &rdev->ring[ridx]; + struct atom_clock_dividers dividers; + int r, i; - if (vm == NULL) - return; + r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + ecclk, false, ÷rs); + if (r) + return r; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); - radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2)); - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + for (i = 0; i < 100; i++) { + if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS) + break; + mdelay(10); + } + if (i == 100) + return -ETIMEDOUT; - /* flush hdp cache */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); - radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); - radeon_ring_write(ring, 1); + WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK)); - /* bits 0-7 are the VM contexts0-7 */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); - radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2)); - radeon_ring_write(ring, 1 << vm->id); -} + for (i = 0; i < 100; i++) { + if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS) + break; + mdelay(10); + } + if (i == 100) + return -ETIMEDOUT; + return 0; +} diff --git a/sys/dev/pci/drm/radeon/ni_dma.c b/sys/dev/pci/drm/radeon/ni_dma.c new file mode 100644 index 00000000000..146656689a2 --- /dev/null +++ b/sys/dev/pci/drm/radeon/ni_dma.c @@ -0,0 +1,472 @@ +/* + * Copyright 2010 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "radeon_trace.h" +#include "nid.h" + +u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev); + +/* + * DMA + * Starting with R600, the GPU has an asynchronous + * DMA engine. The programming model is very similar + * to the 3D engine (ring buffer, IBs, etc.), but the + * DMA controller has it's own packet format that is + * different form the PM4 format used by the 3D engine. + * It supports copying data, writing embedded data, + * solid fills, and a number of other things. It also + * has support for tiling/detiling of buffers. + * Cayman and newer support two asynchronous DMA engines. + */ + +/** + * cayman_dma_get_rptr - get the current read pointer + * + * @rdev: radeon_device pointer + * @ring: radeon ring pointer + * + * Get the current rptr from the hardware (cayman+). + */ +uint32_t cayman_dma_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 rptr, reg; + + if (rdev->wb.enabled) { + rptr = rdev->wb.wb[ring->rptr_offs/4]; + } else { + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + reg = DMA_RB_RPTR + DMA0_REGISTER_OFFSET; + else + reg = DMA_RB_RPTR + DMA1_REGISTER_OFFSET; + + rptr = RREG32(reg); + } + + return (rptr & 0x3fffc) >> 2; +} + +/** + * cayman_dma_get_wptr - get the current write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon ring pointer + * + * Get the current wptr from the hardware (cayman+). + */ +uint32_t cayman_dma_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 reg; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + reg = DMA_RB_WPTR + DMA0_REGISTER_OFFSET; + else + reg = DMA_RB_WPTR + DMA1_REGISTER_OFFSET; + + return (RREG32(reg) & 0x3fffc) >> 2; +} + +/** + * cayman_dma_set_wptr - commit the write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon ring pointer + * + * Write the wptr back to the hardware (cayman+). + */ +void cayman_dma_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 reg; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + reg = DMA_RB_WPTR + DMA0_REGISTER_OFFSET; + else + reg = DMA_RB_WPTR + DMA1_REGISTER_OFFSET; + + WREG32(reg, (ring->wptr << 2) & 0x3fffc); +} + +/** + * cayman_dma_ring_ib_execute - Schedule an IB on the DMA engine + * + * @rdev: radeon_device pointer + * @ib: IB object to schedule + * + * Schedule an IB in the DMA ring (cayman-SI). + */ +void cayman_dma_ring_ib_execute(struct radeon_device *rdev, + struct radeon_ib *ib) +{ + struct radeon_ring *ring = &rdev->ring[ib->ring]; + unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; + + if (rdev->wb.enabled) { + u32 next_rptr = ring->wptr + 4; + while ((next_rptr & 7) != 5) + next_rptr++; + next_rptr += 3; + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1)); + radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); + radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff); + radeon_ring_write(ring, next_rptr); + } + + /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. + * Pad as necessary with NOPs. + */ + while ((ring->wptr & 7) != 5) + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); + radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, vm_id, 0)); + radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); + radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); + +} + +/** + * cayman_dma_stop - stop the async dma engines + * + * @rdev: radeon_device pointer + * + * Stop the async dma engines (cayman-SI). + */ +void cayman_dma_stop(struct radeon_device *rdev) +{ + u32 rb_cntl; + + if ((rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) || + (rdev->asic->copy.copy_ring_index == CAYMAN_RING_TYPE_DMA1_INDEX)) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + + /* dma0 */ + rb_cntl = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); + rb_cntl &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, rb_cntl); + + /* dma1 */ + rb_cntl = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); + rb_cntl &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, rb_cntl); + + rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false; + rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false; +} + +/** + * cayman_dma_resume - setup and start the async dma engines + * + * @rdev: radeon_device pointer + * + * Set up the DMA ring buffers and enable them. (cayman-SI). + * Returns 0 for success, error for failure. + */ +int cayman_dma_resume(struct radeon_device *rdev) +{ + struct radeon_ring *ring; + u32 rb_cntl, dma_cntl, ib_cntl; + u32 rb_bufsz; + u32 reg_offset, wb_offset; + int i, r; + + for (i = 0; i < 2; i++) { + if (i == 0) { + ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; + reg_offset = DMA0_REGISTER_OFFSET; + wb_offset = R600_WB_DMA_RPTR_OFFSET; + } else { + ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; + reg_offset = DMA1_REGISTER_OFFSET; + wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET; + } + + WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0); + WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0); + + /* Set ring buffer size in dwords */ + rb_bufsz = order_base_2(ring->ring_size / 4); + rb_cntl = rb_bufsz << 1; +#ifdef __BIG_ENDIAN + rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE; +#endif + WREG32(DMA_RB_CNTL + reg_offset, rb_cntl); + + /* Initialize the ring buffer's read and write pointers */ + WREG32(DMA_RB_RPTR + reg_offset, 0); + WREG32(DMA_RB_WPTR + reg_offset, 0); + + /* set the wb address whether it's enabled or not */ + WREG32(DMA_RB_RPTR_ADDR_HI + reg_offset, + upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFF); + WREG32(DMA_RB_RPTR_ADDR_LO + reg_offset, + ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC)); + + if (rdev->wb.enabled) + rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE; + + WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8); + + /* enable DMA IBs */ + ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE; +#ifdef __BIG_ENDIAN + ib_cntl |= DMA_IB_SWAP_ENABLE; +#endif + WREG32(DMA_IB_CNTL + reg_offset, ib_cntl); + + dma_cntl = RREG32(DMA_CNTL + reg_offset); + dma_cntl &= ~CTXEMPTY_INT_ENABLE; + WREG32(DMA_CNTL + reg_offset, dma_cntl); + + ring->wptr = 0; + WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2); + + WREG32(DMA_RB_CNTL + reg_offset, rb_cntl | DMA_RB_ENABLE); + + ring->ready = true; + + r = radeon_ring_test(rdev, ring->idx, ring); + if (r) { + ring->ready = false; + return r; + } + } + + if ((rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) || + (rdev->asic->copy.copy_ring_index == CAYMAN_RING_TYPE_DMA1_INDEX)) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); + + return 0; +} + +/** + * cayman_dma_fini - tear down the async dma engines + * + * @rdev: radeon_device pointer + * + * Stop the async dma engines and free the rings (cayman-SI). + */ +void cayman_dma_fini(struct radeon_device *rdev) +{ + cayman_dma_stop(rdev); + radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]); + radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]); +} + +/** + * cayman_dma_is_lockup - Check if the DMA engine is locked up + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Check if the async DMA engine is locked up. + * Returns true if the engine appears to be locked up, false if not. + */ +bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +{ + u32 reset_mask = cayman_gpu_check_soft_reset(rdev); + u32 mask; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + mask = RADEON_RESET_DMA; + else + mask = RADEON_RESET_DMA1; + + if (!(reset_mask & mask)) { + radeon_ring_lockup_update(rdev, ring); + return false; + } + return radeon_ring_test_lockup(rdev, ring); +} + +/** + * cayman_dma_vm_copy_pages - update PTEs by copying them from the GART + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @src: src addr where to copy from + * @count: number of page entries to update + * + * Update PTEs by copying them from the GART using the DMA (cayman/TN). + */ +void cayman_dma_vm_copy_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, uint64_t src, + unsigned count) +{ + unsigned ndw; + + while (count) { + ndw = count * 2; + if (ndw > 0xFFFFE) + ndw = 0xFFFFE; + + ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY, + 0, 0, ndw); + ib->ptr[ib->length_dw++] = lower_32_bits(pe); + ib->ptr[ib->length_dw++] = lower_32_bits(src); + ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; + ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff; + + pe += ndw * 4; + src += ndw * 4; + count -= ndw / 2; + } +} + +/** + * cayman_dma_vm_write_pages - update PTEs by writing them manually + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: hw access flags + * + * Update PTEs by writing them manually using the DMA (cayman/TN). + */ +void cayman_dma_vm_write_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags) +{ + uint64_t value; + unsigned ndw; + + while (count) { + ndw = count * 2; + if (ndw > 0xFFFFE) + ndw = 0xFFFFE; + + /* for non-physically contiguous pages (system) */ + ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, + 0, 0, ndw); + ib->ptr[ib->length_dw++] = pe; + ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; + for (; ndw > 0; ndw -= 2, --count, pe += 8) { + if (flags & R600_PTE_SYSTEM) { + value = radeon_vm_map_gart(rdev, addr); + } else if (flags & R600_PTE_VALID) { + value = addr; + } else { + value = 0; + } + addr += incr; + value |= flags; + ib->ptr[ib->length_dw++] = value; + ib->ptr[ib->length_dw++] = upper_32_bits(value); + } + } +} + +/** + * cayman_dma_vm_set_pages - update the page tables using the DMA + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: hw access flags + * + * Update the page tables using the DMA (cayman/TN). + */ +void cayman_dma_vm_set_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags) +{ + uint64_t value; + unsigned ndw; + + while (count) { + ndw = count * 2; + if (ndw > 0xFFFFE) + ndw = 0xFFFFE; + + if (flags & R600_PTE_VALID) + value = addr; + else + value = 0; + + /* for physically contiguous pages (vram) */ + ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw); + ib->ptr[ib->length_dw++] = pe; /* dst addr */ + ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; + ib->ptr[ib->length_dw++] = flags; /* mask */ + ib->ptr[ib->length_dw++] = 0; + ib->ptr[ib->length_dw++] = value; /* value */ + ib->ptr[ib->length_dw++] = upper_32_bits(value); + ib->ptr[ib->length_dw++] = incr; /* increment size */ + ib->ptr[ib->length_dw++] = 0; + + pe += ndw * 4; + addr += (ndw / 2) * incr; + count -= ndw / 2; + } +} + +/** + * cayman_dma_vm_pad_ib - pad the IB to the required number of dw + * + * @ib: indirect buffer to fill with padding + * + */ +void cayman_dma_vm_pad_ib(struct radeon_ib *ib) +{ + while (ib->length_dw & 0x7) + ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0); +} + +void cayman_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) +{ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); + radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2)); + radeon_ring_write(ring, pd_addr >> 12); + + /* flush hdp cache */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); + radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); + radeon_ring_write(ring, 1); + + /* bits 0-7 are the VM contexts0-7 */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); + radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2)); + radeon_ring_write(ring, 1 << vm_id); + + /* wait for invalidate to complete */ + radeon_ring_write(ring, DMA_SRBM_READ_PACKET); + radeon_ring_write(ring, (0xff << 20) | (VM_INVALIDATE_REQUEST >> 2)); + radeon_ring_write(ring, 0); /* mask */ + radeon_ring_write(ring, 0); /* value */ +} + diff --git a/sys/dev/pci/drm/radeon/ni_dpm.c b/sys/dev/pci/drm/radeon/ni_dpm.c new file mode 100644 index 00000000000..b30db1be6d0 --- /dev/null +++ b/sys/dev/pci/drm/radeon/ni_dpm.c @@ -0,0 +1,4377 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "nid.h" +#include "r600_dpm.h" +#include "ni_dpm.h" +#include "atom.h" + +#define MC_CG_ARB_FREQ_F0 0x0a +#define MC_CG_ARB_FREQ_F1 0x0b +#define MC_CG_ARB_FREQ_F2 0x0c +#define MC_CG_ARB_FREQ_F3 0x0d + +#define SMC_RAM_END 0xC000 + +static const struct ni_cac_weights cac_weights_cayman_xt = +{ + 0x15, + 0x2, + 0x19, + 0x2, + 0x8, + 0x14, + 0x2, + 0x16, + 0xE, + 0x17, + 0x13, + 0x2B, + 0x10, + 0x7, + 0x5, + 0x5, + 0x5, + 0x2, + 0x3, + 0x9, + 0x10, + 0x10, + 0x2B, + 0xA, + 0x9, + 0x4, + 0xD, + 0xD, + 0x3E, + 0x18, + 0x14, + 0, + 0x3, + 0x3, + 0x5, + 0, + 0x2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0x1CC, + 0, + 0x164, + 1, + 1, + 1, + 1, + 12, + 12, + 12, + 0x12, + 0x1F, + 132, + 5, + 7, + 0, + { 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0 }, + true +}; + +static const struct ni_cac_weights cac_weights_cayman_pro = +{ + 0x16, + 0x4, + 0x10, + 0x2, + 0xA, + 0x16, + 0x2, + 0x18, + 0x10, + 0x1A, + 0x16, + 0x2D, + 0x12, + 0xA, + 0x6, + 0x6, + 0x6, + 0x2, + 0x4, + 0xB, + 0x11, + 0x11, + 0x2D, + 0xC, + 0xC, + 0x7, + 0x10, + 0x10, + 0x3F, + 0x1A, + 0x16, + 0, + 0x7, + 0x4, + 0x6, + 1, + 0x2, + 0x1, + 0, + 0, + 0, + 0, + 0, + 0, + 0x30, + 0, + 0x1CF, + 0, + 0x166, + 1, + 1, + 1, + 1, + 12, + 12, + 12, + 0x15, + 0x1F, + 132, + 6, + 6, + 0, + { 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0 }, + true +}; + +static const struct ni_cac_weights cac_weights_cayman_le = +{ + 0x7, + 0xE, + 0x1, + 0xA, + 0x1, + 0x3F, + 0x2, + 0x18, + 0x10, + 0x1A, + 0x1, + 0x3F, + 0x1, + 0xE, + 0x6, + 0x6, + 0x6, + 0x2, + 0x4, + 0x9, + 0x1A, + 0x1A, + 0x2C, + 0xA, + 0x11, + 0x8, + 0x19, + 0x19, + 0x1, + 0x1, + 0x1A, + 0, + 0x8, + 0x5, + 0x8, + 0x1, + 0x3, + 0x1, + 0, + 0, + 0, + 0, + 0, + 0, + 0x38, + 0x38, + 0x239, + 0x3, + 0x18A, + 1, + 1, + 1, + 1, + 12, + 12, + 12, + 0x15, + 0x22, + 132, + 6, + 6, + 0, + { 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0 }, + true +}; + +#define NISLANDS_MGCG_SEQUENCE 300 + +static const u32 cayman_cgcg_cgls_default[] = +{ + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff +}; +#define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32)) + +static const u32 cayman_cgcg_cgls_disable[] = +{ + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x00000644, 0x000f7902, 0x001f4180, + 0x00000644, 0x000f3802, 0x001f4180 +}; +#define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32)) + +static const u32 cayman_cgcg_cgls_enable[] = +{ + 0x00000644, 0x000f7882, 0x001f4080, + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000020, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000021, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000022, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000023, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000024, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000025, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000026, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000027, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000028, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000029, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000002a, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x0000002b, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff +}; +#define CAYMAN_CGCG_CGLS_ENABLE_LENGTH sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32)) + +static const u32 cayman_mgcg_default[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x00003fc4, 0xc0000000, 0xffffffff, + 0x00005448, 0x00000100, 0xffffffff, + 0x000055e4, 0x00000100, 0xffffffff, + 0x0000160c, 0x00000100, 0xffffffff, + 0x00008984, 0x06000100, 0xffffffff, + 0x0000c164, 0x00000100, 0xffffffff, + 0x00008a18, 0x00000100, 0xffffffff, + 0x0000897c, 0x06000100, 0xffffffff, + 0x00008b28, 0x00000100, 0xffffffff, + 0x00009144, 0x00800200, 0xffffffff, + 0x00009a60, 0x00000100, 0xffffffff, + 0x00009868, 0x00000100, 0xffffffff, + 0x00008d58, 0x00000100, 0xffffffff, + 0x00009510, 0x00000100, 0xffffffff, + 0x0000949c, 0x00000100, 0xffffffff, + 0x00009654, 0x00000100, 0xffffffff, + 0x00009030, 0x00000100, 0xffffffff, + 0x00009034, 0x00000100, 0xffffffff, + 0x00009038, 0x00000100, 0xffffffff, + 0x0000903c, 0x00000100, 0xffffffff, + 0x00009040, 0x00000100, 0xffffffff, + 0x0000a200, 0x00000100, 0xffffffff, + 0x0000a204, 0x00000100, 0xffffffff, + 0x0000a208, 0x00000100, 0xffffffff, + 0x0000a20c, 0x00000100, 0xffffffff, + 0x00009744, 0x00000100, 0xffffffff, + 0x00003f80, 0x00000100, 0xffffffff, + 0x0000a210, 0x00000100, 0xffffffff, + 0x0000a214, 0x00000100, 0xffffffff, + 0x000004d8, 0x00000100, 0xffffffff, + 0x00009664, 0x00000100, 0xffffffff, + 0x00009698, 0x00000100, 0xffffffff, + 0x000004d4, 0x00000200, 0xffffffff, + 0x000004d0, 0x00000000, 0xffffffff, + 0x000030cc, 0x00000104, 0xffffffff, + 0x0000d0c0, 0x00000100, 0xffffffff, + 0x0000d8c0, 0x00000100, 0xffffffff, + 0x0000802c, 0x40000000, 0xffffffff, + 0x00003fc4, 0x40000000, 0xffffffff, + 0x0000915c, 0x00010000, 0xffffffff, + 0x00009160, 0x00030002, 0xffffffff, + 0x00009164, 0x00050004, 0xffffffff, + 0x00009168, 0x00070006, 0xffffffff, + 0x00009178, 0x00070000, 0xffffffff, + 0x0000917c, 0x00030002, 0xffffffff, + 0x00009180, 0x00050004, 0xffffffff, + 0x0000918c, 0x00010006, 0xffffffff, + 0x00009190, 0x00090008, 0xffffffff, + 0x00009194, 0x00070000, 0xffffffff, + 0x00009198, 0x00030002, 0xffffffff, + 0x0000919c, 0x00050004, 0xffffffff, + 0x000091a8, 0x00010006, 0xffffffff, + 0x000091ac, 0x00090008, 0xffffffff, + 0x000091b0, 0x00070000, 0xffffffff, + 0x000091b4, 0x00030002, 0xffffffff, + 0x000091b8, 0x00050004, 0xffffffff, + 0x000091c4, 0x00010006, 0xffffffff, + 0x000091c8, 0x00090008, 0xffffffff, + 0x000091cc, 0x00070000, 0xffffffff, + 0x000091d0, 0x00030002, 0xffffffff, + 0x000091d4, 0x00050004, 0xffffffff, + 0x000091e0, 0x00010006, 0xffffffff, + 0x000091e4, 0x00090008, 0xffffffff, + 0x000091e8, 0x00000000, 0xffffffff, + 0x000091ec, 0x00070000, 0xffffffff, + 0x000091f0, 0x00030002, 0xffffffff, + 0x000091f4, 0x00050004, 0xffffffff, + 0x00009200, 0x00010006, 0xffffffff, + 0x00009204, 0x00090008, 0xffffffff, + 0x00009208, 0x00070000, 0xffffffff, + 0x0000920c, 0x00030002, 0xffffffff, + 0x00009210, 0x00050004, 0xffffffff, + 0x0000921c, 0x00010006, 0xffffffff, + 0x00009220, 0x00090008, 0xffffffff, + 0x00009224, 0x00070000, 0xffffffff, + 0x00009228, 0x00030002, 0xffffffff, + 0x0000922c, 0x00050004, 0xffffffff, + 0x00009238, 0x00010006, 0xffffffff, + 0x0000923c, 0x00090008, 0xffffffff, + 0x00009240, 0x00070000, 0xffffffff, + 0x00009244, 0x00030002, 0xffffffff, + 0x00009248, 0x00050004, 0xffffffff, + 0x00009254, 0x00010006, 0xffffffff, + 0x00009258, 0x00090008, 0xffffffff, + 0x0000925c, 0x00070000, 0xffffffff, + 0x00009260, 0x00030002, 0xffffffff, + 0x00009264, 0x00050004, 0xffffffff, + 0x00009270, 0x00010006, 0xffffffff, + 0x00009274, 0x00090008, 0xffffffff, + 0x00009278, 0x00070000, 0xffffffff, + 0x0000927c, 0x00030002, 0xffffffff, + 0x00009280, 0x00050004, 0xffffffff, + 0x0000928c, 0x00010006, 0xffffffff, + 0x00009290, 0x00090008, 0xffffffff, + 0x000092a8, 0x00070000, 0xffffffff, + 0x000092ac, 0x00030002, 0xffffffff, + 0x000092b0, 0x00050004, 0xffffffff, + 0x000092bc, 0x00010006, 0xffffffff, + 0x000092c0, 0x00090008, 0xffffffff, + 0x000092c4, 0x00070000, 0xffffffff, + 0x000092c8, 0x00030002, 0xffffffff, + 0x000092cc, 0x00050004, 0xffffffff, + 0x000092d8, 0x00010006, 0xffffffff, + 0x000092dc, 0x00090008, 0xffffffff, + 0x00009294, 0x00000000, 0xffffffff, + 0x0000802c, 0x40010000, 0xffffffff, + 0x00003fc4, 0x40010000, 0xffffffff, + 0x0000915c, 0x00010000, 0xffffffff, + 0x00009160, 0x00030002, 0xffffffff, + 0x00009164, 0x00050004, 0xffffffff, + 0x00009168, 0x00070006, 0xffffffff, + 0x00009178, 0x00070000, 0xffffffff, + 0x0000917c, 0x00030002, 0xffffffff, + 0x00009180, 0x00050004, 0xffffffff, + 0x0000918c, 0x00010006, 0xffffffff, + 0x00009190, 0x00090008, 0xffffffff, + 0x00009194, 0x00070000, 0xffffffff, + 0x00009198, 0x00030002, 0xffffffff, + 0x0000919c, 0x00050004, 0xffffffff, + 0x000091a8, 0x00010006, 0xffffffff, + 0x000091ac, 0x00090008, 0xffffffff, + 0x000091b0, 0x00070000, 0xffffffff, + 0x000091b4, 0x00030002, 0xffffffff, + 0x000091b8, 0x00050004, 0xffffffff, + 0x000091c4, 0x00010006, 0xffffffff, + 0x000091c8, 0x00090008, 0xffffffff, + 0x000091cc, 0x00070000, 0xffffffff, + 0x000091d0, 0x00030002, 0xffffffff, + 0x000091d4, 0x00050004, 0xffffffff, + 0x000091e0, 0x00010006, 0xffffffff, + 0x000091e4, 0x00090008, 0xffffffff, + 0x000091e8, 0x00000000, 0xffffffff, + 0x000091ec, 0x00070000, 0xffffffff, + 0x000091f0, 0x00030002, 0xffffffff, + 0x000091f4, 0x00050004, 0xffffffff, + 0x00009200, 0x00010006, 0xffffffff, + 0x00009204, 0x00090008, 0xffffffff, + 0x00009208, 0x00070000, 0xffffffff, + 0x0000920c, 0x00030002, 0xffffffff, + 0x00009210, 0x00050004, 0xffffffff, + 0x0000921c, 0x00010006, 0xffffffff, + 0x00009220, 0x00090008, 0xffffffff, + 0x00009224, 0x00070000, 0xffffffff, + 0x00009228, 0x00030002, 0xffffffff, + 0x0000922c, 0x00050004, 0xffffffff, + 0x00009238, 0x00010006, 0xffffffff, + 0x0000923c, 0x00090008, 0xffffffff, + 0x00009240, 0x00070000, 0xffffffff, + 0x00009244, 0x00030002, 0xffffffff, + 0x00009248, 0x00050004, 0xffffffff, + 0x00009254, 0x00010006, 0xffffffff, + 0x00009258, 0x00090008, 0xffffffff, + 0x0000925c, 0x00070000, 0xffffffff, + 0x00009260, 0x00030002, 0xffffffff, + 0x00009264, 0x00050004, 0xffffffff, + 0x00009270, 0x00010006, 0xffffffff, + 0x00009274, 0x00090008, 0xffffffff, + 0x00009278, 0x00070000, 0xffffffff, + 0x0000927c, 0x00030002, 0xffffffff, + 0x00009280, 0x00050004, 0xffffffff, + 0x0000928c, 0x00010006, 0xffffffff, + 0x00009290, 0x00090008, 0xffffffff, + 0x000092a8, 0x00070000, 0xffffffff, + 0x000092ac, 0x00030002, 0xffffffff, + 0x000092b0, 0x00050004, 0xffffffff, + 0x000092bc, 0x00010006, 0xffffffff, + 0x000092c0, 0x00090008, 0xffffffff, + 0x000092c4, 0x00070000, 0xffffffff, + 0x000092c8, 0x00030002, 0xffffffff, + 0x000092cc, 0x00050004, 0xffffffff, + 0x000092d8, 0x00010006, 0xffffffff, + 0x000092dc, 0x00090008, 0xffffffff, + 0x00009294, 0x00000000, 0xffffffff, + 0x0000802c, 0xc0000000, 0xffffffff, + 0x00003fc4, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000010, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000011, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000012, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000013, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000014, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000015, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000016, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000017, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000018, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000019, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001a, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x0000001b, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff +}; +#define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32)) + +static const u32 cayman_mgcg_disable[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000002, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x000008f8, 0x00000003, 0xffffffff, + 0x000008fc, 0xffffffff, 0xffffffff, + 0x00009150, 0x00600000, 0xffffffff +}; +#define CAYMAN_MGCG_DISABLE_LENGTH sizeof(cayman_mgcg_disable) / (3 * sizeof(u32)) + +static const u32 cayman_mgcg_enable[] = +{ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x000008f8, 0x00000002, 0xffffffff, + 0x000008fc, 0x00600000, 0xffffffff, + 0x000008f8, 0x00000003, 0xffffffff, + 0x000008fc, 0x00000000, 0xffffffff, + 0x00009150, 0x96944200, 0xffffffff +}; + +#define CAYMAN_MGCG_ENABLE_LENGTH sizeof(cayman_mgcg_enable) / (3 * sizeof(u32)) + +#define NISLANDS_SYSLS_SEQUENCE 100 + +static const u32 cayman_sysls_default[] = +{ + /* Register, Value, Mask bits */ + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x0000d8bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x000020c0, 0x000c0c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x00002f50, 0x00000404, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x000064ec, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00008dfc, 0x00000000, 0xffffffff +}; +#define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32)) + +static const u32 cayman_sysls_disable[] = +{ + /* Register, Value, Mask bits */ + 0x0000d0c0, 0x00000000, 0xffffffff, + 0x0000d8c0, 0x00000000, 0xffffffff, + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x0000d8bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x00041401, 0xffffffff, + 0x0000264c, 0x00040400, 0xffffffff, + 0x00002648, 0x00040400, 0xffffffff, + 0x00002650, 0x00040400, 0xffffffff, + 0x000020b8, 0x00040400, 0xffffffff, + 0x000020bc, 0x00040400, 0xffffffff, + 0x000020c0, 0x00040c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680000, 0xffffffff, + 0x00002f50, 0x00000404, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x000064ec, 0x00007ffd, 0xffffffff, + 0x00000c7c, 0x0000ff00, 0xffffffff, + 0x00008dfc, 0x0000007f, 0xffffffff +}; +#define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32)) + +static const u32 cayman_sysls_enable[] = +{ + /* Register, Value, Mask bits */ + 0x000055e8, 0x00000001, 0xffffffff, + 0x0000d0bc, 0x00000100, 0xffffffff, + 0x0000d8bc, 0x00000100, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x000020c0, 0x000c0c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x00002f50, 0x00000903, 0xffffffff, + 0x000004c8, 0x00000000, 0xffffffff, + 0x000064ec, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00008dfc, 0x00000000, 0xffffffff +}; +#define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32)) + +struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev); +struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev); + +extern int ni_mc_load_microcode(struct radeon_device *rdev); + +struct ni_power_info *ni_get_pi(struct radeon_device *rdev) +{ + struct ni_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +struct ni_ps *ni_get_ps(struct radeon_ps *rps) +{ + struct ni_ps *ps = rps->ps_priv; + + return ps; +} + +static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff, + u16 v, s32 t, + u32 ileakage, + u32 *leakage) +{ + s64 kt, kv, leakage_w, i_leakage, vddc, temperature; + + i_leakage = div64_s64(drm_int2fixp(ileakage), 1000); + vddc = div64_s64(drm_int2fixp(v), 1000); + temperature = div64_s64(drm_int2fixp(t), 1000); + + kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000), + drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature))); + kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000), + drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc))); + + leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc); + + *leakage = drm_fixp2int(leakage_w * 1000); +} + +static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev, + const struct ni_leakage_coeffients *coeff, + u16 v, + s32 t, + u32 i_leakage, + u32 *leakage) +{ + ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage); +} + +bool ni_dpm_vblank_too_short(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 vblank_time = r600_dpm_get_vblank_time(rdev); + /* we never hit the non-gddr5 limit so disable it */ + u32 switch_limit = pi->mem_gddr5 ? 450 : 0; + + if (vblank_time < switch_limit) + return true; + else + return false; + +} + +static void ni_apply_state_adjust_rules(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct ni_ps *ps = ni_get_ps(rps); + struct radeon_clock_and_voltage_limits *max_limits; + bool disable_mclk_switching; + u32 mclk; + u16 vddci; + int i; + + if ((rdev->pm.dpm.new_active_crtc_count > 1) || + ni_dpm_vblank_too_short(rdev)) + disable_mclk_switching = true; + else + disable_mclk_switching = false; + + if (rdev->pm.dpm.ac_power) + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + else + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc; + + if (rdev->pm.dpm.ac_power == false) { + for (i = 0; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].mclk > max_limits->mclk) + ps->performance_levels[i].mclk = max_limits->mclk; + if (ps->performance_levels[i].sclk > max_limits->sclk) + ps->performance_levels[i].sclk = max_limits->sclk; + if (ps->performance_levels[i].vddc > max_limits->vddc) + ps->performance_levels[i].vddc = max_limits->vddc; + if (ps->performance_levels[i].vddci > max_limits->vddci) + ps->performance_levels[i].vddci = max_limits->vddci; + } + } + + /* XXX validate the min clocks required for display */ + + /* adjust low state */ + if (disable_mclk_switching) { + ps->performance_levels[0].mclk = + ps->performance_levels[ps->performance_level_count - 1].mclk; + ps->performance_levels[0].vddci = + ps->performance_levels[ps->performance_level_count - 1].vddci; + } + + btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk, + &ps->performance_levels[0].sclk, + &ps->performance_levels[0].mclk); + + for (i = 1; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk) + ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk; + if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc) + ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc; + } + + /* adjust remaining states */ + if (disable_mclk_switching) { + mclk = ps->performance_levels[0].mclk; + vddci = ps->performance_levels[0].vddci; + for (i = 1; i < ps->performance_level_count; i++) { + if (mclk < ps->performance_levels[i].mclk) + mclk = ps->performance_levels[i].mclk; + if (vddci < ps->performance_levels[i].vddci) + vddci = ps->performance_levels[i].vddci; + } + for (i = 0; i < ps->performance_level_count; i++) { + ps->performance_levels[i].mclk = mclk; + ps->performance_levels[i].vddci = vddci; + } + } else { + for (i = 1; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk) + ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk; + if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci) + ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci; + } + } + + for (i = 1; i < ps->performance_level_count; i++) + btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk, + &ps->performance_levels[i].sclk, + &ps->performance_levels[i].mclk); + + for (i = 0; i < ps->performance_level_count; i++) + btc_adjust_clock_combinations(rdev, max_limits, + &ps->performance_levels[i]); + + for (i = 0; i < ps->performance_level_count; i++) { + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk, + ps->performance_levels[i].sclk, + max_limits->vddc, &ps->performance_levels[i].vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + ps->performance_levels[i].mclk, + max_limits->vddci, &ps->performance_levels[i].vddci); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + ps->performance_levels[i].mclk, + max_limits->vddc, &ps->performance_levels[i].vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk, + rdev->clock.current_dispclk, + max_limits->vddc, &ps->performance_levels[i].vddc); + } + + for (i = 0; i < ps->performance_level_count; i++) { + btc_apply_voltage_delta_rules(rdev, + max_limits->vddc, max_limits->vddci, + &ps->performance_levels[i].vddc, + &ps->performance_levels[i].vddci); + } + + ps->dc_compatible = true; + for (i = 0; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc) + ps->dc_compatible = false; + + if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2) + ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2; + } +} + +static void ni_cg_clockgating_default(struct radeon_device *rdev) +{ + u32 count; + const u32 *ps = NULL; + + ps = (const u32 *)&cayman_cgcg_cgls_default; + count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH; + + btc_program_mgcg_hw_sequence(rdev, ps, count); +} + +static void ni_gfx_clockgating_enable(struct radeon_device *rdev, + bool enable) +{ + u32 count; + const u32 *ps = NULL; + + if (enable) { + ps = (const u32 *)&cayman_cgcg_cgls_enable; + count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH; + } else { + ps = (const u32 *)&cayman_cgcg_cgls_disable; + count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH; + } + + btc_program_mgcg_hw_sequence(rdev, ps, count); +} + +static void ni_mg_clockgating_default(struct radeon_device *rdev) +{ + u32 count; + const u32 *ps = NULL; + + ps = (const u32 *)&cayman_mgcg_default; + count = CAYMAN_MGCG_DEFAULT_LENGTH; + + btc_program_mgcg_hw_sequence(rdev, ps, count); +} + +static void ni_mg_clockgating_enable(struct radeon_device *rdev, + bool enable) +{ + u32 count; + const u32 *ps = NULL; + + if (enable) { + ps = (const u32 *)&cayman_mgcg_enable; + count = CAYMAN_MGCG_ENABLE_LENGTH; + } else { + ps = (const u32 *)&cayman_mgcg_disable; + count = CAYMAN_MGCG_DISABLE_LENGTH; + } + + btc_program_mgcg_hw_sequence(rdev, ps, count); +} + +static void ni_ls_clockgating_default(struct radeon_device *rdev) +{ + u32 count; + const u32 *ps = NULL; + + ps = (const u32 *)&cayman_sysls_default; + count = CAYMAN_SYSLS_DEFAULT_LENGTH; + + btc_program_mgcg_hw_sequence(rdev, ps, count); +} + +static void ni_ls_clockgating_enable(struct radeon_device *rdev, + bool enable) +{ + u32 count; + const u32 *ps = NULL; + + if (enable) { + ps = (const u32 *)&cayman_sysls_enable; + count = CAYMAN_SYSLS_ENABLE_LENGTH; + } else { + ps = (const u32 *)&cayman_sysls_disable; + count = CAYMAN_SYSLS_DISABLE_LENGTH; + } + + btc_program_mgcg_hw_sequence(rdev, ps, count); + +} + +static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev, + struct radeon_clock_voltage_dependency_table *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 i; + + if (table) { + for (i = 0; i < table->count; i++) { + if (0xff01 == table->entries[i].v) { + if (pi->max_vddc == 0) + return -EINVAL; + table->entries[i].v = pi->max_vddc; + } + } + } + return 0; +} + +static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev) +{ + int ret = 0; + + ret = ni_patch_single_dependency_table_based_on_leakage(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk); + + ret = ni_patch_single_dependency_table_based_on_leakage(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk); + return ret; +} + +static void ni_stop_dpm(struct radeon_device *rdev) +{ + WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN); +} + +#if 0 +static int ni_notify_hw_of_power_source(struct radeon_device *rdev, + bool ac_power) +{ + if (ac_power) + return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ? + 0 : -EINVAL; + + return 0; +} +#endif + +static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev, + PPSMC_Msg msg, u32 parameter) +{ + WREG32(SMC_SCRATCH0, parameter); + return rv770_send_msg_to_smc(rdev, msg); +} + +static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev) +{ + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK) + return -EINVAL; + + return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +int ni_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level) +{ + if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK) + return -EINVAL; + + if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK) + return -EINVAL; + } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { + if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK) + return -EINVAL; + + if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK) + return -EINVAL; + } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) { + if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK) + return -EINVAL; + + if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK) + return -EINVAL; + } + + rdev->pm.dpm.forced_level = level; + + return 0; +} + +static void ni_stop_smc(struct radeon_device *rdev) +{ + u32 tmp; + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK; + if (tmp != 1) + break; + udelay(1); + } + + udelay(100); + + r7xx_stop_smc(rdev); +} + +static int ni_process_firmware_header(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 tmp; + int ret; + + ret = rv770_read_smc_sram_dword(rdev, + NISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + NISLANDS_SMC_FIRMWARE_HEADER_stateTable, + &tmp, pi->sram_end); + + if (ret) + return ret; + + pi->state_table_start = (u16)tmp; + + ret = rv770_read_smc_sram_dword(rdev, + NISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + NISLANDS_SMC_FIRMWARE_HEADER_softRegisters, + &tmp, pi->sram_end); + + if (ret) + return ret; + + pi->soft_regs_start = (u16)tmp; + + ret = rv770_read_smc_sram_dword(rdev, + NISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable, + &tmp, pi->sram_end); + + if (ret) + return ret; + + eg_pi->mc_reg_table_start = (u16)tmp; + + ret = rv770_read_smc_sram_dword(rdev, + NISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + NISLANDS_SMC_FIRMWARE_HEADER_fanTable, + &tmp, pi->sram_end); + + if (ret) + return ret; + + ni_pi->fan_table_start = (u16)tmp; + + ret = rv770_read_smc_sram_dword(rdev, + NISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable, + &tmp, pi->sram_end); + + if (ret) + return ret; + + ni_pi->arb_table_start = (u16)tmp; + + ret = rv770_read_smc_sram_dword(rdev, + NISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + NISLANDS_SMC_FIRMWARE_HEADER_cacTable, + &tmp, pi->sram_end); + + if (ret) + return ret; + + ni_pi->cac_table_start = (u16)tmp; + + ret = rv770_read_smc_sram_dword(rdev, + NISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + NISLANDS_SMC_FIRMWARE_HEADER_spllTable, + &tmp, pi->sram_end); + + if (ret) + return ret; + + ni_pi->spll_table_start = (u16)tmp; + + + return ret; +} + +static void ni_read_clock_registers(struct radeon_device *rdev) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + + ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL); + ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2); + ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3); + ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4); + ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM); + ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2); + ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL); + ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2); + ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL); + ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2); + ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL); + ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL); + ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1); + ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2); +} + +#if 0 +static int ni_enter_ulp_state(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (pi->gfx_clock_gating) { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN); + WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON); + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON); + RREG32(GB_ADDR_CONFIG); + } + + WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower), + ~HOST_SMC_MSG_MASK); + + udelay(25000); + + return 0; +} +#endif + +static void ni_program_response_times(struct radeon_device *rdev) +{ + u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out; + u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit; + u32 reference_clock; + + rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1); + + voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time; + backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time; + + if (voltage_response_time == 0) + voltage_response_time = 1000; + + if (backbias_response_time == 0) + backbias_response_time = 1000; + + acpi_delay_time = 15000; + vbi_time_out = 100000; + + reference_clock = radeon_get_xclk(rdev); + + vddc_dly = (voltage_response_time * reference_clock) / 1600; + bb_dly = (backbias_response_time * reference_clock) / 1600; + acpi_dly = (acpi_delay_time * reference_clock) / 1600; + vbi_dly = (vbi_time_out * reference_clock) / 1600; + + mclk_switch_limit = (460 * reference_clock) / 100; + + rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg, vddc_dly); + rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly); + rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi, acpi_dly); + rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly); + rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA); + rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit); +} + +static void ni_populate_smc_voltage_table(struct radeon_device *rdev, + struct atom_voltage_table *voltage_table, + NISLANDS_SMC_STATETABLE *table) +{ + unsigned int i; + + for (i = 0; i < voltage_table->count; i++) { + table->highSMIO[i] = 0; + table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low); + } +} + +static void ni_populate_smc_voltage_tables(struct radeon_device *rdev, + NISLANDS_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + unsigned char i; + + if (eg_pi->vddc_voltage_table.count) { + ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table); + table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0; + table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = + cpu_to_be32(eg_pi->vddc_voltage_table.mask_low); + + for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) { + if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) { + table->maxVDDCIndexInPPTable = i; + break; + } + } + } + + if (eg_pi->vddci_voltage_table.count) { + ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table); + + table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0; + table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = + cpu_to_be32(eg_pi->vddci_voltage_table.mask_low); + } +} + +static int ni_populate_voltage_value(struct radeon_device *rdev, + struct atom_voltage_table *table, + u16 value, + NISLANDS_SMC_VOLTAGE_VALUE *voltage) +{ + unsigned int i; + + for (i = 0; i < table->count; i++) { + if (value <= table->entries[i].value) { + voltage->index = (u8)i; + voltage->value = cpu_to_be16(table->entries[i].value); + break; + } + } + + if (i >= table->count) + return -EINVAL; + + return 0; +} + +static void ni_populate_mvdd_value(struct radeon_device *rdev, + u32 mclk, + NISLANDS_SMC_VOLTAGE_VALUE *voltage) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (!pi->mvdd_control) { + voltage->index = eg_pi->mvdd_high_index; + voltage->value = cpu_to_be16(MVDD_HIGH_VALUE); + return; + } + + if (mclk <= pi->mvdd_split_frequency) { + voltage->index = eg_pi->mvdd_low_index; + voltage->value = cpu_to_be16(MVDD_LOW_VALUE); + } else { + voltage->index = eg_pi->mvdd_high_index; + voltage->value = cpu_to_be16(MVDD_HIGH_VALUE); + } +} + +static int ni_get_std_voltage_value(struct radeon_device *rdev, + NISLANDS_SMC_VOLTAGE_VALUE *voltage, + u16 *std_voltage) +{ + if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries && + ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count)) + *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc; + else + *std_voltage = be16_to_cpu(voltage->value); + + return 0; +} + +static void ni_populate_std_voltage_value(struct radeon_device *rdev, + u16 value, u8 index, + NISLANDS_SMC_VOLTAGE_VALUE *voltage) +{ + voltage->index = index; + voltage->value = cpu_to_be16(value); +} + +static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev) +{ + u32 xclk_period; + u32 xclk = radeon_get_xclk(rdev); + u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK; + + xclk_period = (1000000000UL / xclk); + xclk_period /= 10000UL; + + return tmp * xclk_period; +} + +static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor) +{ + return (power_in_watts * scaling_factor) << 2; +} + +static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + u32 near_tdp_limit) +{ + struct ni_ps *state = ni_get_ps(radeon_state); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 power_boost_limit = 0; + int ret; + + if (ni_pi->enable_power_containment && + ni_pi->use_power_boost_limit) { + NISLANDS_SMC_VOLTAGE_VALUE vddc; + u16 std_vddc_med; + u16 std_vddc_high; + u64 tmp, n, d; + + if (state->performance_level_count < 3) + return 0; + + ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table, + state->performance_levels[state->performance_level_count - 2].vddc, + &vddc); + if (ret) + return 0; + + ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med); + if (ret) + return 0; + + ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table, + state->performance_levels[state->performance_level_count - 1].vddc, + &vddc); + if (ret) + return 0; + + ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high); + if (ret) + return 0; + + n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90); + d = ((u64)std_vddc_high * (u64)std_vddc_high * 100); + tmp = div64_u64(n, d); + + if (tmp >> 32) + return 0; + power_boost_limit = (u32)tmp; + } + + return power_boost_limit; +} + +static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev, + bool adjust_polarity, + u32 tdp_adjustment, + u32 *tdp_limit, + u32 *near_tdp_limit) +{ + if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit) + return -EINVAL; + + if (adjust_polarity) { + *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100; + *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit); + } else { + *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100; + *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit); + } + + return 0; +} + +static int ni_populate_smc_tdp_limits(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + + if (ni_pi->enable_power_containment) { + NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable; + u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev); + u32 tdp_limit; + u32 near_tdp_limit; + u32 power_boost_limit; + int ret; + + if (scaling_factor == 0) + return -EINVAL; + + memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE)); + + ret = ni_calculate_adjusted_tdp_limits(rdev, + false, /* ??? */ + rdev->pm.dpm.tdp_adjustment, + &tdp_limit, + &near_tdp_limit); + if (ret) + return ret; + + power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, + near_tdp_limit); + + smc_table->dpm2Params.TDPLimit = + cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor)); + smc_table->dpm2Params.NearTDPLimit = + cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor)); + smc_table->dpm2Params.SafePowerLimit = + cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100, + scaling_factor)); + smc_table->dpm2Params.PowerBoostLimit = + cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor)); + + ret = rv770_copy_bytes_to_smc(rdev, + (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) + + offsetof(PP_NIslands_DPM2Parameters, TDPLimit)), + (u8 *)(&smc_table->dpm2Params.TDPLimit), + sizeof(u32) * 4, pi->sram_end); + if (ret) + return ret; + } + + return 0; +} + +int ni_copy_and_switch_arb_sets(struct radeon_device *rdev, + u32 arb_freq_src, u32 arb_freq_dest) +{ + u32 mc_arb_dram_timing; + u32 mc_arb_dram_timing2; + u32 burst_time; + u32 mc_cg_config; + + switch (arb_freq_src) { + case MC_CG_ARB_FREQ_F0: + mc_arb_dram_timing = RREG32(MC_ARB_DRAM_TIMING); + mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2); + burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT; + break; + case MC_CG_ARB_FREQ_F1: + mc_arb_dram_timing = RREG32(MC_ARB_DRAM_TIMING_1); + mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1); + burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT; + break; + case MC_CG_ARB_FREQ_F2: + mc_arb_dram_timing = RREG32(MC_ARB_DRAM_TIMING_2); + mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2); + burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT; + break; + case MC_CG_ARB_FREQ_F3: + mc_arb_dram_timing = RREG32(MC_ARB_DRAM_TIMING_3); + mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3); + burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT; + break; + default: + return -EINVAL; + } + + switch (arb_freq_dest) { + case MC_CG_ARB_FREQ_F0: + WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing); + WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2); + WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK); + break; + case MC_CG_ARB_FREQ_F1: + WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing); + WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2); + WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK); + break; + case MC_CG_ARB_FREQ_F2: + WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing); + WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2); + WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK); + break; + case MC_CG_ARB_FREQ_F3: + WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing); + WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2); + WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK); + break; + default: + return -EINVAL; + } + + mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F; + WREG32(MC_CG_CONFIG, mc_cg_config); + WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK); + + return 0; +} + +static int ni_init_arb_table_index(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 tmp; + int ret; + + ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start, + &tmp, pi->sram_end); + if (ret) + return ret; + + tmp &= 0x00FFFFFF; + tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24; + + return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start, + tmp, pi->sram_end); +} + +static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev) +{ + return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1); +} + +static int ni_force_switch_to_arb_f0(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 tmp; + int ret; + + ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start, + &tmp, pi->sram_end); + if (ret) + return ret; + + tmp = (tmp >> 24) & 0xff; + + if (tmp == MC_CG_ARB_FREQ_F0) + return 0; + + return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0); +} + +static int ni_populate_memory_timing_parameters(struct radeon_device *rdev, + struct rv7xx_pl *pl, + SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs) +{ + u32 dram_timing; + u32 dram_timing2; + + arb_regs->mc_arb_rfsh_rate = + (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk); + + + radeon_atom_set_engine_dram_timings(rdev, + pl->sclk, + pl->mclk); + + dram_timing = RREG32(MC_ARB_DRAM_TIMING); + dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2); + + arb_regs->mc_arb_dram_timing = cpu_to_be32(dram_timing); + arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2); + + return 0; +} + +static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + unsigned int first_arb_set) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 }; + int i, ret = 0; + + for (i = 0; i < state->performance_level_count; i++) { + ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs); + if (ret) + break; + + ret = rv770_copy_bytes_to_smc(rdev, + (u16)(ni_pi->arb_table_start + + offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) + + sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)), + (u8 *)&arb_regs, + (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet), + pi->sram_end); + if (ret) + break; + } + return ret; +} + +static int ni_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + return ni_do_program_memory_timing_parameters(rdev, radeon_new_state, + NISLANDS_DRIVER_STATE_ARB_INDEX); +} + +static void ni_populate_initial_mvdd_value(struct radeon_device *rdev, + struct NISLANDS_SMC_VOLTAGE_VALUE *voltage) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + voltage->index = eg_pi->mvdd_high_index; + voltage->value = cpu_to_be16(MVDD_HIGH_VALUE); +} + +static int ni_populate_smc_initial_state(struct radeon_device *rdev, + struct radeon_ps *radeon_initial_state, + NISLANDS_SMC_STATETABLE *table) +{ + struct ni_ps *initial_state = ni_get_ps(radeon_initial_state); + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 reg; + int ret; + + table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = + cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl); + table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = + cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2); + table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = + cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl); + table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = + cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2); + table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL = + cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl); + table->initialState.levels[0].mclk.vDLL_CNTL = + cpu_to_be32(ni_pi->clock_registers.dll_cntl); + table->initialState.levels[0].mclk.vMPLL_SS = + cpu_to_be32(ni_pi->clock_registers.mpll_ss1); + table->initialState.levels[0].mclk.vMPLL_SS2 = + cpu_to_be32(ni_pi->clock_registers.mpll_ss2); + table->initialState.levels[0].mclk.mclk_value = + cpu_to_be32(initial_state->performance_levels[0].mclk); + + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = + cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = + cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = + cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = + cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM = + cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 = + cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2); + table->initialState.levels[0].sclk.sclk_value = + cpu_to_be32(initial_state->performance_levels[0].sclk); + table->initialState.levels[0].arbRefreshState = + NISLANDS_INITIAL_STATE_ARB_INDEX; + + table->initialState.levels[0].ACIndex = 0; + + ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table, + initial_state->performance_levels[0].vddc, + &table->initialState.levels[0].vddc); + if (!ret) { + u16 std_vddc; + + ret = ni_get_std_voltage_value(rdev, + &table->initialState.levels[0].vddc, + &std_vddc); + if (!ret) + ni_populate_std_voltage_value(rdev, std_vddc, + table->initialState.levels[0].vddc.index, + &table->initialState.levels[0].std_vddc); + } + + if (eg_pi->vddci_control) + ni_populate_voltage_value(rdev, + &eg_pi->vddci_voltage_table, + initial_state->performance_levels[0].vddci, + &table->initialState.levels[0].vddci); + + ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd); + + reg = CG_R(0xffff) | CG_L(0); + table->initialState.levels[0].aT = cpu_to_be32(reg); + + table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp); + + if (pi->boot_in_gen2) + table->initialState.levels[0].gen2PCIE = 1; + else + table->initialState.levels[0].gen2PCIE = 0; + + if (pi->mem_gddr5) { + table->initialState.levels[0].strobeMode = + cypress_get_strobe_mode_settings(rdev, + initial_state->performance_levels[0].mclk); + + if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold) + table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG; + else + table->initialState.levels[0].mcFlags = 0; + } + + table->initialState.levelCount = 1; + + table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC; + + table->initialState.levels[0].dpm2.MaxPS = 0; + table->initialState.levels[0].dpm2.NearTDPDec = 0; + table->initialState.levels[0].dpm2.AboveSafeInc = 0; + table->initialState.levels[0].dpm2.BelowSafeInc = 0; + + reg = MIN_POWER_MASK | MAX_POWER_MASK; + table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg); + + reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK; + table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg); + + return 0; +} + +static int ni_populate_smc_acpi_state(struct radeon_device *rdev, + NISLANDS_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2; + u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl; + u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3; + u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4; + u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl; + u32 dll_cntl = ni_pi->clock_registers.dll_cntl; + u32 reg; + int ret; + + table->ACPIState = table->initialState; + + table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC; + + if (pi->acpi_vddc) { + ret = ni_populate_voltage_value(rdev, + &eg_pi->vddc_voltage_table, + pi->acpi_vddc, &table->ACPIState.levels[0].vddc); + if (!ret) { + u16 std_vddc; + + ret = ni_get_std_voltage_value(rdev, + &table->ACPIState.levels[0].vddc, &std_vddc); + if (!ret) + ni_populate_std_voltage_value(rdev, std_vddc, + table->ACPIState.levels[0].vddc.index, + &table->ACPIState.levels[0].std_vddc); + } + + if (pi->pcie_gen2) { + if (pi->acpi_pcie_gen2) + table->ACPIState.levels[0].gen2PCIE = 1; + else + table->ACPIState.levels[0].gen2PCIE = 0; + } else { + table->ACPIState.levels[0].gen2PCIE = 0; + } + } else { + ret = ni_populate_voltage_value(rdev, + &eg_pi->vddc_voltage_table, + pi->min_vddc_in_table, + &table->ACPIState.levels[0].vddc); + if (!ret) { + u16 std_vddc; + + ret = ni_get_std_voltage_value(rdev, + &table->ACPIState.levels[0].vddc, + &std_vddc); + if (!ret) + ni_populate_std_voltage_value(rdev, std_vddc, + table->ACPIState.levels[0].vddc.index, + &table->ACPIState.levels[0].std_vddc); + } + table->ACPIState.levels[0].gen2PCIE = 0; + } + + if (eg_pi->acpi_vddci) { + if (eg_pi->vddci_control) + ni_populate_voltage_value(rdev, + &eg_pi->vddci_voltage_table, + eg_pi->acpi_vddci, + &table->ACPIState.levels[0].vddci); + } + + + mpll_ad_func_cntl &= ~PDNB; + + mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN; + + if (pi->mem_gddr5) + mpll_dq_func_cntl &= ~PDNB; + mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS; + + + mclk_pwrmgt_cntl |= (MRDCKA0_RESET | + MRDCKA1_RESET | + MRDCKB0_RESET | + MRDCKB1_RESET | + MRDCKC0_RESET | + MRDCKC1_RESET | + MRDCKD0_RESET | + MRDCKD1_RESET); + + mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB | + MRDCKA1_PDNB | + MRDCKB0_PDNB | + MRDCKB1_PDNB | + MRDCKC0_PDNB | + MRDCKC1_PDNB | + MRDCKD0_PDNB | + MRDCKD1_PDNB); + + dll_cntl |= (MRDCKA0_BYPASS | + MRDCKA1_BYPASS | + MRDCKB0_BYPASS | + MRDCKB1_BYPASS | + MRDCKC0_BYPASS | + MRDCKC1_BYPASS | + MRDCKD0_BYPASS | + MRDCKD1_BYPASS); + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(4); + + table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl); + table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2); + table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl); + table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2); + table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl); + + table->ACPIState.levels[0].mclk.mclk_value = 0; + + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4); + + table->ACPIState.levels[0].sclk.sclk_value = 0; + + ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd); + + if (eg_pi->dynamic_ac_timing) + table->ACPIState.levels[0].ACIndex = 1; + + table->ACPIState.levels[0].dpm2.MaxPS = 0; + table->ACPIState.levels[0].dpm2.NearTDPDec = 0; + table->ACPIState.levels[0].dpm2.AboveSafeInc = 0; + table->ACPIState.levels[0].dpm2.BelowSafeInc = 0; + + reg = MIN_POWER_MASK | MAX_POWER_MASK; + table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg); + + reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK; + table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg); + + return 0; +} + +static int ni_init_smc_table(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + int ret; + struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps; + NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable; + + memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE)); + + ni_populate_smc_voltage_tables(rdev, table); + + switch (rdev->pm.int_thermal_type) { + case THERMAL_TYPE_NI: + case THERMAL_TYPE_EMC2103_WITH_INTERNAL: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL; + break; + case THERMAL_TYPE_NONE: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE; + break; + default: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL; + break; + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) + table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT) + table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) + table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC; + + if (pi->mem_gddr5) + table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5; + + ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table); + if (ret) + return ret; + + ret = ni_populate_smc_acpi_state(rdev, table); + if (ret) + return ret; + + table->driverState = table->initialState; + + table->ULVState = table->initialState; + + ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state, + NISLANDS_INITIAL_STATE_ARB_INDEX); + if (ret) + return ret; + + return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table, + sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end); +} + +static int ni_calculate_sclk_params(struct radeon_device *rdev, + u32 engine_clock, + NISLANDS_SMC_SCLK_VALUE *sclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct atom_clock_dividers dividers; + u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl; + u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3; + u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4; + u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2; + u64 tmp; + u32 reference_clock = rdev->clock.spll.reference_freq; + u32 reference_divider; + u32 fbdiv; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + engine_clock, false, ÷rs); + if (ret) + return ret; + + reference_divider = 1 + dividers.ref_div; + + + tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834; + do_div(tmp, reference_clock); + fbdiv = (u32) tmp; + + spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK); + spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div); + spll_func_cntl |= SPLL_PDIV_A(dividers.post_div); + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(2); + + spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK; + spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv); + spll_func_cntl_3 |= SPLL_DITHEN; + + if (pi->sclk_ss) { + struct radeon_atom_ss ss; + u32 vco_freq = engine_clock * dividers.post_div; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_ENGINE_SS, vco_freq)) { + u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate); + u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000); + + cg_spll_spread_spectrum &= ~CLK_S_MASK; + cg_spll_spread_spectrum |= CLK_S(clk_s); + cg_spll_spread_spectrum |= SSEN; + + cg_spll_spread_spectrum_2 &= ~CLK_V_MASK; + cg_spll_spread_spectrum_2 |= CLK_V(clk_v); + } + } + + sclk->sclk_value = engine_clock; + sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl; + sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2; + sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3; + sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4; + sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum; + sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2; + + return 0; +} + +static int ni_populate_sclk_value(struct radeon_device *rdev, + u32 engine_clock, + NISLANDS_SMC_SCLK_VALUE *sclk) +{ + NISLANDS_SMC_SCLK_VALUE sclk_tmp; + int ret; + + ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp); + if (!ret) { + sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value); + sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL); + sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2); + sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3); + sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4); + sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM); + sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2); + } + + return ret; +} + +static int ni_init_smc_spll_table(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + SMC_NISLANDS_SPLL_DIV_TABLE *spll_table; + NISLANDS_SMC_SCLK_VALUE sclk_params; + u32 fb_div; + u32 p_div; + u32 clk_s; + u32 clk_v; + u32 sclk = 0; + int i, ret; + u32 tmp; + + if (ni_pi->spll_table_start == 0) + return -EINVAL; + + spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL); + if (spll_table == NULL) + return -ENOMEM; + + for (i = 0; i < 256; i++) { + ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params); + if (ret) + break; + + p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT; + fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT; + clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT; + clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT; + + fb_div &= ~0x00001FFF; + fb_div >>= 1; + clk_v >>= 6; + + if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT)) + ret = -EINVAL; + + if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT)) + ret = -EINVAL; + + if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT)) + ret = -EINVAL; + + if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT)) + ret = -EINVAL; + + if (ret) + break; + + tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) | + ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK); + spll_table->freq[i] = cpu_to_be32(tmp); + + tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) | + ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK); + spll_table->ss[i] = cpu_to_be32(tmp); + + sclk += 512; + } + + if (!ret) + ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table, + sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end); + + kfree(spll_table); + + return ret; +} + +static int ni_populate_mclk_value(struct radeon_device *rdev, + u32 engine_clock, + u32 memory_clock, + NISLANDS_SMC_MCLK_VALUE *mclk, + bool strobe_mode, + bool dll_state_on) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2; + u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl; + u32 dll_cntl = ni_pi->clock_registers.dll_cntl; + u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1; + u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2; + struct atom_clock_dividers dividers; + u32 ibias; + u32 dll_speed; + int ret; + u32 mc_seq_misc7; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, + memory_clock, strobe_mode, ÷rs); + if (ret) + return ret; + + if (!strobe_mode) { + mc_seq_misc7 = RREG32(MC_SEQ_MISC7); + + if (mc_seq_misc7 & 0x8000000) + dividers.post_div = 1; + } + + ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div); + + mpll_ad_func_cntl &= ~(CLKR_MASK | + YCLK_POST_DIV_MASK | + CLKF_MASK | + CLKFRAC_MASK | + IBIAS_MASK); + mpll_ad_func_cntl |= CLKR(dividers.ref_div); + mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div); + mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div); + mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div); + mpll_ad_func_cntl |= IBIAS(ibias); + + if (dividers.vco_mode) + mpll_ad_func_cntl_2 |= VCO_MODE; + else + mpll_ad_func_cntl_2 &= ~VCO_MODE; + + if (pi->mem_gddr5) { + mpll_dq_func_cntl &= ~(CLKR_MASK | + YCLK_POST_DIV_MASK | + CLKF_MASK | + CLKFRAC_MASK | + IBIAS_MASK); + mpll_dq_func_cntl |= CLKR(dividers.ref_div); + mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div); + mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div); + mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div); + mpll_dq_func_cntl |= IBIAS(ibias); + + if (strobe_mode) + mpll_dq_func_cntl &= ~PDNB; + else + mpll_dq_func_cntl |= PDNB; + + if (dividers.vco_mode) + mpll_dq_func_cntl_2 |= VCO_MODE; + else + mpll_dq_func_cntl_2 &= ~VCO_MODE; + } + + if (pi->mclk_ss) { + struct radeon_atom_ss ss; + u32 vco_freq = memory_clock * dividers.post_div; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_MEMORY_SS, vco_freq)) { + u32 reference_clock = rdev->clock.mpll.reference_freq; + u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div); + u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate); + u32 clk_v = ss.percentage * + (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625); + + mpll_ss1 &= ~CLKV_MASK; + mpll_ss1 |= CLKV(clk_v); + + mpll_ss2 &= ~CLKS_MASK; + mpll_ss2 |= CLKS(clk_s); + } + } + + dll_speed = rv740_get_dll_speed(pi->mem_gddr5, + memory_clock); + + mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK; + mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed); + if (dll_state_on) + mclk_pwrmgt_cntl |= (MRDCKA0_PDNB | + MRDCKA1_PDNB | + MRDCKB0_PDNB | + MRDCKB1_PDNB | + MRDCKC0_PDNB | + MRDCKC1_PDNB | + MRDCKD0_PDNB | + MRDCKD1_PDNB); + else + mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB | + MRDCKA1_PDNB | + MRDCKB0_PDNB | + MRDCKB1_PDNB | + MRDCKC0_PDNB | + MRDCKC1_PDNB | + MRDCKD0_PDNB | + MRDCKD1_PDNB); + + + mclk->mclk_value = cpu_to_be32(memory_clock); + mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl); + mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2); + mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl); + mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2); + mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + mclk->vDLL_CNTL = cpu_to_be32(dll_cntl); + mclk->vMPLL_SS = cpu_to_be32(mpll_ss1); + mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2); + + return 0; +} + +static void ni_populate_smc_sp(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + NISLANDS_SMC_SWSTATE *smc_state) +{ + struct ni_ps *ps = ni_get_ps(radeon_state); + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + int i; + + for (i = 0; i < ps->performance_level_count - 1; i++) + smc_state->levels[i].bSP = cpu_to_be32(pi->dsp); + + smc_state->levels[ps->performance_level_count - 1].bSP = + cpu_to_be32(pi->psp); +} + +static int ni_convert_power_level_to_smc(struct radeon_device *rdev, + struct rv7xx_pl *pl, + NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + int ret; + bool dll_state_on; + u16 std_vddc; + u32 tmp = RREG32(DC_STUTTER_CNTL); + + level->gen2PCIE = pi->pcie_gen2 ? + ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0; + + ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk); + if (ret) + return ret; + + level->mcFlags = 0; + if (pi->mclk_stutter_mode_threshold && + (pl->mclk <= pi->mclk_stutter_mode_threshold) && + !eg_pi->uvd_enabled && + (tmp & DC_STUTTER_ENABLE_A) && + (tmp & DC_STUTTER_ENABLE_B)) + level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN; + + if (pi->mem_gddr5) { + if (pl->mclk > pi->mclk_edc_enable_threshold) + level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG; + if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold) + level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG; + + level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk); + + if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) { + if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >= + ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf)) + dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false; + else + dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false; + } else { + dll_state_on = false; + if (pl->mclk > ni_pi->mclk_rtt_mode_threshold) + level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE; + } + + ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, + &level->mclk, + (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0, + dll_state_on); + } else + ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1); + + if (ret) + return ret; + + ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table, + pl->vddc, &level->vddc); + if (ret) + return ret; + + ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc); + if (ret) + return ret; + + ni_populate_std_voltage_value(rdev, std_vddc, + level->vddc.index, &level->std_vddc); + + if (eg_pi->vddci_control) { + ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table, + pl->vddci, &level->vddci); + if (ret) + return ret; + } + + ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd); + + return ret; +} + +static int ni_populate_smc_t(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + NISLANDS_SMC_SWSTATE *smc_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + u32 a_t; + u32 t_l, t_h; + u32 high_bsp; + int i, ret; + + if (state->performance_level_count >= 9) + return -EINVAL; + + if (state->performance_level_count < 2) { + a_t = CG_R(0xffff) | CG_L(0); + smc_state->levels[0].aT = cpu_to_be32(a_t); + return 0; + } + + smc_state->levels[0].aT = cpu_to_be32(0); + + for (i = 0; i <= state->performance_level_count - 2; i++) { + if (eg_pi->uvd_enabled) + ret = r600_calculate_at( + 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2), + 100 * R600_AH_DFLT, + state->performance_levels[i + 1].sclk, + state->performance_levels[i].sclk, + &t_l, + &t_h); + else + ret = r600_calculate_at( + 1000 * (i + 1), + 100 * R600_AH_DFLT, + state->performance_levels[i + 1].sclk, + state->performance_levels[i].sclk, + &t_l, + &t_h); + + if (ret) { + t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT; + t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT; + } + + a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK; + a_t |= CG_R(t_l * pi->bsp / 20000); + smc_state->levels[i].aT = cpu_to_be32(a_t); + + high_bsp = (i == state->performance_level_count - 2) ? + pi->pbsp : pi->bsp; + + a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000); + smc_state->levels[i + 1].aT = cpu_to_be32(a_t); + } + + return 0; +} + +static int ni_populate_power_containment_values(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + NISLANDS_SMC_SWSTATE *smc_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + u32 prev_sclk; + u32 max_sclk; + u32 min_sclk; + int i, ret; + u32 tdp_limit; + u32 near_tdp_limit; + u32 power_boost_limit; + u8 max_ps_percent; + + if (ni_pi->enable_power_containment == false) + return 0; + + if (state->performance_level_count == 0) + return -EINVAL; + + if (smc_state->levelCount != state->performance_level_count) + return -EINVAL; + + ret = ni_calculate_adjusted_tdp_limits(rdev, + false, /* ??? */ + rdev->pm.dpm.tdp_adjustment, + &tdp_limit, + &near_tdp_limit); + if (ret) + return ret; + + power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit); + + ret = rv770_write_smc_sram_dword(rdev, + pi->state_table_start + + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) + + offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit), + ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)), + pi->sram_end); + if (ret) + power_boost_limit = 0; + + smc_state->levels[0].dpm2.MaxPS = 0; + smc_state->levels[0].dpm2.NearTDPDec = 0; + smc_state->levels[0].dpm2.AboveSafeInc = 0; + smc_state->levels[0].dpm2.BelowSafeInc = 0; + smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0; + + for (i = 1; i < state->performance_level_count; i++) { + prev_sclk = state->performance_levels[i-1].sclk; + max_sclk = state->performance_levels[i].sclk; + max_ps_percent = (i != (state->performance_level_count - 1)) ? + NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H; + + if (max_sclk < prev_sclk) + return -EINVAL; + + if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled) + min_sclk = max_sclk; + else if (1 == i) + min_sclk = prev_sclk; + else + min_sclk = (prev_sclk * (u32)max_ps_percent) / 100; + + if (min_sclk < state->performance_levels[0].sclk) + min_sclk = state->performance_levels[0].sclk; + + if (min_sclk == 0) + return -EINVAL; + + smc_state->levels[i].dpm2.MaxPS = + (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk); + smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC; + smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC; + smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC; + smc_state->levels[i].stateFlags |= + ((i != (state->performance_level_count - 1)) && power_boost_limit) ? + PPSMC_STATEFLAG_POWERBOOST : 0; + } + + return 0; +} + +static int ni_populate_sq_ramping_values(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + NISLANDS_SMC_SWSTATE *smc_state) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + u32 sq_power_throttle; + u32 sq_power_throttle2; + bool enable_sq_ramping = ni_pi->enable_sq_ramping; + int i; + + if (state->performance_level_count == 0) + return -EINVAL; + + if (smc_state->levelCount != state->performance_level_count) + return -EINVAL; + + if (rdev->pm.dpm.sq_ramping_threshold == 0) + return -EINVAL; + + if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT)) + enable_sq_ramping = false; + + if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT)) + enable_sq_ramping = false; + + if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT)) + enable_sq_ramping = false; + + if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT)) + enable_sq_ramping = false; + + if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT)) + enable_sq_ramping = false; + + for (i = 0; i < state->performance_level_count; i++) { + sq_power_throttle = 0; + sq_power_throttle2 = 0; + + if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) && + enable_sq_ramping) { + sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER); + sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER); + sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA); + sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE); + sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO); + } else { + sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK; + sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK; + } + + smc_state->levels[i].SQPowerThrottle = cpu_to_be32(sq_power_throttle); + smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2); + } + + return 0; +} + +static int ni_enable_power_containment(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + bool enable) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + PPSMC_Result smc_result; + int ret = 0; + + if (ni_pi->enable_power_containment) { + if (enable) { + if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) { + smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive); + if (smc_result != PPSMC_Result_OK) { + ret = -EINVAL; + ni_pi->pc_enabled = false; + } else { + ni_pi->pc_enabled = true; + } + } + } else { + smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive); + if (smc_result != PPSMC_Result_OK) + ret = -EINVAL; + ni_pi->pc_enabled = false; + } + } + + return ret; +} + +static int ni_convert_power_state_to_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + NISLANDS_SMC_SWSTATE *smc_state) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + int i, ret; + u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100; + + if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC)) + smc_state->flags |= PPSMC_SWSTATE_FLAG_DC; + + smc_state->levelCount = 0; + + if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE) + return -EINVAL; + + for (i = 0; i < state->performance_level_count; i++) { + ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i], + &smc_state->levels[i]); + smc_state->levels[i].arbRefreshState = + (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i); + + if (ret) + return ret; + + if (ni_pi->enable_power_containment) + smc_state->levels[i].displayWatermark = + (state->performance_levels[i].sclk < threshold) ? + PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH; + else + smc_state->levels[i].displayWatermark = (i < 2) ? + PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH; + + if (eg_pi->dynamic_ac_timing) + smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i; + else + smc_state->levels[i].ACIndex = 0; + + smc_state->levelCount++; + } + + rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold, + cpu_to_be32(threshold / 512)); + + ni_populate_smc_sp(rdev, radeon_state, smc_state); + + ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state); + if (ret) + ni_pi->enable_power_containment = false; + + ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state); + if (ret) + ni_pi->enable_sq_ramping = false; + + return ni_populate_smc_t(rdev, radeon_state, smc_state); +} + +static int ni_upload_sw_state(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u16 address = pi->state_table_start + + offsetof(NISLANDS_SMC_STATETABLE, driverState); + u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) + + ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL)); + int ret; + NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL); + + if (smc_state == NULL) + return -ENOMEM; + + ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state); + if (ret) + goto done; + + ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end); + +done: + kfree(smc_state); + + return ret; +} + +static int ni_set_mc_special_registers(struct radeon_device *rdev, + struct ni_mc_reg_table *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u8 i, j, k; + u32 temp_reg; + + for (i = 0, j = table->last; i < table->last; i++) { + switch (table->mc_reg_address[i].s1) { + case MC_SEQ_MISC1 >> 2: + if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + temp_reg = RREG32(MC_PMG_CMD_EMRS); + table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2; + for (k = 0; k < table->num_entries; k++) + table->mc_reg_table_entry[k].mc_data[j] = + ((temp_reg & 0xffff0000)) | + ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16); + j++; + if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + + temp_reg = RREG32(MC_PMG_CMD_MRS); + table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2; + for(k = 0; k < table->num_entries; k++) { + table->mc_reg_table_entry[k].mc_data[j] = + (temp_reg & 0xffff0000) | + (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); + if (!pi->mem_gddr5) + table->mc_reg_table_entry[k].mc_data[j] |= 0x100; + } + j++; + if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + break; + case MC_SEQ_RESERVE_M >> 2: + temp_reg = RREG32(MC_PMG_CMD_MRS1); + table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2; + for (k = 0; k < table->num_entries; k++) + table->mc_reg_table_entry[k].mc_data[j] = + (temp_reg & 0xffff0000) | + (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); + j++; + if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + break; + default: + break; + } + } + + table->last = j; + + return 0; +} + +static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg) +{ + bool result = true; + + switch (in_reg) { + case MC_SEQ_RAS_TIMING >> 2: + *out_reg = MC_SEQ_RAS_TIMING_LP >> 2; + break; + case MC_SEQ_CAS_TIMING >> 2: + *out_reg = MC_SEQ_CAS_TIMING_LP >> 2; + break; + case MC_SEQ_MISC_TIMING >> 2: + *out_reg = MC_SEQ_MISC_TIMING_LP >> 2; + break; + case MC_SEQ_MISC_TIMING2 >> 2: + *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2; + break; + case MC_SEQ_RD_CTL_D0 >> 2: + *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2; + break; + case MC_SEQ_RD_CTL_D1 >> 2: + *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2; + break; + case MC_SEQ_WR_CTL_D0 >> 2: + *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2; + break; + case MC_SEQ_WR_CTL_D1 >> 2: + *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2; + break; + case MC_PMG_CMD_EMRS >> 2: + *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2; + break; + case MC_PMG_CMD_MRS >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2; + break; + case MC_PMG_CMD_MRS1 >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2; + break; + case MC_SEQ_PMG_TIMING >> 2: + *out_reg = MC_SEQ_PMG_TIMING_LP >> 2; + break; + case MC_PMG_CMD_MRS2 >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2; + break; + default: + result = false; + break; + } + + return result; +} + +static void ni_set_valid_flag(struct ni_mc_reg_table *table) +{ + u8 i, j; + + for (i = 0; i < table->last; i++) { + for (j = 1; j < table->num_entries; j++) { + if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) { + table->valid_flag |= 1 << i; + break; + } + } + } +} + +static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table) +{ + u32 i; + u16 address; + + for (i = 0; i < table->last; i++) + table->mc_reg_address[i].s0 = + ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ? + address : table->mc_reg_address[i].s1; +} + +static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table, + struct ni_mc_reg_table *ni_table) +{ + u8 i, j; + + if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + if (table->num_entries > MAX_AC_TIMING_ENTRIES) + return -EINVAL; + + for (i = 0; i < table->last; i++) + ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1; + ni_table->last = table->last; + + for (i = 0; i < table->num_entries; i++) { + ni_table->mc_reg_table_entry[i].mclk_max = + table->mc_reg_table_entry[i].mclk_max; + for (j = 0; j < table->last; j++) + ni_table->mc_reg_table_entry[i].mc_data[j] = + table->mc_reg_table_entry[i].mc_data[j]; + } + ni_table->num_entries = table->num_entries; + + return 0; +} + +static int ni_initialize_mc_reg_table(struct radeon_device *rdev) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + int ret; + struct atom_mc_reg_table *table; + struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table; + u8 module_index = rv770_get_memory_module_index(rdev); + + table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING)); + WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING)); + WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING)); + WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2)); + WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS)); + WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS)); + WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1)); + WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0)); + WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1)); + WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0)); + WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1)); + WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING)); + WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2)); + + ret = radeon_atom_init_mc_reg_table(rdev, module_index, table); + + if (ret) + goto init_mc_done; + + ret = ni_copy_vbios_mc_reg_table(table, ni_table); + + if (ret) + goto init_mc_done; + + ni_set_s0_mc_reg_index(ni_table); + + ret = ni_set_mc_special_registers(rdev, ni_table); + + if (ret) + goto init_mc_done; + + ni_set_valid_flag(ni_table); + +init_mc_done: + kfree(table); + + return ret; +} + +static void ni_populate_mc_reg_addresses(struct radeon_device *rdev, + SMC_NIslands_MCRegisters *mc_reg_table) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 i, j; + + for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) { + if (ni_pi->mc_reg_table.valid_flag & (1 << j)) { + if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE) + break; + mc_reg_table->address[i].s0 = + cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0); + mc_reg_table->address[i].s1 = + cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1); + i++; + } + } + mc_reg_table->last = (u8)i; +} + + +static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry, + SMC_NIslands_MCRegisterSet *data, + u32 num_entries, u32 valid_flag) +{ + u32 i, j; + + for (i = 0, j = 0; j < num_entries; j++) { + if (valid_flag & (1 << j)) { + data->value[i] = cpu_to_be32(entry->mc_data[j]); + i++; + } + } +} + +static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev, + struct rv7xx_pl *pl, + SMC_NIslands_MCRegisterSet *mc_reg_table_data) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 i = 0; + + for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) { + if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max) + break; + } + + if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0)) + --i; + + ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i], + mc_reg_table_data, + ni_pi->mc_reg_table.last, + ni_pi->mc_reg_table.valid_flag); +} + +static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + SMC_NIslands_MCRegisters *mc_reg_table) +{ + struct ni_ps *state = ni_get_ps(radeon_state); + int i; + + for (i = 0; i < state->performance_level_count; i++) { + ni_convert_mc_reg_table_entry_to_smc(rdev, + &state->performance_levels[i], + &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]); + } +} + +static int ni_populate_mc_reg_table(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct ni_ps *boot_state = ni_get_ps(radeon_boot_state); + SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table; + + memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters)); + + rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1); + + ni_populate_mc_reg_addresses(rdev, mc_reg_table); + + ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0], + &mc_reg_table->data[0]); + + ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0], + &mc_reg_table->data[1], + ni_pi->mc_reg_table.last, + ni_pi->mc_reg_table.valid_flag); + + ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table); + + return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start, + (u8 *)mc_reg_table, + sizeof(SMC_NIslands_MCRegisters), + pi->sram_end); +} + +static int ni_upload_mc_reg_table(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state); + SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table; + u16 address; + + memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters)); + + ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table); + + address = eg_pi->mc_reg_table_start + + (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]); + + return rv770_copy_bytes_to_smc(rdev, address, + (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT], + sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count, + pi->sram_end); +} + +static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev, + PP_NIslands_CACTABLES *cac_tables) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 leakage = 0; + unsigned int i, j, table_size; + s32 t; + u32 smc_leakage, max_leakage = 0; + u32 scaling_factor; + + table_size = eg_pi->vddc_voltage_table.count; + + if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size) + table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; + + scaling_factor = ni_get_smc_power_scaling_factor(rdev); + + for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) { + for (j = 0; j < table_size; j++) { + t = (1000 * ((i + 1) * 8)); + + if (t < ni_pi->cac_data.leakage_minimum_temperature) + t = ni_pi->cac_data.leakage_minimum_temperature; + + ni_calculate_leakage_for_v_and_t(rdev, + &ni_pi->cac_data.leakage_coefficients, + eg_pi->vddc_voltage_table.entries[j].value, + t, + ni_pi->cac_data.i_leakage, + &leakage); + + smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000; + if (smc_leakage > max_leakage) + max_leakage = smc_leakage; + + cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage); + } + } + + for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) { + for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) + cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage); + } + return 0; +} + +static int ni_init_simplified_leakage_table(struct radeon_device *rdev, + PP_NIslands_CACTABLES *cac_tables) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_cac_leakage_table *leakage_table = + &rdev->pm.dpm.dyn_state.cac_leakage_table; + u32 i, j, table_size; + u32 smc_leakage, max_leakage = 0; + u32 scaling_factor; + + if (!leakage_table) + return -EINVAL; + + table_size = leakage_table->count; + + if (eg_pi->vddc_voltage_table.count != table_size) + table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ? + eg_pi->vddc_voltage_table.count : leakage_table->count; + + if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size) + table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; + + if (table_size == 0) + return -EINVAL; + + scaling_factor = ni_get_smc_power_scaling_factor(rdev); + + for (j = 0; j < table_size; j++) { + smc_leakage = leakage_table->entries[j].leakage; + + if (smc_leakage > max_leakage) + max_leakage = smc_leakage; + + for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) + cac_tables->cac_lkge_lut[i][j] = + cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor)); + } + + for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) { + for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) + cac_tables->cac_lkge_lut[i][j] = + cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor)); + } + return 0; +} + +static int ni_initialize_smc_cac_tables(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + PP_NIslands_CACTABLES *cac_tables = NULL; + int i, ret; + u32 reg; + + if (ni_pi->enable_cac == false) + return 0; + + cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL); + if (!cac_tables) + return -ENOMEM; + + reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK); + reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) | + TID_UNIT(ni_pi->cac_weights->tid_unit)); + WREG32(CG_CAC_CTRL, reg); + + for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++) + ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i]; + + for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++) + cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i]; + + ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage; + ni_pi->cac_data.pwr_const = 0; + ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0]; + ni_pi->cac_data.bif_cac_value = 0; + ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight; + ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight; + ni_pi->cac_data.allow_ovrflw = 0; + ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size; + ni_pi->cac_data.num_win_tdp = 0; + ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate; + + if (ni_pi->driver_calculate_cac_leakage) + ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables); + else + ret = ni_init_simplified_leakage_table(rdev, cac_tables); + + if (ret) + goto done_free; + + cac_tables->pwr_const = cpu_to_be32(ni_pi->cac_data.pwr_const); + cac_tables->dc_cacValue = cpu_to_be32(ni_pi->cac_data.dc_cac_value); + cac_tables->bif_cacValue = cpu_to_be32(ni_pi->cac_data.bif_cac_value); + cac_tables->AllowOvrflw = ni_pi->cac_data.allow_ovrflw; + cac_tables->MCWrWeight = ni_pi->cac_data.mc_wr_weight; + cac_tables->MCRdWeight = ni_pi->cac_data.mc_rd_weight; + cac_tables->numWin_TDP = ni_pi->cac_data.num_win_tdp; + cac_tables->l2numWin_TDP = ni_pi->cac_data.l2num_win_tdp; + cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n; + + ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables, + sizeof(PP_NIslands_CACTABLES), pi->sram_end); + +done_free: + if (ret) { + ni_pi->enable_cac = false; + ni_pi->enable_power_containment = false; + } + + kfree(cac_tables); + + return 0; +} + +static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + u32 reg; + + if (!ni_pi->enable_cac || + !ni_pi->cac_configuration_required) + return 0; + + if (ni_pi->cac_weights == NULL) + return -EINVAL; + + reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK | + WEIGHT_TCP_SIG1_MASK | + WEIGHT_TA_SIG_MASK); + reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) | + WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) | + WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig)); + WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg); + + reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK | + WEIGHT_TCC_EN1_MASK | + WEIGHT_TCC_EN2_MASK); + reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) | + WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) | + WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2)); + WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg); + + reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK | + WEIGHT_CB_EN1_MASK | + WEIGHT_CB_EN2_MASK | + WEIGHT_CB_EN3_MASK); + reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) | + WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) | + WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) | + WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3)); + WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg); + + reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK | + WEIGHT_DB_SIG1_MASK | + WEIGHT_DB_SIG2_MASK | + WEIGHT_DB_SIG3_MASK); + reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) | + WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) | + WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) | + WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3)); + WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg); + + reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK | + WEIGHT_SXM_SIG1_MASK | + WEIGHT_SXM_SIG2_MASK | + WEIGHT_SXS_SIG0_MASK | + WEIGHT_SXS_SIG1_MASK); + reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) | + WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) | + WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) | + WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) | + WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1)); + WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg); + + reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK | + WEIGHT_XBR_1_MASK | + WEIGHT_XBR_2_MASK | + WEIGHT_SPI_SIG0_MASK); + reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) | + WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) | + WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) | + WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0)); + WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg); + + reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK | + WEIGHT_SPI_SIG2_MASK | + WEIGHT_SPI_SIG3_MASK | + WEIGHT_SPI_SIG4_MASK | + WEIGHT_SPI_SIG5_MASK); + reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) | + WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) | + WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) | + WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) | + WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5)); + WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg); + + reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK | + WEIGHT_LDS_SIG1_MASK | + WEIGHT_SC_MASK); + reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) | + WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) | + WEIGHT_SC(ni_pi->cac_weights->weight_sc)); + WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg); + + reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK | + WEIGHT_CP_MASK | + WEIGHT_PA_SIG0_MASK | + WEIGHT_PA_SIG1_MASK | + WEIGHT_VGT_SIG0_MASK); + reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) | + WEIGHT_CP(ni_pi->cac_weights->weight_cp) | + WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) | + WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) | + WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0)); + WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg); + + reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK | + WEIGHT_VGT_SIG2_MASK | + WEIGHT_DC_SIG0_MASK | + WEIGHT_DC_SIG1_MASK | + WEIGHT_DC_SIG2_MASK); + reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) | + WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) | + WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) | + WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) | + WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2)); + WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg); + + reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK | + WEIGHT_UVD_SIG0_MASK | + WEIGHT_UVD_SIG1_MASK | + WEIGHT_SPARE0_MASK | + WEIGHT_SPARE1_MASK); + reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) | + WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) | + WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) | + WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) | + WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1)); + WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg); + + reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK | + WEIGHT_SQ_VSP0_MASK); + reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) | + WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0)); + WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg); + + reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK); + reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr); + WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg); + + reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK | + OVR_VAL_SPARE_0_MASK | + OVR_MODE_SPARE_1_MASK | + OVR_VAL_SPARE_1_MASK); + reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) | + OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) | + OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) | + OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1)); + WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg); + + reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK | + VSP0_MASK | + GPR_MASK); + reg |= (VSP(ni_pi->cac_weights->vsp) | + VSP0(ni_pi->cac_weights->vsp0) | + GPR(ni_pi->cac_weights->gpr)); + WREG32(SQ_CAC_THRESHOLD, reg); + + reg = (MCDW_WR_ENABLE | + MCDX_WR_ENABLE | + MCDY_WR_ENABLE | + MCDZ_WR_ENABLE | + INDEX(0x09D4)); + WREG32(MC_CG_CONFIG, reg); + + reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) | + WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) | + ALLOW_OVERFLOW); + WREG32(MC_CG_DATAPORT, reg); + + return 0; +} + +static int ni_enable_smc_cac(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + bool enable) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + int ret = 0; + PPSMC_Result smc_result; + + if (ni_pi->enable_cac) { + if (enable) { + if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) { + smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln); + + if (ni_pi->support_cac_long_term_average) { + smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable); + if (PPSMC_Result_OK != smc_result) + ni_pi->support_cac_long_term_average = false; + } + + smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac); + if (PPSMC_Result_OK != smc_result) + ret = -EINVAL; + + ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false; + } + } else if (ni_pi->cac_enabled) { + smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac); + + ni_pi->cac_enabled = false; + + if (ni_pi->support_cac_long_term_average) { + smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable); + if (PPSMC_Result_OK != smc_result) + ni_pi->support_cac_long_term_average = false; + } + } + } + + return ret; +} + +static int ni_pcie_performance_request(struct radeon_device *rdev, + u8 perf_req, bool advertise) +{ +#if defined(CONFIG_ACPI) + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) || + (perf_req == PCIE_PERF_REQ_PECI_GEN2)) { + if (eg_pi->pcie_performance_request_registered == false) + radeon_acpi_pcie_notify_device_ready(rdev); + eg_pi->pcie_performance_request_registered = true; + return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise); + } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) && + eg_pi->pcie_performance_request_registered) { + eg_pi->pcie_performance_request_registered = false; + return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise); + } +#endif + return 0; +} + +static int ni_advertise_gen2_capability(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) && + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) + pi->pcie_gen2 = true; + else + pi->pcie_gen2 = false; + + if (!pi->pcie_gen2) + ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true); + + return 0; +} + +static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 tmp, bif; + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) && + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) { + if (enable) { + if (!pi->boot_in_gen2) { + bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK; + bif |= CG_CLIENT_REQ(0xd); + WREG32(CG_BIF_REQ_AND_RSP, bif); + } + tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK; + tmp |= LC_HW_VOLTAGE_IF_CONTROL(1); + tmp |= LC_GEN2_EN_STRAP; + + tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + udelay(10); + tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + } else { + if (!pi->boot_in_gen2) { + bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK; + bif |= CG_CLIENT_REQ(0xd); + WREG32(CG_BIF_REQ_AND_RSP, bif); + + tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK; + tmp &= ~LC_GEN2_EN_STRAP; + } + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + } + } +} + +static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev, + bool enable) +{ + ni_enable_bif_dynamic_pcie_gen2(rdev, enable); + + if (enable) + WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE); + else + WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE); +} + +void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct ni_ps *new_state = ni_get_ps(new_ps); + struct ni_ps *current_state = ni_get_ps(old_ps); + + if ((new_ps->vclk == old_ps->vclk) && + (new_ps->dclk == old_ps->dclk)) + return; + + if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >= + current_state->performance_levels[current_state->performance_level_count - 1].sclk) + return; + + radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk); +} + +void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct ni_ps *new_state = ni_get_ps(new_ps); + struct ni_ps *current_state = ni_get_ps(old_ps); + + if ((new_ps->vclk == old_ps->vclk) && + (new_ps->dclk == old_ps->dclk)) + return; + + if (new_state->performance_levels[new_state->performance_level_count - 1].sclk < + current_state->performance_levels[current_state->performance_level_count - 1].sclk) + return; + + radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk); +} + +void ni_dpm_setup_asic(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + int r; + + r = ni_mc_load_microcode(rdev); + if (r) + DRM_ERROR("Failed to load MC firmware!\n"); + ni_read_clock_registers(rdev); + btc_read_arb_registers(rdev); + rv770_get_memory_type(rdev); + if (eg_pi->pcie_performance_request) + ni_advertise_gen2_capability(rdev); + rv770_get_pcie_gen2_status(rdev); + rv770_enable_acpi_pm(rdev); +} + +void ni_update_current_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct ni_ps *new_ps = ni_get_ps(rps); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + + eg_pi->current_rps = *rps; + ni_pi->current_ps = *new_ps; + eg_pi->current_rps.ps_priv = &ni_pi->current_ps; +} + +void ni_update_requested_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct ni_ps *new_ps = ni_get_ps(rps); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + + eg_pi->requested_rps = *rps; + ni_pi->requested_ps = *new_ps; + eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps; +} + +int ni_dpm_enable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + int ret; + + if (pi->gfx_clock_gating) + ni_cg_clockgating_default(rdev); + if (btc_dpm_enabled(rdev)) + return -EINVAL; + if (pi->mg_clock_gating) + ni_mg_clockgating_default(rdev); + if (eg_pi->ls_clock_gating) + ni_ls_clockgating_default(rdev); + if (pi->voltage_control) { + rv770_enable_voltage_control(rdev, true); + ret = cypress_construct_voltage_tables(rdev); + if (ret) { + DRM_ERROR("cypress_construct_voltage_tables failed\n"); + return ret; + } + } + if (eg_pi->dynamic_ac_timing) { + ret = ni_initialize_mc_reg_table(rdev); + if (ret) + eg_pi->dynamic_ac_timing = false; + } + if (pi->dynamic_ss) + cypress_enable_spread_spectrum(rdev, true); + if (pi->thermal_protection) + rv770_enable_thermal_protection(rdev, true); + rv770_setup_bsp(rdev); + rv770_program_git(rdev); + rv770_program_tp(rdev); + rv770_program_tpp(rdev); + rv770_program_sstp(rdev); + cypress_enable_display_gap(rdev); + rv770_program_vc(rdev); + if (pi->dynamic_pcie_gen2) + ni_enable_dynamic_pcie_gen2(rdev, true); + ret = rv770_upload_firmware(rdev); + if (ret) { + DRM_ERROR("rv770_upload_firmware failed\n"); + return ret; + } + ret = ni_process_firmware_header(rdev); + if (ret) { + DRM_ERROR("ni_process_firmware_header failed\n"); + return ret; + } + ret = ni_initial_switch_from_arb_f0_to_f1(rdev); + if (ret) { + DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n"); + return ret; + } + ret = ni_init_smc_table(rdev); + if (ret) { + DRM_ERROR("ni_init_smc_table failed\n"); + return ret; + } + ret = ni_init_smc_spll_table(rdev); + if (ret) { + DRM_ERROR("ni_init_smc_spll_table failed\n"); + return ret; + } + ret = ni_init_arb_table_index(rdev); + if (ret) { + DRM_ERROR("ni_init_arb_table_index failed\n"); + return ret; + } + if (eg_pi->dynamic_ac_timing) { + ret = ni_populate_mc_reg_table(rdev, boot_ps); + if (ret) { + DRM_ERROR("ni_populate_mc_reg_table failed\n"); + return ret; + } + } + ret = ni_initialize_smc_cac_tables(rdev); + if (ret) { + DRM_ERROR("ni_initialize_smc_cac_tables failed\n"); + return ret; + } + ret = ni_initialize_hardware_cac_manager(rdev); + if (ret) { + DRM_ERROR("ni_initialize_hardware_cac_manager failed\n"); + return ret; + } + ret = ni_populate_smc_tdp_limits(rdev, boot_ps); + if (ret) { + DRM_ERROR("ni_populate_smc_tdp_limits failed\n"); + return ret; + } + ni_program_response_times(rdev); + r7xx_start_smc(rdev); + ret = cypress_notify_smc_display_change(rdev, false); + if (ret) { + DRM_ERROR("cypress_notify_smc_display_change failed\n"); + return ret; + } + cypress_enable_sclk_control(rdev, true); + if (eg_pi->memory_transition) + cypress_enable_mclk_control(rdev, true); + cypress_start_dpm(rdev); + if (pi->gfx_clock_gating) + ni_gfx_clockgating_enable(rdev, true); + if (pi->mg_clock_gating) + ni_mg_clockgating_enable(rdev, true); + if (eg_pi->ls_clock_gating) + ni_ls_clockgating_enable(rdev, true); + + rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + + ni_update_current_ps(rdev, boot_ps); + + return 0; +} + +void ni_dpm_disable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + + if (!btc_dpm_enabled(rdev)) + return; + rv770_clear_vc(rdev); + if (pi->thermal_protection) + rv770_enable_thermal_protection(rdev, false); + ni_enable_power_containment(rdev, boot_ps, false); + ni_enable_smc_cac(rdev, boot_ps, false); + cypress_enable_spread_spectrum(rdev, false); + rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false); + if (pi->dynamic_pcie_gen2) + ni_enable_dynamic_pcie_gen2(rdev, false); + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + rdev->irq.dpm_thermal = false; + radeon_irq_set(rdev); + } + + if (pi->gfx_clock_gating) + ni_gfx_clockgating_enable(rdev, false); + if (pi->mg_clock_gating) + ni_mg_clockgating_enable(rdev, false); + if (eg_pi->ls_clock_gating) + ni_ls_clockgating_enable(rdev, false); + ni_stop_dpm(rdev); + btc_reset_to_default(rdev); + ni_stop_smc(rdev); + ni_force_switch_to_arb_f0(rdev); + + ni_update_current_ps(rdev, boot_ps); +} + +static int ni_power_control_set_level(struct radeon_device *rdev) +{ + struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps; + int ret; + + ret = ni_restrict_performance_levels_before_switch(rdev); + if (ret) + return ret; + ret = rv770_halt_smc(rdev); + if (ret) + return ret; + ret = ni_populate_smc_tdp_limits(rdev, new_ps); + if (ret) + return ret; + ret = rv770_resume_smc(rdev); + if (ret) + return ret; + ret = rv770_set_sw_state(rdev); + if (ret) + return ret; + + return 0; +} + +int ni_dpm_pre_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps; + struct radeon_ps *new_ps = &requested_ps; + + ni_update_requested_ps(rdev, new_ps); + + ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps); + + return 0; +} + +int ni_dpm_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *new_ps = &eg_pi->requested_rps; + struct radeon_ps *old_ps = &eg_pi->current_rps; + int ret; + + ret = ni_restrict_performance_levels_before_switch(rdev); + if (ret) { + DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n"); + return ret; + } + ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps); + ret = ni_enable_power_containment(rdev, new_ps, false); + if (ret) { + DRM_ERROR("ni_enable_power_containment failed\n"); + return ret; + } + ret = ni_enable_smc_cac(rdev, new_ps, false); + if (ret) { + DRM_ERROR("ni_enable_smc_cac failed\n"); + return ret; + } + ret = rv770_halt_smc(rdev); + if (ret) { + DRM_ERROR("rv770_halt_smc failed\n"); + return ret; + } + if (eg_pi->smu_uvd_hs) + btc_notify_uvd_to_smc(rdev, new_ps); + ret = ni_upload_sw_state(rdev, new_ps); + if (ret) { + DRM_ERROR("ni_upload_sw_state failed\n"); + return ret; + } + if (eg_pi->dynamic_ac_timing) { + ret = ni_upload_mc_reg_table(rdev, new_ps); + if (ret) { + DRM_ERROR("ni_upload_mc_reg_table failed\n"); + return ret; + } + } + ret = ni_program_memory_timing_parameters(rdev, new_ps); + if (ret) { + DRM_ERROR("ni_program_memory_timing_parameters failed\n"); + return ret; + } + ret = rv770_resume_smc(rdev); + if (ret) { + DRM_ERROR("rv770_resume_smc failed\n"); + return ret; + } + ret = rv770_set_sw_state(rdev); + if (ret) { + DRM_ERROR("rv770_set_sw_state failed\n"); + return ret; + } + ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps); + ret = ni_enable_smc_cac(rdev, new_ps, true); + if (ret) { + DRM_ERROR("ni_enable_smc_cac failed\n"); + return ret; + } + ret = ni_enable_power_containment(rdev, new_ps, true); + if (ret) { + DRM_ERROR("ni_enable_power_containment failed\n"); + return ret; + } + + /* update tdp */ + ret = ni_power_control_set_level(rdev); + if (ret) { + DRM_ERROR("ni_power_control_set_level failed\n"); + return ret; + } + + return 0; +} + +void ni_dpm_post_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *new_ps = &eg_pi->requested_rps; + + ni_update_current_ps(rdev, new_ps); +} + +#if 0 +void ni_dpm_reset_asic(struct radeon_device *rdev) +{ + ni_restrict_performance_levels_before_switch(rdev); + rv770_set_boot_state(rdev); +} +#endif + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; +}; + +union pplib_clock_info { + struct _ATOM_PPLIB_R600_CLOCK_INFO r600; + struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; + struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; + struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; +}; + +union pplib_power_state { + struct _ATOM_PPLIB_STATE v1; + struct _ATOM_PPLIB_STATE_V2 v2; +}; + +static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info, + u8 table_rev) +{ + rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); + rps->class = le16_to_cpu(non_clock_info->usClassification); + rps->class2 = le16_to_cpu(non_clock_info->usClassification2); + + if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) { + rps->vclk = le32_to_cpu(non_clock_info->ulVCLK); + rps->dclk = le32_to_cpu(non_clock_info->ulDCLK); + } else if (r600_is_uvd_state(rps->class, rps->class2)) { + rps->vclk = RV770_DEFAULT_VCLK_FREQ; + rps->dclk = RV770_DEFAULT_DCLK_FREQ; + } else { + rps->vclk = 0; + rps->dclk = 0; + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) + rdev->pm.dpm.boot_ps = rps; + if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + rdev->pm.dpm.uvd_ps = rps; +} + +static void ni_parse_pplib_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, int index, + union pplib_clock_info *clock_info) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_ps *ps = ni_get_ps(rps); + struct rv7xx_pl *pl = &ps->performance_levels[index]; + + ps->performance_level_count = index + 1; + + pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow); + pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16; + pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow); + pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16; + + pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC); + pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI); + pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags); + + /* patch up vddc if necessary */ + if (pl->vddc == 0xff01) { + if (pi->max_vddc) + pl->vddc = pi->max_vddc; + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) { + pi->acpi_vddc = pl->vddc; + eg_pi->acpi_vddci = pl->vddci; + if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) + pi->acpi_pcie_gen2 = true; + else + pi->acpi_pcie_gen2 = false; + } + + if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) { + eg_pi->ulv.supported = true; + eg_pi->ulv.pl = pl; + } + + if (pi->min_vddc_in_table > pl->vddc) + pi->min_vddc_in_table = pl->vddc; + + if (pi->max_vddc_in_table < pl->vddc) + pi->max_vddc_in_table = pl->vddc; + + /* patch up boot state */ + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { + u16 vddc, vddci, mvdd; + radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd); + pl->mclk = rdev->clock.default_mclk; + pl->sclk = rdev->clock.default_sclk; + pl->vddc = vddc; + pl->vddci = vddci; + } + + if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == + ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) { + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci; + } +} + +static int ni_parse_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; + union pplib_power_state *power_state; + int i, j; + union pplib_clock_info *clock_info; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + struct ni_ps *ps; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) * + power_info->pplib.ucNumStates, GFP_KERNEL); + if (!rdev->pm.dpm.ps) + return -ENOMEM; + + for (i = 0; i < power_info->pplib.ucNumStates; i++) { + power_state = (union pplib_power_state *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usStateArrayOffset) + + i * power_info->pplib.ucStateEntrySize); + non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) + + (power_state->v1.ucNonClockStateIndex * + power_info->pplib.ucNonClockSize)); + if (power_info->pplib.ucStateEntrySize - 1) { + u8 *idx; + ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL); + if (ps == NULL) { + kfree(rdev->pm.dpm.ps); + return -ENOMEM; + } + rdev->pm.dpm.ps[i].ps_priv = ps; + ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i], + non_clock_info, + power_info->pplib.ucNonClockSize); + idx = (u8 *)&power_state->v1.ucClockStateIndices[0]; + for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) { + clock_info = (union pplib_clock_info *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) + + (idx[j] * power_info->pplib.ucClockInfoSize)); + ni_parse_pplib_clock_info(rdev, + &rdev->pm.dpm.ps[i], j, + clock_info); + } + } + } + rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates; + return 0; +} + +int ni_dpm_init(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi; + struct evergreen_power_info *eg_pi; + struct ni_power_info *ni_pi; + struct atom_clock_dividers dividers; + int ret; + + ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL); + if (ni_pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = ni_pi; + eg_pi = &ni_pi->eg; + pi = &eg_pi->rv7xx; + + rv770_get_max_vddc(rdev); + + eg_pi->ulv.supported = false; + pi->acpi_vddc = 0; + eg_pi->acpi_vddci = 0; + pi->min_vddc_in_table = 0; + pi->max_vddc_in_table = 0; + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = ni_parse_power_table(rdev); + if (ret) + return ret; + ret = r600_parse_extended_power_table(rdev); + if (ret) + return ret; + + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries = + kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900; + + ni_patch_dependency_tables_based_on_leakage(rdev); + + if (rdev->pm.dpm.voltage_response_time == 0) + rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT; + if (rdev->pm.dpm.backbias_response_time == 0) + rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + 0, false, ÷rs); + if (ret) + pi->ref_div = dividers.ref_div + 1; + else + pi->ref_div = R600_REFERENCEDIVIDER_DFLT; + + pi->rlp = RV770_RLP_DFLT; + pi->rmp = RV770_RMP_DFLT; + pi->lhp = RV770_LHP_DFLT; + pi->lmp = RV770_LMP_DFLT; + + eg_pi->ats[0].rlp = RV770_RLP_DFLT; + eg_pi->ats[0].rmp = RV770_RMP_DFLT; + eg_pi->ats[0].lhp = RV770_LHP_DFLT; + eg_pi->ats[0].lmp = RV770_LMP_DFLT; + + eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT; + eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT; + eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT; + eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT; + + eg_pi->smu_uvd_hs = true; + + if (rdev->pdev->device == 0x6707) { + pi->mclk_strobe_mode_threshold = 55000; + pi->mclk_edc_enable_threshold = 55000; + eg_pi->mclk_edc_wr_enable_threshold = 55000; + } else { + pi->mclk_strobe_mode_threshold = 40000; + pi->mclk_edc_enable_threshold = 40000; + eg_pi->mclk_edc_wr_enable_threshold = 40000; + } + ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold; + + pi->voltage_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0); + + pi->mvdd_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0); + + eg_pi->vddci_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0); + + rv770_get_engine_memory_ss(rdev); + + pi->asi = RV770_ASI_DFLT; + pi->pasi = CYPRESS_HASI_DFLT; + pi->vrc = CYPRESS_VRC_DFLT; + + pi->power_gating = false; + + pi->gfx_clock_gating = true; + + pi->mg_clock_gating = true; + pi->mgcgtssm = true; + eg_pi->ls_clock_gating = false; + eg_pi->sclk_deep_sleep = false; + + pi->dynamic_pcie_gen2 = true; + + if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE) + pi->thermal_protection = true; + else + pi->thermal_protection = false; + + pi->display_gap = true; + + pi->dcodt = true; + + pi->ulps = true; + + eg_pi->dynamic_ac_timing = true; + eg_pi->abm = true; + eg_pi->mcls = true; + eg_pi->light_sleep = true; + eg_pi->memory_transition = true; +#if defined(CONFIG_ACPI) + eg_pi->pcie_performance_request = + radeon_acpi_is_pcie_performance_request_supported(rdev); +#else + eg_pi->pcie_performance_request = false; +#endif + + eg_pi->dll_default_on = false; + + eg_pi->sclk_deep_sleep = false; + + pi->mclk_stutter_mode_threshold = 0; + + pi->sram_end = SMC_RAM_END; + + rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3; + rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200; + rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900; + rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk); + rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk; + rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0; + rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL; + rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500; + + ni_pi->cac_data.leakage_coefficients.at = 516; + ni_pi->cac_data.leakage_coefficients.bt = 18; + ni_pi->cac_data.leakage_coefficients.av = 51; + ni_pi->cac_data.leakage_coefficients.bv = 2957; + + switch (rdev->pdev->device) { + case 0x6700: + case 0x6701: + case 0x6702: + case 0x6703: + case 0x6718: + ni_pi->cac_weights = &cac_weights_cayman_xt; + break; + case 0x6705: + case 0x6719: + case 0x671D: + case 0x671C: + default: + ni_pi->cac_weights = &cac_weights_cayman_pro; + break; + case 0x6704: + case 0x6706: + case 0x6707: + case 0x6708: + case 0x6709: + ni_pi->cac_weights = &cac_weights_cayman_le; + break; + } + + if (ni_pi->cac_weights->enable_power_containment_by_default) { + ni_pi->enable_power_containment = true; + ni_pi->enable_cac = true; + ni_pi->enable_sq_ramping = true; + } else { + ni_pi->enable_power_containment = false; + ni_pi->enable_cac = false; + ni_pi->enable_sq_ramping = false; + } + + ni_pi->driver_calculate_cac_leakage = false; + ni_pi->cac_configuration_required = true; + + if (ni_pi->cac_configuration_required) { + ni_pi->support_cac_long_term_average = true; + ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size; + ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate; + } else { + ni_pi->support_cac_long_term_average = false; + ni_pi->lta_window_size = 0; + ni_pi->lts_truncate = 0; + } + + ni_pi->use_power_boost_limit = true; + + /* make sure dc limits are valid */ + if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) || + (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0)) + rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc = + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + + return 0; +} + +void ni_dpm_fini(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); + kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries); + r600_free_extended_power_table(rdev); +} + +void ni_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct ni_ps *ps = ni_get_ps(rps); + struct rv7xx_pl *pl; + int i; + + r600_dpm_print_class_info(rps->class, rps->class2); + r600_dpm_print_cap_info(rps->caps); + printk("\tuvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + for (i = 0; i < ps->performance_level_count; i++) { + pl = &ps->performance_levels[i]; + if (rdev->family >= CHIP_TAHITI) + printk("\t\tpower level %d sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n", + i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1); + else + printk("\t\tpower level %d sclk: %u mclk: %u vddc: %u vddci: %u\n", + i, pl->sclk, pl->mclk, pl->vddc, pl->vddci); + } + r600_dpm_print_ps_status(rdev, rps); +} + +void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *rps = &eg_pi->current_rps; + struct ni_ps *ps = ni_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >> + CURRENT_STATE_INDEX_SHIFT; + + if (current_index >= ps->performance_level_count) { + seq_printf(m, "invalid dpm profile %d\n", current_index); + } else { + pl = &ps->performance_levels[current_index]; + seq_printf(m, "uvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + seq_printf(m, "power level %d sclk: %u mclk: %u vddc: %u vddci: %u\n", + current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci); + } +} + +u32 ni_dpm_get_current_sclk(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *rps = &eg_pi->current_rps; + struct ni_ps *ps = ni_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >> + CURRENT_STATE_INDEX_SHIFT; + + if (current_index >= ps->performance_level_count) { + return 0; + } else { + pl = &ps->performance_levels[current_index]; + return pl->sclk; + } +} + +u32 ni_dpm_get_current_mclk(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *rps = &eg_pi->current_rps; + struct ni_ps *ps = ni_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >> + CURRENT_STATE_INDEX_SHIFT; + + if (current_index >= ps->performance_level_count) { + return 0; + } else { + pl = &ps->performance_levels[current_index]; + return pl->mclk; + } +} + +u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps); + + if (low) + return requested_state->performance_levels[0].sclk; + else + return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk; +} + +u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps); + + if (low) + return requested_state->performance_levels[0].mclk; + else + return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk; +} + diff --git a/sys/dev/pci/drm/radeon/ni_dpm.h b/sys/dev/pci/drm/radeon/ni_dpm.h new file mode 100644 index 00000000000..6bbee918090 --- /dev/null +++ b/sys/dev/pci/drm/radeon/ni_dpm.h @@ -0,0 +1,250 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __NI_DPM_H__ +#define __NI_DPM_H__ + +#include "cypress_dpm.h" +#include "btc_dpm.h" +#include "nislands_smc.h" + +struct ni_clock_registers { + u32 cg_spll_func_cntl; + u32 cg_spll_func_cntl_2; + u32 cg_spll_func_cntl_3; + u32 cg_spll_func_cntl_4; + u32 cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2; + u32 mclk_pwrmgt_cntl; + u32 dll_cntl; + u32 mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2; + u32 mpll_ss1; + u32 mpll_ss2; +}; + +struct ni_mc_reg_entry { + u32 mclk_max; + u32 mc_data[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE]; +}; + +struct ni_mc_reg_table { + u8 last; + u8 num_entries; + u16 valid_flag; + struct ni_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES]; + SMC_NIslands_MCRegisterAddress mc_reg_address[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE]; +}; + +#define NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT 2 + +enum ni_dc_cac_level +{ + NISLANDS_DCCAC_LEVEL_0 = 0, + NISLANDS_DCCAC_LEVEL_1, + NISLANDS_DCCAC_LEVEL_2, + NISLANDS_DCCAC_LEVEL_3, + NISLANDS_DCCAC_LEVEL_4, + NISLANDS_DCCAC_LEVEL_5, + NISLANDS_DCCAC_LEVEL_6, + NISLANDS_DCCAC_LEVEL_7, + NISLANDS_DCCAC_MAX_LEVELS +}; + +struct ni_leakage_coeffients +{ + u32 at; + u32 bt; + u32 av; + u32 bv; + s32 t_slope; + s32 t_intercept; + u32 t_ref; +}; + +struct ni_cac_data +{ + struct ni_leakage_coeffients leakage_coefficients; + u32 i_leakage; + s32 leakage_minimum_temperature; + u32 pwr_const; + u32 dc_cac_value; + u32 bif_cac_value; + u32 lkge_pwr; + u8 mc_wr_weight; + u8 mc_rd_weight; + u8 allow_ovrflw; + u8 num_win_tdp; + u8 l2num_win_tdp; + u8 lts_truncate_n; +}; + +struct ni_cac_weights +{ + u32 weight_tcp_sig0; + u32 weight_tcp_sig1; + u32 weight_ta_sig; + u32 weight_tcc_en0; + u32 weight_tcc_en1; + u32 weight_tcc_en2; + u32 weight_cb_en0; + u32 weight_cb_en1; + u32 weight_cb_en2; + u32 weight_cb_en3; + u32 weight_db_sig0; + u32 weight_db_sig1; + u32 weight_db_sig2; + u32 weight_db_sig3; + u32 weight_sxm_sig0; + u32 weight_sxm_sig1; + u32 weight_sxm_sig2; + u32 weight_sxs_sig0; + u32 weight_sxs_sig1; + u32 weight_xbr_0; + u32 weight_xbr_1; + u32 weight_xbr_2; + u32 weight_spi_sig0; + u32 weight_spi_sig1; + u32 weight_spi_sig2; + u32 weight_spi_sig3; + u32 weight_spi_sig4; + u32 weight_spi_sig5; + u32 weight_lds_sig0; + u32 weight_lds_sig1; + u32 weight_sc; + u32 weight_bif; + u32 weight_cp; + u32 weight_pa_sig0; + u32 weight_pa_sig1; + u32 weight_vgt_sig0; + u32 weight_vgt_sig1; + u32 weight_vgt_sig2; + u32 weight_dc_sig0; + u32 weight_dc_sig1; + u32 weight_dc_sig2; + u32 weight_dc_sig3; + u32 weight_uvd_sig0; + u32 weight_uvd_sig1; + u32 weight_spare0; + u32 weight_spare1; + u32 weight_sq_vsp; + u32 weight_sq_vsp0; + u32 weight_sq_gpr; + u32 ovr_mode_spare_0; + u32 ovr_val_spare_0; + u32 ovr_mode_spare_1; + u32 ovr_val_spare_1; + u32 vsp; + u32 vsp0; + u32 gpr; + u8 mc_read_weight; + u8 mc_write_weight; + u32 tid_cnt; + u32 tid_unit; + u32 l2_lta_window_size; + u32 lts_truncate; + u32 dc_cac[NISLANDS_DCCAC_MAX_LEVELS]; + u32 pcie_cac[SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES]; + bool enable_power_containment_by_default; +}; + +struct ni_ps { + u16 performance_level_count; + bool dc_compatible; + struct rv7xx_pl performance_levels[NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE]; +}; + +struct ni_power_info { + /* must be first! */ + struct evergreen_power_info eg; + struct ni_clock_registers clock_registers; + struct ni_mc_reg_table mc_reg_table; + u32 mclk_rtt_mode_threshold; + /* flags */ + bool use_power_boost_limit; + bool support_cac_long_term_average; + bool cac_enabled; + bool cac_configuration_required; + bool driver_calculate_cac_leakage; + bool pc_enabled; + bool enable_power_containment; + bool enable_cac; + bool enable_sq_ramping; + /* smc offsets */ + u16 arb_table_start; + u16 fan_table_start; + u16 cac_table_start; + u16 spll_table_start; + /* CAC stuff */ + struct ni_cac_data cac_data; + u32 dc_cac_table[NISLANDS_DCCAC_MAX_LEVELS]; + const struct ni_cac_weights *cac_weights; + u8 lta_window_size; + u8 lts_truncate; + struct ni_ps current_ps; + struct ni_ps requested_ps; + /* scratch structs */ + SMC_NIslands_MCRegisters smc_mc_reg_table; + NISLANDS_SMC_STATETABLE smc_statetable; +}; + +#define NISLANDS_INITIAL_STATE_ARB_INDEX 0 +#define NISLANDS_ACPI_STATE_ARB_INDEX 1 +#define NISLANDS_ULV_STATE_ARB_INDEX 2 +#define NISLANDS_DRIVER_STATE_ARB_INDEX 3 + +#define NISLANDS_DPM2_MAX_PULSE_SKIP 256 + +#define NISLANDS_DPM2_NEAR_TDP_DEC 10 +#define NISLANDS_DPM2_ABOVE_SAFE_INC 5 +#define NISLANDS_DPM2_BELOW_SAFE_INC 20 + +#define NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT 80 + +#define NISLANDS_DPM2_MAXPS_PERCENT_H 90 +#define NISLANDS_DPM2_MAXPS_PERCENT_M 0 + +#define NISLANDS_DPM2_SQ_RAMP_MAX_POWER 0x3FFF +#define NISLANDS_DPM2_SQ_RAMP_MIN_POWER 0x12 +#define NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA 0x15 +#define NISLANDS_DPM2_SQ_RAMP_STI_SIZE 0x1E +#define NISLANDS_DPM2_SQ_RAMP_LTI_RATIO 0xF + +int ni_copy_and_switch_arb_sets(struct radeon_device *rdev, + u32 arb_freq_src, u32 arb_freq_dest); +void ni_update_current_ps(struct radeon_device *rdev, + struct radeon_ps *rps); +void ni_update_requested_ps(struct radeon_device *rdev, + struct radeon_ps *rps); + +void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps); +void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps); + +bool ni_dpm_vblank_too_short(struct radeon_device *rdev); + +#endif diff --git a/sys/dev/pci/drm/radeon/ni_reg.h b/sys/dev/pci/drm/radeon/ni_reg.h index 90c1bc7ac0f..da310a70c0f 100644 --- a/sys/dev/pci/drm/radeon/ni_reg.h +++ b/sys/dev/pci/drm/radeon/ni_reg.h @@ -1,4 +1,3 @@ -/* $OpenBSD: ni_reg.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * @@ -84,4 +83,48 @@ # define NI_REGAMMA_PROG_B 4 # define NI_OVL_REGAMMA_MODE(x) (((x) & 0x7) << 4) +#define NI_DP_MSE_LINK_TIMING 0x73a0 +# define NI_DP_MSE_LINK_FRAME (((x) & 0x3ff) << 0) +# define NI_DP_MSE_LINK_LINE (((x) & 0x3) << 16) + +#define NI_DP_MSE_MISC_CNTL 0x736c +# define NI_DP_MSE_BLANK_CODE (((x) & 0x1) << 0) +# define NI_DP_MSE_TIMESTAMP_MODE (((x) & 0x1) << 4) +# define NI_DP_MSE_ZERO_ENCODER (((x) & 0x1) << 8) + +#define NI_DP_MSE_RATE_CNTL 0x7384 +# define NI_DP_MSE_RATE_Y(x) (((x) & 0x3ffffff) << 0) +# define NI_DP_MSE_RATE_X(x) (((x) & 0x3f) << 26) + +#define NI_DP_MSE_RATE_UPDATE 0x738c + +#define NI_DP_MSE_SAT0 0x7390 +# define NI_DP_MSE_SAT_SRC0(x) (((x) & 0x7) << 0) +# define NI_DP_MSE_SAT_SLOT_COUNT0(x) (((x) & 0x3f) << 8) +# define NI_DP_MSE_SAT_SRC1(x) (((x) & 0x7) << 16) +# define NI_DP_MSE_SAT_SLOT_COUNT1(x) (((x) & 0x3f) << 24) + +#define NI_DP_MSE_SAT1 0x7394 + +#define NI_DP_MSE_SAT2 0x7398 + +#define NI_DP_MSE_SAT_UPDATE 0x739c + +#define NI_DIG_BE_CNTL 0x7140 +# define NI_DIG_FE_SOURCE_SELECT(x) (((x) & 0x7f) << 8) +# define NI_DIG_FE_DIG_MODE(x) (((x) & 0x7) << 16) +# define NI_DIG_MODE_DP_SST 0 +# define NI_DIG_MODE_LVDS 1 +# define NI_DIG_MODE_TMDS_DVI 2 +# define NI_DIG_MODE_TMDS_HDMI 3 +# define NI_DIG_MODE_DP_MST 5 +# define NI_DIG_HPD_SELECT(x) (((x) & 0x7) << 28) + +#define NI_DIG_FE_CNTL 0x7000 +# define NI_DIG_SOURCE_SELECT(x) (((x) & 0x3) << 0) +# define NI_DIG_STEREOSYNC_SELECT(x) (((x) & 0x3) << 4) +# define NI_DIG_STEREOSYNC_GATE_EN(x) (((x) & 0x1) << 8) +# define NI_DIG_DUAL_LINK_ENABLE(x) (((x) & 0x1) << 16) +# define NI_DIG_SWAP(x) (((x) & 0x1) << 18) +# define NI_DIG_SYMCLK_FE_ON (0x1 << 24) #endif diff --git a/sys/dev/pci/drm/radeon/nid.h b/sys/dev/pci/drm/radeon/nid.h index 35f3cd1a20e..47eb49b77d3 100644 --- a/sys/dev/pci/drm/radeon/nid.h +++ b/sys/dev/pci/drm/radeon/nid.h @@ -1,4 +1,3 @@ -/* $OpenBSD: nid.h,v 1.5 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * @@ -47,6 +46,13 @@ #define DMIF_ADDR_CONFIG 0xBD4 +/* fusion vce clocks */ +#define CG_ECLK_CNTL 0x620 +# define ECLK_DIVIDER_MASK 0x7f +# define ECLK_DIR_CNTL_EN (1 << 8) +#define CG_ECLK_STATUS 0x624 +# define ECLK_STATUS (1 << 0) + /* DCE6 only */ #define DMIF_ADDR_CALC 0xC00 @@ -54,6 +60,16 @@ #define RINGID(x) (((x) & 0x3) << 0) #define VMID(x) (((x) & 0x7) << 0) #define SRBM_STATUS 0x0E50 +#define RLC_RQ_PENDING (1 << 3) +#define GRBM_RQ_PENDING (1 << 5) +#define VMC_BUSY (1 << 8) +#define MCB_BUSY (1 << 9) +#define MCB_NON_DISPLAY_BUSY (1 << 10) +#define MCC_BUSY (1 << 11) +#define MCD_BUSY (1 << 12) +#define SEM_BUSY (1 << 14) +#define RLC_BUSY (1 << 15) +#define IH_BUSY (1 << 17) #define SRBM_SOFT_RESET 0x0E60 #define SOFT_RESET_BIF (1 << 1) @@ -73,6 +89,14 @@ #define SOFT_RESET_REGBB (1 << 22) #define SOFT_RESET_ORB (1 << 23) +#define SRBM_READ_ERROR 0xE98 +#define SRBM_INT_CNTL 0xEA0 +#define SRBM_INT_ACK 0xEA8 + +#define SRBM_STATUS2 0x0EC4 +#define DMA_BUSY (1 << 5) +#define DMA1_BUSY (1 << 6) + #define VM_CONTEXT0_REQUEST_RESPONSE 0x1470 #define REQUEST_TYPE(x) (((x) & 0xf) << 0) #define RESPONSE_TYPE_MASK 0x000000F0 @@ -115,11 +139,28 @@ #define READ_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 16) #define WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT (1 << 18) #define WRITE_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 19) +#define PAGE_TABLE_BLOCK_SIZE(x) (((x) & 0xF) << 24) #define VM_CONTEXT1_CNTL 0x1414 #define VM_CONTEXT0_CNTL2 0x1430 #define VM_CONTEXT1_CNTL2 0x1434 #define VM_INVALIDATE_REQUEST 0x1478 #define VM_INVALIDATE_RESPONSE 0x147c +#define VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x14FC +#define VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x14DC +#define PROTECTIONS_MASK (0xf << 0) +#define PROTECTIONS_SHIFT 0 + /* bit 0: range + * bit 2: pde0 + * bit 3: valid + * bit 4: read + * bit 5: write + */ +#define MEMORY_CLIENT_ID_MASK (0xff << 12) +#define MEMORY_CLIENT_ID_SHIFT 12 +#define MEMORY_CLIENT_RW_MASK (1 << 24) +#define MEMORY_CLIENT_RW_SHIFT 24 +#define FAULT_VMID_MASK (0x7 << 25) +#define FAULT_VMID_SHIFT 25 #define VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR 0x1518 #define VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR 0x151c #define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR 0x153C @@ -476,6 +517,312 @@ # define CACHE_FLUSH_AND_INV_EVENT_TS (0x14 << 0) # define CACHE_FLUSH_AND_INV_EVENT (0x16 << 0) +/* TN SMU registers */ +#define TN_CURRENT_GNB_TEMP 0x1F390 + +/* pm registers */ +#define SMC_MSG 0x20c +#define HOST_SMC_MSG(x) ((x) << 0) +#define HOST_SMC_MSG_MASK (0xff << 0) +#define HOST_SMC_MSG_SHIFT 0 +#define HOST_SMC_RESP(x) ((x) << 8) +#define HOST_SMC_RESP_MASK (0xff << 8) +#define HOST_SMC_RESP_SHIFT 8 +#define SMC_HOST_MSG(x) ((x) << 16) +#define SMC_HOST_MSG_MASK (0xff << 16) +#define SMC_HOST_MSG_SHIFT 16 +#define SMC_HOST_RESP(x) ((x) << 24) +#define SMC_HOST_RESP_MASK (0xff << 24) +#define SMC_HOST_RESP_SHIFT 24 + +#define CG_SPLL_FUNC_CNTL 0x600 +#define SPLL_RESET (1 << 0) +#define SPLL_SLEEP (1 << 1) +#define SPLL_BYPASS_EN (1 << 3) +#define SPLL_REF_DIV(x) ((x) << 4) +#define SPLL_REF_DIV_MASK (0x3f << 4) +#define SPLL_PDIV_A(x) ((x) << 20) +#define SPLL_PDIV_A_MASK (0x7f << 20) +#define SPLL_PDIV_A_SHIFT 20 +#define CG_SPLL_FUNC_CNTL_2 0x604 +#define SCLK_MUX_SEL(x) ((x) << 0) +#define SCLK_MUX_SEL_MASK (0x1ff << 0) +#define CG_SPLL_FUNC_CNTL_3 0x608 +#define SPLL_FB_DIV(x) ((x) << 0) +#define SPLL_FB_DIV_MASK (0x3ffffff << 0) +#define SPLL_FB_DIV_SHIFT 0 +#define SPLL_DITHEN (1 << 28) + +#define MPLL_CNTL_MODE 0x61c +# define SS_SSEN (1 << 24) +# define SS_DSMODE_EN (1 << 25) + +#define MPLL_AD_FUNC_CNTL 0x624 +#define CLKF(x) ((x) << 0) +#define CLKF_MASK (0x7f << 0) +#define CLKR(x) ((x) << 7) +#define CLKR_MASK (0x1f << 7) +#define CLKFRAC(x) ((x) << 12) +#define CLKFRAC_MASK (0x1f << 12) +#define YCLK_POST_DIV(x) ((x) << 17) +#define YCLK_POST_DIV_MASK (3 << 17) +#define IBIAS(x) ((x) << 20) +#define IBIAS_MASK (0x3ff << 20) +#define RESET (1 << 30) +#define PDNB (1 << 31) +#define MPLL_AD_FUNC_CNTL_2 0x628 +#define BYPASS (1 << 19) +#define BIAS_GEN_PDNB (1 << 24) +#define RESET_EN (1 << 25) +#define VCO_MODE (1 << 29) +#define MPLL_DQ_FUNC_CNTL 0x62c +#define MPLL_DQ_FUNC_CNTL_2 0x630 + +#define GENERAL_PWRMGT 0x63c +# define GLOBAL_PWRMGT_EN (1 << 0) +# define STATIC_PM_EN (1 << 1) +# define THERMAL_PROTECTION_DIS (1 << 2) +# define THERMAL_PROTECTION_TYPE (1 << 3) +# define ENABLE_GEN2PCIE (1 << 4) +# define ENABLE_GEN2XSP (1 << 5) +# define SW_SMIO_INDEX(x) ((x) << 6) +# define SW_SMIO_INDEX_MASK (3 << 6) +# define SW_SMIO_INDEX_SHIFT 6 +# define LOW_VOLT_D2_ACPI (1 << 8) +# define LOW_VOLT_D3_ACPI (1 << 9) +# define VOLT_PWRMGT_EN (1 << 10) +# define BACKBIAS_PAD_EN (1 << 18) +# define BACKBIAS_VALUE (1 << 19) +# define DYN_SPREAD_SPECTRUM_EN (1 << 23) +# define AC_DC_SW (1 << 24) + +#define SCLK_PWRMGT_CNTL 0x644 +# define SCLK_PWRMGT_OFF (1 << 0) +# define SCLK_LOW_D1 (1 << 1) +# define FIR_RESET (1 << 4) +# define FIR_FORCE_TREND_SEL (1 << 5) +# define FIR_TREND_MODE (1 << 6) +# define DYN_GFX_CLK_OFF_EN (1 << 7) +# define GFX_CLK_FORCE_ON (1 << 8) +# define GFX_CLK_REQUEST_OFF (1 << 9) +# define GFX_CLK_FORCE_OFF (1 << 10) +# define GFX_CLK_OFF_ACPI_D1 (1 << 11) +# define GFX_CLK_OFF_ACPI_D2 (1 << 12) +# define GFX_CLK_OFF_ACPI_D3 (1 << 13) +# define DYN_LIGHT_SLEEP_EN (1 << 14) +#define MCLK_PWRMGT_CNTL 0x648 +# define DLL_SPEED(x) ((x) << 0) +# define DLL_SPEED_MASK (0x1f << 0) +# define MPLL_PWRMGT_OFF (1 << 5) +# define DLL_READY (1 << 6) +# define MC_INT_CNTL (1 << 7) +# define MRDCKA0_PDNB (1 << 8) +# define MRDCKA1_PDNB (1 << 9) +# define MRDCKB0_PDNB (1 << 10) +# define MRDCKB1_PDNB (1 << 11) +# define MRDCKC0_PDNB (1 << 12) +# define MRDCKC1_PDNB (1 << 13) +# define MRDCKD0_PDNB (1 << 14) +# define MRDCKD1_PDNB (1 << 15) +# define MRDCKA0_RESET (1 << 16) +# define MRDCKA1_RESET (1 << 17) +# define MRDCKB0_RESET (1 << 18) +# define MRDCKB1_RESET (1 << 19) +# define MRDCKC0_RESET (1 << 20) +# define MRDCKC1_RESET (1 << 21) +# define MRDCKD0_RESET (1 << 22) +# define MRDCKD1_RESET (1 << 23) +# define DLL_READY_READ (1 << 24) +# define USE_DISPLAY_GAP (1 << 25) +# define USE_DISPLAY_URGENT_NORMAL (1 << 26) +# define MPLL_TURNOFF_D2 (1 << 28) +#define DLL_CNTL 0x64c +# define MRDCKA0_BYPASS (1 << 24) +# define MRDCKA1_BYPASS (1 << 25) +# define MRDCKB0_BYPASS (1 << 26) +# define MRDCKB1_BYPASS (1 << 27) +# define MRDCKC0_BYPASS (1 << 28) +# define MRDCKC1_BYPASS (1 << 29) +# define MRDCKD0_BYPASS (1 << 30) +# define MRDCKD1_BYPASS (1 << 31) + +#define TARGET_AND_CURRENT_PROFILE_INDEX 0x66c +# define CURRENT_STATE_INDEX_MASK (0xf << 4) +# define CURRENT_STATE_INDEX_SHIFT 4 + +#define CG_AT 0x6d4 +# define CG_R(x) ((x) << 0) +# define CG_R_MASK (0xffff << 0) +# define CG_L(x) ((x) << 16) +# define CG_L_MASK (0xffff << 16) + +#define CG_BIF_REQ_AND_RSP 0x7f4 +#define CG_CLIENT_REQ(x) ((x) << 0) +#define CG_CLIENT_REQ_MASK (0xff << 0) +#define CG_CLIENT_REQ_SHIFT 0 +#define CG_CLIENT_RESP(x) ((x) << 8) +#define CG_CLIENT_RESP_MASK (0xff << 8) +#define CG_CLIENT_RESP_SHIFT 8 +#define CLIENT_CG_REQ(x) ((x) << 16) +#define CLIENT_CG_REQ_MASK (0xff << 16) +#define CLIENT_CG_REQ_SHIFT 16 +#define CLIENT_CG_RESP(x) ((x) << 24) +#define CLIENT_CG_RESP_MASK (0xff << 24) +#define CLIENT_CG_RESP_SHIFT 24 + +#define CG_SPLL_SPREAD_SPECTRUM 0x790 +#define SSEN (1 << 0) +#define CLK_S(x) ((x) << 4) +#define CLK_S_MASK (0xfff << 4) +#define CLK_S_SHIFT 4 +#define CG_SPLL_SPREAD_SPECTRUM_2 0x794 +#define CLK_V(x) ((x) << 0) +#define CLK_V_MASK (0x3ffffff << 0) +#define CLK_V_SHIFT 0 + +#define SMC_SCRATCH0 0x81c + +#define CG_SPLL_FUNC_CNTL_4 0x850 + +#define MPLL_SS1 0x85c +#define CLKV(x) ((x) << 0) +#define CLKV_MASK (0x3ffffff << 0) +#define MPLL_SS2 0x860 +#define CLKS(x) ((x) << 0) +#define CLKS_MASK (0xfff << 0) + +#define CG_CAC_CTRL 0x88c +#define TID_CNT(x) ((x) << 0) +#define TID_CNT_MASK (0x3fff << 0) +#define TID_UNIT(x) ((x) << 14) +#define TID_UNIT_MASK (0xf << 14) + +#define CG_IND_ADDR 0x8f8 +#define CG_IND_DATA 0x8fc +/* CGIND regs */ +#define CG_CGTT_LOCAL_0 0x00 +#define CG_CGTT_LOCAL_1 0x01 + +#define MC_CG_CONFIG 0x25bc +#define MCDW_WR_ENABLE (1 << 0) +#define MCDX_WR_ENABLE (1 << 1) +#define MCDY_WR_ENABLE (1 << 2) +#define MCDZ_WR_ENABLE (1 << 3) +#define MC_RD_ENABLE(x) ((x) << 4) +#define MC_RD_ENABLE_MASK (3 << 4) +#define INDEX(x) ((x) << 6) +#define INDEX_MASK (0xfff << 6) +#define INDEX_SHIFT 6 + +#define MC_ARB_CAC_CNTL 0x2750 +#define ENABLE (1 << 0) +#define READ_WEIGHT(x) ((x) << 1) +#define READ_WEIGHT_MASK (0x3f << 1) +#define READ_WEIGHT_SHIFT 1 +#define WRITE_WEIGHT(x) ((x) << 7) +#define WRITE_WEIGHT_MASK (0x3f << 7) +#define WRITE_WEIGHT_SHIFT 7 +#define ALLOW_OVERFLOW (1 << 13) + +#define MC_ARB_DRAM_TIMING 0x2774 +#define MC_ARB_DRAM_TIMING2 0x2778 + +#define MC_ARB_RFSH_RATE 0x27b0 +#define POWERMODE0(x) ((x) << 0) +#define POWERMODE0_MASK (0xff << 0) +#define POWERMODE0_SHIFT 0 +#define POWERMODE1(x) ((x) << 8) +#define POWERMODE1_MASK (0xff << 8) +#define POWERMODE1_SHIFT 8 +#define POWERMODE2(x) ((x) << 16) +#define POWERMODE2_MASK (0xff << 16) +#define POWERMODE2_SHIFT 16 +#define POWERMODE3(x) ((x) << 24) +#define POWERMODE3_MASK (0xff << 24) +#define POWERMODE3_SHIFT 24 + +#define MC_ARB_CG 0x27e8 +#define CG_ARB_REQ(x) ((x) << 0) +#define CG_ARB_REQ_MASK (0xff << 0) +#define CG_ARB_REQ_SHIFT 0 +#define CG_ARB_RESP(x) ((x) << 8) +#define CG_ARB_RESP_MASK (0xff << 8) +#define CG_ARB_RESP_SHIFT 8 +#define ARB_CG_REQ(x) ((x) << 16) +#define ARB_CG_REQ_MASK (0xff << 16) +#define ARB_CG_REQ_SHIFT 16 +#define ARB_CG_RESP(x) ((x) << 24) +#define ARB_CG_RESP_MASK (0xff << 24) +#define ARB_CG_RESP_SHIFT 24 + +#define MC_ARB_DRAM_TIMING_1 0x27f0 +#define MC_ARB_DRAM_TIMING_2 0x27f4 +#define MC_ARB_DRAM_TIMING_3 0x27f8 +#define MC_ARB_DRAM_TIMING2_1 0x27fc +#define MC_ARB_DRAM_TIMING2_2 0x2800 +#define MC_ARB_DRAM_TIMING2_3 0x2804 +#define MC_ARB_BURST_TIME 0x2808 +#define STATE0(x) ((x) << 0) +#define STATE0_MASK (0x1f << 0) +#define STATE0_SHIFT 0 +#define STATE1(x) ((x) << 5) +#define STATE1_MASK (0x1f << 5) +#define STATE1_SHIFT 5 +#define STATE2(x) ((x) << 10) +#define STATE2_MASK (0x1f << 10) +#define STATE2_SHIFT 10 +#define STATE3(x) ((x) << 15) +#define STATE3_MASK (0x1f << 15) +#define STATE3_SHIFT 15 + +#define MC_CG_DATAPORT 0x2884 + +#define MC_SEQ_RAS_TIMING 0x28a0 +#define MC_SEQ_CAS_TIMING 0x28a4 +#define MC_SEQ_MISC_TIMING 0x28a8 +#define MC_SEQ_MISC_TIMING2 0x28ac +#define MC_SEQ_PMG_TIMING 0x28b0 +#define MC_SEQ_RD_CTL_D0 0x28b4 +#define MC_SEQ_RD_CTL_D1 0x28b8 +#define MC_SEQ_WR_CTL_D0 0x28bc +#define MC_SEQ_WR_CTL_D1 0x28c0 + +#define MC_SEQ_MISC0 0x2a00 +#define MC_SEQ_MISC0_GDDR5_SHIFT 28 +#define MC_SEQ_MISC0_GDDR5_MASK 0xf0000000 +#define MC_SEQ_MISC0_GDDR5_VALUE 5 +#define MC_SEQ_MISC1 0x2a04 +#define MC_SEQ_RESERVE_M 0x2a08 +#define MC_PMG_CMD_EMRS 0x2a0c + +#define MC_SEQ_MISC3 0x2a2c + +#define MC_SEQ_MISC5 0x2a54 +#define MC_SEQ_MISC6 0x2a58 + +#define MC_SEQ_MISC7 0x2a64 + +#define MC_SEQ_RAS_TIMING_LP 0x2a6c +#define MC_SEQ_CAS_TIMING_LP 0x2a70 +#define MC_SEQ_MISC_TIMING_LP 0x2a74 +#define MC_SEQ_MISC_TIMING2_LP 0x2a78 +#define MC_SEQ_WR_CTL_D0_LP 0x2a7c +#define MC_SEQ_WR_CTL_D1_LP 0x2a80 +#define MC_SEQ_PMG_CMD_EMRS_LP 0x2a84 +#define MC_SEQ_PMG_CMD_MRS_LP 0x2a88 + +#define MC_PMG_CMD_MRS 0x2aac + +#define MC_SEQ_RD_CTL_D0_LP 0x2b1c +#define MC_SEQ_RD_CTL_D1_LP 0x2b20 + +#define MC_PMG_CMD_MRS1 0x2b44 +#define MC_SEQ_PMG_CMD_MRS1_LP 0x2b48 +#define MC_SEQ_PMG_TIMING_LP 0x2b4c + +#define MC_PMG_CMD_MRS2 0x2b5c +#define MC_SEQ_PMG_CMD_MRS2_LP 0x2b60 + #define AUX_CONTROL 0x6200 #define AUX_EN (1 << 0) #define AUX_LS_READ_EN (1 << 8) @@ -522,19 +869,282 @@ #define AUX_SW_DATA_INDEX(x) (((x) & 0x1f) << 16) #define AUX_SW_AUTOINCREMENT_DISABLE (1 << 31) +#define LB_SYNC_RESET_SEL 0x6b28 +#define LB_SYNC_RESET_SEL_MASK (3 << 0) +#define LB_SYNC_RESET_SEL_SHIFT 0 + +#define DC_STUTTER_CNTL 0x6b30 +#define DC_STUTTER_ENABLE_A (1 << 0) +#define DC_STUTTER_ENABLE_B (1 << 1) + +#define SQ_CAC_THRESHOLD 0x8e4c +#define VSP(x) ((x) << 0) +#define VSP_MASK (0xff << 0) +#define VSP_SHIFT 0 +#define VSP0(x) ((x) << 8) +#define VSP0_MASK (0xff << 8) +#define VSP0_SHIFT 8 +#define GPR(x) ((x) << 16) +#define GPR_MASK (0xff << 16) +#define GPR_SHIFT 16 + +#define SQ_POWER_THROTTLE 0x8e58 +#define MIN_POWER(x) ((x) << 0) +#define MIN_POWER_MASK (0x3fff << 0) +#define MIN_POWER_SHIFT 0 +#define MAX_POWER(x) ((x) << 16) +#define MAX_POWER_MASK (0x3fff << 16) +#define MAX_POWER_SHIFT 0 +#define SQ_POWER_THROTTLE2 0x8e5c +#define MAX_POWER_DELTA(x) ((x) << 0) +#define MAX_POWER_DELTA_MASK (0x3fff << 0) +#define MAX_POWER_DELTA_SHIFT 0 +#define STI_SIZE(x) ((x) << 16) +#define STI_SIZE_MASK (0x3ff << 16) +#define STI_SIZE_SHIFT 16 +#define LTI_RATIO(x) ((x) << 27) +#define LTI_RATIO_MASK (0xf << 27) +#define LTI_RATIO_SHIFT 27 + +/* CG indirect registers */ +#define CG_CAC_REGION_1_WEIGHT_0 0x83 +#define WEIGHT_TCP_SIG0(x) ((x) << 0) +#define WEIGHT_TCP_SIG0_MASK (0x3f << 0) +#define WEIGHT_TCP_SIG0_SHIFT 0 +#define WEIGHT_TCP_SIG1(x) ((x) << 6) +#define WEIGHT_TCP_SIG1_MASK (0x3f << 6) +#define WEIGHT_TCP_SIG1_SHIFT 6 +#define WEIGHT_TA_SIG(x) ((x) << 12) +#define WEIGHT_TA_SIG_MASK (0x3f << 12) +#define WEIGHT_TA_SIG_SHIFT 12 +#define CG_CAC_REGION_1_WEIGHT_1 0x84 +#define WEIGHT_TCC_EN0(x) ((x) << 0) +#define WEIGHT_TCC_EN0_MASK (0x3f << 0) +#define WEIGHT_TCC_EN0_SHIFT 0 +#define WEIGHT_TCC_EN1(x) ((x) << 6) +#define WEIGHT_TCC_EN1_MASK (0x3f << 6) +#define WEIGHT_TCC_EN1_SHIFT 6 +#define WEIGHT_TCC_EN2(x) ((x) << 12) +#define WEIGHT_TCC_EN2_MASK (0x3f << 12) +#define WEIGHT_TCC_EN2_SHIFT 12 +#define WEIGHT_TCC_EN3(x) ((x) << 18) +#define WEIGHT_TCC_EN3_MASK (0x3f << 18) +#define WEIGHT_TCC_EN3_SHIFT 18 +#define CG_CAC_REGION_2_WEIGHT_0 0x85 +#define WEIGHT_CB_EN0(x) ((x) << 0) +#define WEIGHT_CB_EN0_MASK (0x3f << 0) +#define WEIGHT_CB_EN0_SHIFT 0 +#define WEIGHT_CB_EN1(x) ((x) << 6) +#define WEIGHT_CB_EN1_MASK (0x3f << 6) +#define WEIGHT_CB_EN1_SHIFT 6 +#define WEIGHT_CB_EN2(x) ((x) << 12) +#define WEIGHT_CB_EN2_MASK (0x3f << 12) +#define WEIGHT_CB_EN2_SHIFT 12 +#define WEIGHT_CB_EN3(x) ((x) << 18) +#define WEIGHT_CB_EN3_MASK (0x3f << 18) +#define WEIGHT_CB_EN3_SHIFT 18 +#define CG_CAC_REGION_2_WEIGHT_1 0x86 +#define WEIGHT_DB_SIG0(x) ((x) << 0) +#define WEIGHT_DB_SIG0_MASK (0x3f << 0) +#define WEIGHT_DB_SIG0_SHIFT 0 +#define WEIGHT_DB_SIG1(x) ((x) << 6) +#define WEIGHT_DB_SIG1_MASK (0x3f << 6) +#define WEIGHT_DB_SIG1_SHIFT 6 +#define WEIGHT_DB_SIG2(x) ((x) << 12) +#define WEIGHT_DB_SIG2_MASK (0x3f << 12) +#define WEIGHT_DB_SIG2_SHIFT 12 +#define WEIGHT_DB_SIG3(x) ((x) << 18) +#define WEIGHT_DB_SIG3_MASK (0x3f << 18) +#define WEIGHT_DB_SIG3_SHIFT 18 +#define CG_CAC_REGION_2_WEIGHT_2 0x87 +#define WEIGHT_SXM_SIG0(x) ((x) << 0) +#define WEIGHT_SXM_SIG0_MASK (0x3f << 0) +#define WEIGHT_SXM_SIG0_SHIFT 0 +#define WEIGHT_SXM_SIG1(x) ((x) << 6) +#define WEIGHT_SXM_SIG1_MASK (0x3f << 6) +#define WEIGHT_SXM_SIG1_SHIFT 6 +#define WEIGHT_SXM_SIG2(x) ((x) << 12) +#define WEIGHT_SXM_SIG2_MASK (0x3f << 12) +#define WEIGHT_SXM_SIG2_SHIFT 12 +#define WEIGHT_SXS_SIG0(x) ((x) << 18) +#define WEIGHT_SXS_SIG0_MASK (0x3f << 18) +#define WEIGHT_SXS_SIG0_SHIFT 18 +#define WEIGHT_SXS_SIG1(x) ((x) << 24) +#define WEIGHT_SXS_SIG1_MASK (0x3f << 24) +#define WEIGHT_SXS_SIG1_SHIFT 24 +#define CG_CAC_REGION_3_WEIGHT_0 0x88 +#define WEIGHT_XBR_0(x) ((x) << 0) +#define WEIGHT_XBR_0_MASK (0x3f << 0) +#define WEIGHT_XBR_0_SHIFT 0 +#define WEIGHT_XBR_1(x) ((x) << 6) +#define WEIGHT_XBR_1_MASK (0x3f << 6) +#define WEIGHT_XBR_1_SHIFT 6 +#define WEIGHT_XBR_2(x) ((x) << 12) +#define WEIGHT_XBR_2_MASK (0x3f << 12) +#define WEIGHT_XBR_2_SHIFT 12 +#define WEIGHT_SPI_SIG0(x) ((x) << 18) +#define WEIGHT_SPI_SIG0_MASK (0x3f << 18) +#define WEIGHT_SPI_SIG0_SHIFT 18 +#define CG_CAC_REGION_3_WEIGHT_1 0x89 +#define WEIGHT_SPI_SIG1(x) ((x) << 0) +#define WEIGHT_SPI_SIG1_MASK (0x3f << 0) +#define WEIGHT_SPI_SIG1_SHIFT 0 +#define WEIGHT_SPI_SIG2(x) ((x) << 6) +#define WEIGHT_SPI_SIG2_MASK (0x3f << 6) +#define WEIGHT_SPI_SIG2_SHIFT 6 +#define WEIGHT_SPI_SIG3(x) ((x) << 12) +#define WEIGHT_SPI_SIG3_MASK (0x3f << 12) +#define WEIGHT_SPI_SIG3_SHIFT 12 +#define WEIGHT_SPI_SIG4(x) ((x) << 18) +#define WEIGHT_SPI_SIG4_MASK (0x3f << 18) +#define WEIGHT_SPI_SIG4_SHIFT 18 +#define WEIGHT_SPI_SIG5(x) ((x) << 24) +#define WEIGHT_SPI_SIG5_MASK (0x3f << 24) +#define WEIGHT_SPI_SIG5_SHIFT 24 +#define CG_CAC_REGION_4_WEIGHT_0 0x8a +#define WEIGHT_LDS_SIG0(x) ((x) << 0) +#define WEIGHT_LDS_SIG0_MASK (0x3f << 0) +#define WEIGHT_LDS_SIG0_SHIFT 0 +#define WEIGHT_LDS_SIG1(x) ((x) << 6) +#define WEIGHT_LDS_SIG1_MASK (0x3f << 6) +#define WEIGHT_LDS_SIG1_SHIFT 6 +#define WEIGHT_SC(x) ((x) << 24) +#define WEIGHT_SC_MASK (0x3f << 24) +#define WEIGHT_SC_SHIFT 24 +#define CG_CAC_REGION_4_WEIGHT_1 0x8b +#define WEIGHT_BIF(x) ((x) << 0) +#define WEIGHT_BIF_MASK (0x3f << 0) +#define WEIGHT_BIF_SHIFT 0 +#define WEIGHT_CP(x) ((x) << 6) +#define WEIGHT_CP_MASK (0x3f << 6) +#define WEIGHT_CP_SHIFT 6 +#define WEIGHT_PA_SIG0(x) ((x) << 12) +#define WEIGHT_PA_SIG0_MASK (0x3f << 12) +#define WEIGHT_PA_SIG0_SHIFT 12 +#define WEIGHT_PA_SIG1(x) ((x) << 18) +#define WEIGHT_PA_SIG1_MASK (0x3f << 18) +#define WEIGHT_PA_SIG1_SHIFT 18 +#define WEIGHT_VGT_SIG0(x) ((x) << 24) +#define WEIGHT_VGT_SIG0_MASK (0x3f << 24) +#define WEIGHT_VGT_SIG0_SHIFT 24 +#define CG_CAC_REGION_4_WEIGHT_2 0x8c +#define WEIGHT_VGT_SIG1(x) ((x) << 0) +#define WEIGHT_VGT_SIG1_MASK (0x3f << 0) +#define WEIGHT_VGT_SIG1_SHIFT 0 +#define WEIGHT_VGT_SIG2(x) ((x) << 6) +#define WEIGHT_VGT_SIG2_MASK (0x3f << 6) +#define WEIGHT_VGT_SIG2_SHIFT 6 +#define WEIGHT_DC_SIG0(x) ((x) << 12) +#define WEIGHT_DC_SIG0_MASK (0x3f << 12) +#define WEIGHT_DC_SIG0_SHIFT 12 +#define WEIGHT_DC_SIG1(x) ((x) << 18) +#define WEIGHT_DC_SIG1_MASK (0x3f << 18) +#define WEIGHT_DC_SIG1_SHIFT 18 +#define WEIGHT_DC_SIG2(x) ((x) << 24) +#define WEIGHT_DC_SIG2_MASK (0x3f << 24) +#define WEIGHT_DC_SIG2_SHIFT 24 +#define CG_CAC_REGION_4_WEIGHT_3 0x8d +#define WEIGHT_DC_SIG3(x) ((x) << 0) +#define WEIGHT_DC_SIG3_MASK (0x3f << 0) +#define WEIGHT_DC_SIG3_SHIFT 0 +#define WEIGHT_UVD_SIG0(x) ((x) << 6) +#define WEIGHT_UVD_SIG0_MASK (0x3f << 6) +#define WEIGHT_UVD_SIG0_SHIFT 6 +#define WEIGHT_UVD_SIG1(x) ((x) << 12) +#define WEIGHT_UVD_SIG1_MASK (0x3f << 12) +#define WEIGHT_UVD_SIG1_SHIFT 12 +#define WEIGHT_SPARE0(x) ((x) << 18) +#define WEIGHT_SPARE0_MASK (0x3f << 18) +#define WEIGHT_SPARE0_SHIFT 18 +#define WEIGHT_SPARE1(x) ((x) << 24) +#define WEIGHT_SPARE1_MASK (0x3f << 24) +#define WEIGHT_SPARE1_SHIFT 24 +#define CG_CAC_REGION_5_WEIGHT_0 0x8e +#define WEIGHT_SQ_VSP(x) ((x) << 0) +#define WEIGHT_SQ_VSP_MASK (0x3fff << 0) +#define WEIGHT_SQ_VSP_SHIFT 0 +#define WEIGHT_SQ_VSP0(x) ((x) << 14) +#define WEIGHT_SQ_VSP0_MASK (0x3fff << 14) +#define WEIGHT_SQ_VSP0_SHIFT 14 +#define CG_CAC_REGION_4_OVERRIDE_4 0xab +#define OVR_MODE_SPARE_0(x) ((x) << 16) +#define OVR_MODE_SPARE_0_MASK (0x1 << 16) +#define OVR_MODE_SPARE_0_SHIFT 16 +#define OVR_VAL_SPARE_0(x) ((x) << 17) +#define OVR_VAL_SPARE_0_MASK (0x1 << 17) +#define OVR_VAL_SPARE_0_SHIFT 17 +#define OVR_MODE_SPARE_1(x) ((x) << 18) +#define OVR_MODE_SPARE_1_MASK (0x3f << 18) +#define OVR_MODE_SPARE_1_SHIFT 18 +#define OVR_VAL_SPARE_1(x) ((x) << 19) +#define OVR_VAL_SPARE_1_MASK (0x3f << 19) +#define OVR_VAL_SPARE_1_SHIFT 19 +#define CG_CAC_REGION_5_WEIGHT_1 0xb7 +#define WEIGHT_SQ_GPR(x) ((x) << 0) +#define WEIGHT_SQ_GPR_MASK (0x3fff << 0) +#define WEIGHT_SQ_GPR_SHIFT 0 +#define WEIGHT_SQ_LDS(x) ((x) << 14) +#define WEIGHT_SQ_LDS_MASK (0x3fff << 14) +#define WEIGHT_SQ_LDS_SHIFT 14 + +/* PCIE link stuff */ +#define PCIE_LC_TRAINING_CNTL 0xa1 /* PCIE_P */ +#define PCIE_LC_LINK_WIDTH_CNTL 0xa2 /* PCIE_P */ +# define LC_LINK_WIDTH_SHIFT 0 +# define LC_LINK_WIDTH_MASK 0x7 +# define LC_LINK_WIDTH_X0 0 +# define LC_LINK_WIDTH_X1 1 +# define LC_LINK_WIDTH_X2 2 +# define LC_LINK_WIDTH_X4 3 +# define LC_LINK_WIDTH_X8 4 +# define LC_LINK_WIDTH_X16 6 +# define LC_LINK_WIDTH_RD_SHIFT 4 +# define LC_LINK_WIDTH_RD_MASK 0x70 +# define LC_RECONFIG_ARC_MISSING_ESCAPE (1 << 7) +# define LC_RECONFIG_NOW (1 << 8) +# define LC_RENEGOTIATION_SUPPORT (1 << 9) +# define LC_RENEGOTIATE_EN (1 << 10) +# define LC_SHORT_RECONFIG_EN (1 << 11) +# define LC_UPCONFIGURE_SUPPORT (1 << 12) +# define LC_UPCONFIGURE_DIS (1 << 13) +#define PCIE_LC_SPEED_CNTL 0xa4 /* PCIE_P */ +# define LC_GEN2_EN_STRAP (1 << 0) +# define LC_TARGET_LINK_SPEED_OVERRIDE_EN (1 << 1) +# define LC_FORCE_EN_HW_SPEED_CHANGE (1 << 5) +# define LC_FORCE_DIS_HW_SPEED_CHANGE (1 << 6) +# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK (0x3 << 8) +# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT 3 +# define LC_CURRENT_DATA_RATE (1 << 11) +# define LC_HW_VOLTAGE_IF_CONTROL(x) ((x) << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_MASK (3 << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_SHIFT 12 +# define LC_VOLTAGE_TIMER_SEL_MASK (0xf << 14) +# define LC_CLR_FAILED_SPD_CHANGE_CNT (1 << 21) +# define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 23) +# define LC_OTHER_SIDE_SUPPORTS_GEN2 (1 << 24) +#define MM_CFGREGS_CNTL 0x544c +# define MM_WR_TO_CFG_EN (1 << 3) +#define LINK_CNTL2 0x88 /* F0 */ +# define TARGET_LINK_SPEED_MASK (0xf << 0) +# define SELECTABLE_DEEMPHASIS (1 << 6) + +/* + * UVD + */ +#define UVD_SEMA_ADDR_LOW 0xEF00 +#define UVD_SEMA_ADDR_HIGH 0xEF04 +#define UVD_SEMA_CMD 0xEF08 +#define UVD_UDEC_ADDR_CONFIG 0xEF4C +#define UVD_UDEC_DB_ADDR_CONFIG 0xEF50 +#define UVD_UDEC_DBW_ADDR_CONFIG 0xEF54 +#define UVD_RBC_RB_RPTR 0xF690 +#define UVD_RBC_RB_WPTR 0xF694 +#define UVD_STATUS 0xf6bc + /* * PM4 */ -#define PACKET_TYPE0 0 -#define PACKET_TYPE1 1 -#define PACKET_TYPE2 2 -#define PACKET_TYPE3 3 - -#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) -#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) -#define CP_PACKET0_GET_REG(h) (((h) & 0xFFFF) << 2) -#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) -#define PACKET0(reg, n) ((PACKET_TYPE0 << 30) | \ +#define PACKET0(reg, n) ((RADEON_PACKET_TYPE0 << 30) | \ (((reg) >> 2) & 0xFFFF) | \ ((n) & 0x3FFF) << 16) #define CP_PACKET2 0x80000000 @@ -543,7 +1153,7 @@ #define PACKET2(v) (CP_PACKET2 | REG_SET(PACKET2_PAD, (v))) -#define PACKET3(op, n) ((PACKET_TYPE3 << 30) | \ +#define PACKET3(op, n) ((RADEON_PACKET_TYPE3 << 30) | \ (((op) & 0xFF) << 8) | \ ((n) & 0x3FFF) << 16) @@ -581,6 +1191,23 @@ #define PACKET3_MEM_SEMAPHORE 0x39 #define PACKET3_MPEG_INDEX 0x3A #define PACKET3_WAIT_REG_MEM 0x3C +#define WAIT_REG_MEM_FUNCTION(x) ((x) << 0) + /* 0 - always + * 1 - < + * 2 - <= + * 3 - == + * 4 - != + * 5 - >= + * 6 - > + */ +#define WAIT_REG_MEM_MEM_SPACE(x) ((x) << 4) + /* 0 - reg + * 1 - mem + */ +#define WAIT_REG_MEM_ENGINE(x) ((x) << 8) + /* 0 - me + * 1 - pfp + */ #define PACKET3_MEM_WRITE 0x3D #define PACKET3_PFP_SYNC_ME 0x42 #define PACKET3_SURFACE_SYNC 0x43 @@ -715,6 +1342,18 @@ (((vmid) & 0xF) << 20) | \ (((n) & 0xFFFFF) << 0)) +#define DMA_PTE_PDE_PACKET(n) ((2 << 28) | \ + (1 << 26) | \ + (1 << 21) | \ + (((n) & 0xFFFFF) << 0)) + +#define DMA_SRBM_POLL_PACKET ((9 << 28) | \ + (1 << 27) | \ + (1 << 26)) + +#define DMA_SRBM_READ_PACKET ((9 << 28) | \ + (1 << 27)) + /* async DMA Packet types */ #define DMA_PACKET_WRITE 0x2 #define DMA_PACKET_COPY 0x3 diff --git a/sys/dev/pci/drm/radeon/nislands_smc.h b/sys/dev/pci/drm/radeon/nislands_smc.h new file mode 100644 index 00000000000..3cf8fc0d83f --- /dev/null +++ b/sys/dev/pci/drm/radeon/nislands_smc.h @@ -0,0 +1,329 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __NISLANDS_SMC_H__ +#define __NISLANDS_SMC_H__ + +#pragma pack(push, 1) + +#define NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE 16 + +struct PP_NIslands_Dpm2PerfLevel +{ + uint8_t MaxPS; + uint8_t TgtAct; + uint8_t MaxPS_StepInc; + uint8_t MaxPS_StepDec; + uint8_t PSST; + uint8_t NearTDPDec; + uint8_t AboveSafeInc; + uint8_t BelowSafeInc; + uint8_t PSDeltaLimit; + uint8_t PSDeltaWin; + uint8_t Reserved[6]; +}; + +typedef struct PP_NIslands_Dpm2PerfLevel PP_NIslands_Dpm2PerfLevel; + +struct PP_NIslands_DPM2Parameters +{ + uint32_t TDPLimit; + uint32_t NearTDPLimit; + uint32_t SafePowerLimit; + uint32_t PowerBoostLimit; +}; +typedef struct PP_NIslands_DPM2Parameters PP_NIslands_DPM2Parameters; + +struct NISLANDS_SMC_SCLK_VALUE +{ + uint32_t vCG_SPLL_FUNC_CNTL; + uint32_t vCG_SPLL_FUNC_CNTL_2; + uint32_t vCG_SPLL_FUNC_CNTL_3; + uint32_t vCG_SPLL_FUNC_CNTL_4; + uint32_t vCG_SPLL_SPREAD_SPECTRUM; + uint32_t vCG_SPLL_SPREAD_SPECTRUM_2; + uint32_t sclk_value; +}; + +typedef struct NISLANDS_SMC_SCLK_VALUE NISLANDS_SMC_SCLK_VALUE; + +struct NISLANDS_SMC_MCLK_VALUE +{ + uint32_t vMPLL_FUNC_CNTL; + uint32_t vMPLL_FUNC_CNTL_1; + uint32_t vMPLL_FUNC_CNTL_2; + uint32_t vMPLL_AD_FUNC_CNTL; + uint32_t vMPLL_AD_FUNC_CNTL_2; + uint32_t vMPLL_DQ_FUNC_CNTL; + uint32_t vMPLL_DQ_FUNC_CNTL_2; + uint32_t vMCLK_PWRMGT_CNTL; + uint32_t vDLL_CNTL; + uint32_t vMPLL_SS; + uint32_t vMPLL_SS2; + uint32_t mclk_value; +}; + +typedef struct NISLANDS_SMC_MCLK_VALUE NISLANDS_SMC_MCLK_VALUE; + +struct NISLANDS_SMC_VOLTAGE_VALUE +{ + uint16_t value; + uint8_t index; + uint8_t padding; +}; + +typedef struct NISLANDS_SMC_VOLTAGE_VALUE NISLANDS_SMC_VOLTAGE_VALUE; + +struct NISLANDS_SMC_HW_PERFORMANCE_LEVEL +{ + uint8_t arbValue; + uint8_t ACIndex; + uint8_t displayWatermark; + uint8_t gen2PCIE; + uint8_t reserved1; + uint8_t reserved2; + uint8_t strobeMode; + uint8_t mcFlags; + uint32_t aT; + uint32_t bSP; + NISLANDS_SMC_SCLK_VALUE sclk; + NISLANDS_SMC_MCLK_VALUE mclk; + NISLANDS_SMC_VOLTAGE_VALUE vddc; + NISLANDS_SMC_VOLTAGE_VALUE mvdd; + NISLANDS_SMC_VOLTAGE_VALUE vddci; + NISLANDS_SMC_VOLTAGE_VALUE std_vddc; + uint32_t powergate_en; + uint8_t hUp; + uint8_t hDown; + uint8_t stateFlags; + uint8_t arbRefreshState; + uint32_t SQPowerThrottle; + uint32_t SQPowerThrottle_2; + uint32_t reserved[2]; + PP_NIslands_Dpm2PerfLevel dpm2; +}; + +#define NISLANDS_SMC_STROBE_RATIO 0x0F +#define NISLANDS_SMC_STROBE_ENABLE 0x10 + +#define NISLANDS_SMC_MC_EDC_RD_FLAG 0x01 +#define NISLANDS_SMC_MC_EDC_WR_FLAG 0x02 +#define NISLANDS_SMC_MC_RTT_ENABLE 0x04 +#define NISLANDS_SMC_MC_STUTTER_EN 0x08 + +typedef struct NISLANDS_SMC_HW_PERFORMANCE_LEVEL NISLANDS_SMC_HW_PERFORMANCE_LEVEL; + +struct NISLANDS_SMC_SWSTATE +{ + uint8_t flags; + uint8_t levelCount; + uint8_t padding2; + uint8_t padding3; + NISLANDS_SMC_HW_PERFORMANCE_LEVEL levels[1]; +}; + +typedef struct NISLANDS_SMC_SWSTATE NISLANDS_SMC_SWSTATE; + +#define NISLANDS_SMC_VOLTAGEMASK_VDDC 0 +#define NISLANDS_SMC_VOLTAGEMASK_MVDD 1 +#define NISLANDS_SMC_VOLTAGEMASK_VDDCI 2 +#define NISLANDS_SMC_VOLTAGEMASK_MAX 4 + +struct NISLANDS_SMC_VOLTAGEMASKTABLE +{ + uint8_t highMask[NISLANDS_SMC_VOLTAGEMASK_MAX]; + uint32_t lowMask[NISLANDS_SMC_VOLTAGEMASK_MAX]; +}; + +typedef struct NISLANDS_SMC_VOLTAGEMASKTABLE NISLANDS_SMC_VOLTAGEMASKTABLE; + +#define NISLANDS_MAX_NO_VREG_STEPS 32 + +struct NISLANDS_SMC_STATETABLE +{ + uint8_t thermalProtectType; + uint8_t systemFlags; + uint8_t maxVDDCIndexInPPTable; + uint8_t extraFlags; + uint8_t highSMIO[NISLANDS_MAX_NO_VREG_STEPS]; + uint32_t lowSMIO[NISLANDS_MAX_NO_VREG_STEPS]; + NISLANDS_SMC_VOLTAGEMASKTABLE voltageMaskTable; + PP_NIslands_DPM2Parameters dpm2Params; + NISLANDS_SMC_SWSTATE initialState; + NISLANDS_SMC_SWSTATE ACPIState; + NISLANDS_SMC_SWSTATE ULVState; + NISLANDS_SMC_SWSTATE driverState; + NISLANDS_SMC_HW_PERFORMANCE_LEVEL dpmLevels[NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1]; +}; + +typedef struct NISLANDS_SMC_STATETABLE NISLANDS_SMC_STATETABLE; + +#define NI_SMC_SOFT_REGISTERS_START 0x108 + +#define NI_SMC_SOFT_REGISTER_mclk_chg_timeout 0x0 +#define NI_SMC_SOFT_REGISTER_delay_bbias 0xC +#define NI_SMC_SOFT_REGISTER_delay_vreg 0x10 +#define NI_SMC_SOFT_REGISTER_delay_acpi 0x2C +#define NI_SMC_SOFT_REGISTER_seq_index 0x64 +#define NI_SMC_SOFT_REGISTER_mvdd_chg_time 0x68 +#define NI_SMC_SOFT_REGISTER_mclk_switch_lim 0x78 +#define NI_SMC_SOFT_REGISTER_watermark_threshold 0x80 +#define NI_SMC_SOFT_REGISTER_mc_block_delay 0x84 +#define NI_SMC_SOFT_REGISTER_uvd_enabled 0x98 + +#define SMC_NISLANDS_MC_TPP_CAC_NUM_OF_ENTRIES 16 +#define SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16 +#define SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES 16 +#define SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES 4 + +struct SMC_NISLANDS_MC_TPP_CAC_TABLE +{ + uint32_t tpp[SMC_NISLANDS_MC_TPP_CAC_NUM_OF_ENTRIES]; + uint32_t cacValue[SMC_NISLANDS_MC_TPP_CAC_NUM_OF_ENTRIES]; +}; + +typedef struct SMC_NISLANDS_MC_TPP_CAC_TABLE SMC_NISLANDS_MC_TPP_CAC_TABLE; + + +struct PP_NIslands_CACTABLES +{ + uint32_t cac_bif_lut[SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES]; + uint32_t cac_lkge_lut[SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES][SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES]; + + uint32_t pwr_const; + + uint32_t dc_cacValue; + uint32_t bif_cacValue; + uint32_t lkge_pwr; + + uint8_t cac_width; + uint8_t window_size_p2; + + uint8_t num_drop_lsb; + uint8_t padding_0; + + uint32_t last_power; + + uint8_t AllowOvrflw; + uint8_t MCWrWeight; + uint8_t MCRdWeight; + uint8_t padding_1[9]; + + uint8_t enableWinAvg; + uint8_t numWin_TDP; + uint8_t l2numWin_TDP; + uint8_t WinIndex; + + uint32_t dynPwr_TDP[4]; + uint32_t lkgePwr_TDP[4]; + uint32_t power_TDP[4]; + uint32_t avg_dynPwr_TDP; + uint32_t avg_lkgePwr_TDP; + uint32_t avg_power_TDP; + uint32_t lts_power_TDP; + uint8_t lts_truncate_n; + uint8_t padding_2[7]; +}; + +typedef struct PP_NIslands_CACTABLES PP_NIslands_CACTABLES; + +#define SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE 32 +#define SMC_NISLANDS_MC_REGISTER_ARRAY_SET_COUNT 20 + +struct SMC_NIslands_MCRegisterAddress +{ + uint16_t s0; + uint16_t s1; +}; + +typedef struct SMC_NIslands_MCRegisterAddress SMC_NIslands_MCRegisterAddress; + + +struct SMC_NIslands_MCRegisterSet +{ + uint32_t value[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE]; +}; + +typedef struct SMC_NIslands_MCRegisterSet SMC_NIslands_MCRegisterSet; + +struct SMC_NIslands_MCRegisters +{ + uint8_t last; + uint8_t reserved[3]; + SMC_NIslands_MCRegisterAddress address[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE]; + SMC_NIslands_MCRegisterSet data[SMC_NISLANDS_MC_REGISTER_ARRAY_SET_COUNT]; +}; + +typedef struct SMC_NIslands_MCRegisters SMC_NIslands_MCRegisters; + +struct SMC_NIslands_MCArbDramTimingRegisterSet +{ + uint32_t mc_arb_dram_timing; + uint32_t mc_arb_dram_timing2; + uint8_t mc_arb_rfsh_rate; + uint8_t padding[3]; +}; + +typedef struct SMC_NIslands_MCArbDramTimingRegisterSet SMC_NIslands_MCArbDramTimingRegisterSet; + +struct SMC_NIslands_MCArbDramTimingRegisters +{ + uint8_t arb_current; + uint8_t reserved[3]; + SMC_NIslands_MCArbDramTimingRegisterSet data[20]; +}; + +typedef struct SMC_NIslands_MCArbDramTimingRegisters SMC_NIslands_MCArbDramTimingRegisters; + +struct SMC_NISLANDS_SPLL_DIV_TABLE +{ + uint32_t freq[256]; + uint32_t ss[256]; +}; + +#define SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK 0x01ffffff +#define SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT 0 +#define SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK 0xfe000000 +#define SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT 25 +#define SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK 0x000fffff +#define SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT 0 +#define SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK 0xfff00000 +#define SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT 20 + +typedef struct SMC_NISLANDS_SPLL_DIV_TABLE SMC_NISLANDS_SPLL_DIV_TABLE; + +#define NISLANDS_SMC_FIRMWARE_HEADER_LOCATION 0x100 + +#define NISLANDS_SMC_FIRMWARE_HEADER_version 0x0 +#define NISLANDS_SMC_FIRMWARE_HEADER_flags 0x4 +#define NISLANDS_SMC_FIRMWARE_HEADER_softRegisters 0x8 +#define NISLANDS_SMC_FIRMWARE_HEADER_stateTable 0xC +#define NISLANDS_SMC_FIRMWARE_HEADER_fanTable 0x10 +#define NISLANDS_SMC_FIRMWARE_HEADER_cacTable 0x14 +#define NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable 0x20 +#define NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable 0x2C +#define NISLANDS_SMC_FIRMWARE_HEADER_spllTable 0x30 + +#pragma pack(pop) + +#endif + diff --git a/sys/dev/pci/drm/radeon/ppsmc.h b/sys/dev/pci/drm/radeon/ppsmc.h new file mode 100644 index 00000000000..7e5724a12f8 --- /dev/null +++ b/sys/dev/pci/drm/radeon/ppsmc.h @@ -0,0 +1,191 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef PP_SMC_H +#define PP_SMC_H + +#pragma pack(push, 1) + +#define PPSMC_SWSTATE_FLAG_DC 0x01 +#define PPSMC_SWSTATE_FLAG_UVD 0x02 +#define PPSMC_SWSTATE_FLAG_VCE 0x04 +#define PPSMC_SWSTATE_FLAG_PCIE_X1 0x08 + +#define PPSMC_THERMAL_PROTECT_TYPE_INTERNAL 0x00 +#define PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL 0x01 +#define PPSMC_THERMAL_PROTECT_TYPE_NONE 0xff + +#define PPSMC_SYSTEMFLAG_GPIO_DC 0x01 +#define PPSMC_SYSTEMFLAG_STEPVDDC 0x02 +#define PPSMC_SYSTEMFLAG_GDDR5 0x04 +#define PPSMC_SYSTEMFLAG_DISABLE_BABYSTEP 0x08 +#define PPSMC_SYSTEMFLAG_REGULATOR_HOT 0x10 +#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_ANALOG 0x20 +#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_PROG_GPIO 0x40 + +#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_MASK 0x07 +#define PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK 0x08 +#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTODPMLOWSTATE 0x00 +#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE 0x01 +#define PPSMC_EXTRAFLAGS_AC2DC_GPIO5_POLARITY_HIGH 0x02 + +#define PPSMC_DISPLAY_WATERMARK_LOW 0 +#define PPSMC_DISPLAY_WATERMARK_HIGH 1 + +#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP 0x01 +#define PPSMC_STATEFLAG_POWERBOOST 0x02 +#define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 0x20 +#define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS 0x40 + +#define FDO_MODE_HARDWARE 0 +#define FDO_MODE_PIECE_WISE_LINEAR 1 + +enum FAN_CONTROL { + FAN_CONTROL_FUZZY, + FAN_CONTROL_TABLE +}; + +#define PPSMC_Result_OK ((uint8_t)0x01) +#define PPSMC_Result_Failed ((uint8_t)0xFF) + +typedef uint8_t PPSMC_Result; + +#define PPSMC_MSG_Halt ((uint8_t)0x10) +#define PPSMC_MSG_Resume ((uint8_t)0x11) +#define PPSMC_MSG_ZeroLevelsDisabled ((uint8_t)0x13) +#define PPSMC_MSG_OneLevelsDisabled ((uint8_t)0x14) +#define PPSMC_MSG_TwoLevelsDisabled ((uint8_t)0x15) +#define PPSMC_MSG_EnableThermalInterrupt ((uint8_t)0x16) +#define PPSMC_MSG_RunningOnAC ((uint8_t)0x17) +#define PPSMC_MSG_SwitchToSwState ((uint8_t)0x20) +#define PPSMC_MSG_SwitchToInitialState ((uint8_t)0x40) +#define PPSMC_MSG_NoForcedLevel ((uint8_t)0x41) +#define PPSMC_MSG_ForceHigh ((uint8_t)0x42) +#define PPSMC_MSG_ForceMediumOrHigh ((uint8_t)0x43) +#define PPSMC_MSG_SwitchToMinimumPower ((uint8_t)0x51) +#define PPSMC_MSG_ResumeFromMinimumPower ((uint8_t)0x52) +#define PPSMC_MSG_EnableCac ((uint8_t)0x53) +#define PPSMC_MSG_DisableCac ((uint8_t)0x54) +#define PPSMC_TDPClampingActive ((uint8_t)0x59) +#define PPSMC_TDPClampingInactive ((uint8_t)0x5A) +#define PPSMC_StartFanControl ((uint8_t)0x5B) +#define PPSMC_StopFanControl ((uint8_t)0x5C) +#define PPSMC_MSG_NoDisplay ((uint8_t)0x5D) +#define PPSMC_MSG_HasDisplay ((uint8_t)0x5E) +#define PPSMC_MSG_UVDPowerOFF ((uint8_t)0x60) +#define PPSMC_MSG_UVDPowerON ((uint8_t)0x61) +#define PPSMC_MSG_EnableULV ((uint8_t)0x62) +#define PPSMC_MSG_DisableULV ((uint8_t)0x63) +#define PPSMC_MSG_EnterULV ((uint8_t)0x64) +#define PPSMC_MSG_ExitULV ((uint8_t)0x65) +#define PPSMC_CACLongTermAvgEnable ((uint8_t)0x6E) +#define PPSMC_CACLongTermAvgDisable ((uint8_t)0x6F) +#define PPSMC_MSG_CollectCAC_PowerCorreln ((uint8_t)0x7A) +#define PPSMC_FlushDataCache ((uint8_t)0x80) +#define PPSMC_MSG_SetEnabledLevels ((uint8_t)0x82) +#define PPSMC_MSG_SetForcedLevels ((uint8_t)0x83) +#define PPSMC_MSG_ResetToDefaults ((uint8_t)0x84) +#define PPSMC_MSG_EnableDTE ((uint8_t)0x87) +#define PPSMC_MSG_DisableDTE ((uint8_t)0x88) +#define PPSMC_MSG_ThrottleOVRDSCLKDS ((uint8_t)0x96) +#define PPSMC_MSG_CancelThrottleOVRDSCLKDS ((uint8_t)0x97) + +/* CI/KV/KB */ +#define PPSMC_MSG_UVDDPM_SetEnabledMask ((uint16_t) 0x12D) +#define PPSMC_MSG_VCEDPM_SetEnabledMask ((uint16_t) 0x12E) +#define PPSMC_MSG_ACPDPM_SetEnabledMask ((uint16_t) 0x12F) +#define PPSMC_MSG_SAMUDPM_SetEnabledMask ((uint16_t) 0x130) +#define PPSMC_MSG_MCLKDPM_ForceState ((uint16_t) 0x131) +#define PPSMC_MSG_MCLKDPM_NoForcedLevel ((uint16_t) 0x132) +#define PPSMC_MSG_Thermal_Cntl_Disable ((uint16_t) 0x133) +#define PPSMC_MSG_Voltage_Cntl_Disable ((uint16_t) 0x135) +#define PPSMC_MSG_PCIeDPM_Enable ((uint16_t) 0x136) +#define PPSMC_MSG_PCIeDPM_Disable ((uint16_t) 0x13d) +#define PPSMC_MSG_ACPPowerOFF ((uint16_t) 0x137) +#define PPSMC_MSG_ACPPowerON ((uint16_t) 0x138) +#define PPSMC_MSG_SAMPowerOFF ((uint16_t) 0x139) +#define PPSMC_MSG_SAMPowerON ((uint16_t) 0x13a) +#define PPSMC_MSG_PCIeDPM_Disable ((uint16_t) 0x13d) +#define PPSMC_MSG_NBDPM_Enable ((uint16_t) 0x140) +#define PPSMC_MSG_NBDPM_Disable ((uint16_t) 0x141) +#define PPSMC_MSG_SCLKDPM_SetEnabledMask ((uint16_t) 0x145) +#define PPSMC_MSG_MCLKDPM_SetEnabledMask ((uint16_t) 0x146) +#define PPSMC_MSG_PCIeDPM_ForceLevel ((uint16_t) 0x147) +#define PPSMC_MSG_PCIeDPM_UnForceLevel ((uint16_t) 0x148) +#define PPSMC_MSG_EnableVRHotGPIOInterrupt ((uint16_t) 0x14a) +#define PPSMC_MSG_DPM_Enable ((uint16_t) 0x14e) +#define PPSMC_MSG_DPM_Disable ((uint16_t) 0x14f) +#define PPSMC_MSG_MCLKDPM_Enable ((uint16_t) 0x150) +#define PPSMC_MSG_MCLKDPM_Disable ((uint16_t) 0x151) +#define PPSMC_MSG_UVDDPM_Enable ((uint16_t) 0x154) +#define PPSMC_MSG_UVDDPM_Disable ((uint16_t) 0x155) +#define PPSMC_MSG_SAMUDPM_Enable ((uint16_t) 0x156) +#define PPSMC_MSG_SAMUDPM_Disable ((uint16_t) 0x157) +#define PPSMC_MSG_ACPDPM_Enable ((uint16_t) 0x158) +#define PPSMC_MSG_ACPDPM_Disable ((uint16_t) 0x159) +#define PPSMC_MSG_VCEDPM_Enable ((uint16_t) 0x15a) +#define PPSMC_MSG_VCEDPM_Disable ((uint16_t) 0x15b) +#define PPSMC_MSG_VddC_Request ((uint16_t) 0x15f) +#define PPSMC_MSG_SCLKDPM_GetEnabledMask ((uint16_t) 0x162) +#define PPSMC_MSG_PCIeDPM_SetEnabledMask ((uint16_t) 0x167) +#define PPSMC_MSG_TDCLimitEnable ((uint16_t) 0x169) +#define PPSMC_MSG_TDCLimitDisable ((uint16_t) 0x16a) +#define PPSMC_MSG_PkgPwrLimitEnable ((uint16_t) 0x185) +#define PPSMC_MSG_PkgPwrLimitDisable ((uint16_t) 0x186) +#define PPSMC_MSG_PkgPwrSetLimit ((uint16_t) 0x187) +#define PPSMC_MSG_OverDriveSetTargetTdp ((uint16_t) 0x188) +#define PPSMC_MSG_SCLKDPM_FreezeLevel ((uint16_t) 0x189) +#define PPSMC_MSG_SCLKDPM_UnfreezeLevel ((uint16_t) 0x18A) +#define PPSMC_MSG_MCLKDPM_FreezeLevel ((uint16_t) 0x18B) +#define PPSMC_MSG_MCLKDPM_UnfreezeLevel ((uint16_t) 0x18C) +#define PPSMC_MSG_MASTER_DeepSleep_ON ((uint16_t) 0x18F) +#define PPSMC_MSG_MASTER_DeepSleep_OFF ((uint16_t) 0x190) +#define PPSMC_MSG_Remove_DC_Clamp ((uint16_t) 0x191) +#define PPSMC_MSG_SetFanPwmMax ((uint16_t) 0x19A) + +#define PPSMC_MSG_ENABLE_THERMAL_DPM ((uint16_t) 0x19C) +#define PPSMC_MSG_DISABLE_THERMAL_DPM ((uint16_t) 0x19D) + +#define PPSMC_MSG_API_GetSclkFrequency ((uint16_t) 0x200) +#define PPSMC_MSG_API_GetMclkFrequency ((uint16_t) 0x201) + +/* TN */ +#define PPSMC_MSG_DPM_Config ((uint32_t) 0x102) +#define PPSMC_MSG_DPM_ForceState ((uint32_t) 0x104) +#define PPSMC_MSG_PG_SIMD_Config ((uint32_t) 0x108) +#define PPSMC_MSG_Thermal_Cntl_Enable ((uint32_t) 0x10a) +#define PPSMC_MSG_Voltage_Cntl_Enable ((uint32_t) 0x109) +#define PPSMC_MSG_VCEPowerOFF ((uint32_t) 0x10e) +#define PPSMC_MSG_VCEPowerON ((uint32_t) 0x10f) +#define PPSMC_MSG_DPM_N_LevelsDisabled ((uint32_t) 0x112) +#define PPSMC_MSG_DCE_RemoveVoltageAdjustment ((uint32_t) 0x11d) +#define PPSMC_MSG_DCE_AllowVoltageAdjustment ((uint32_t) 0x11e) +#define PPSMC_MSG_EnableBAPM ((uint32_t) 0x120) +#define PPSMC_MSG_DisableBAPM ((uint32_t) 0x121) +#define PPSMC_MSG_UVD_DPM_Config ((uint32_t) 0x124) + + +typedef uint16_t PPSMC_Msg; + +#pragma pack(pop) + +#endif diff --git a/sys/dev/pci/drm/radeon/pptable.h b/sys/dev/pci/drm/radeon/pptable.h new file mode 100644 index 00000000000..4c2eec49dad --- /dev/null +++ b/sys/dev/pci/drm/radeon/pptable.h @@ -0,0 +1,690 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _PPTABLE_H +#define _PPTABLE_H + +#pragma pack(1) + +typedef struct _ATOM_PPLIB_THERMALCONTROLLER + +{ + UCHAR ucType; // one of ATOM_PP_THERMALCONTROLLER_* + UCHAR ucI2cLine; // as interpreted by DAL I2C + UCHAR ucI2cAddress; + UCHAR ucFanParameters; // Fan Control Parameters. + UCHAR ucFanMinRPM; // Fan Minimum RPM (hundreds) -- for display purposes only. + UCHAR ucFanMaxRPM; // Fan Maximum RPM (hundreds) -- for display purposes only. + UCHAR ucReserved; // ---- + UCHAR ucFlags; // to be defined +} ATOM_PPLIB_THERMALCONTROLLER; + +#define ATOM_PP_FANPARAMETERS_TACHOMETER_PULSES_PER_REVOLUTION_MASK 0x0f +#define ATOM_PP_FANPARAMETERS_NOFAN 0x80 // No fan is connected to this controller. + +#define ATOM_PP_THERMALCONTROLLER_NONE 0 +#define ATOM_PP_THERMALCONTROLLER_LM63 1 // Not used by PPLib +#define ATOM_PP_THERMALCONTROLLER_ADM1032 2 // Not used by PPLib +#define ATOM_PP_THERMALCONTROLLER_ADM1030 3 // Not used by PPLib +#define ATOM_PP_THERMALCONTROLLER_MUA6649 4 // Not used by PPLib +#define ATOM_PP_THERMALCONTROLLER_LM64 5 +#define ATOM_PP_THERMALCONTROLLER_F75375 6 // Not used by PPLib +#define ATOM_PP_THERMALCONTROLLER_RV6xx 7 +#define ATOM_PP_THERMALCONTROLLER_RV770 8 +#define ATOM_PP_THERMALCONTROLLER_ADT7473 9 +#define ATOM_PP_THERMALCONTROLLER_KONG 10 +#define ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO 11 +#define ATOM_PP_THERMALCONTROLLER_EVERGREEN 12 +#define ATOM_PP_THERMALCONTROLLER_EMC2103 13 /* 0x0D */ // Only fan control will be implemented, do NOT show this in PPGen. +#define ATOM_PP_THERMALCONTROLLER_SUMO 14 /* 0x0E */ // Sumo type, used internally +#define ATOM_PP_THERMALCONTROLLER_NISLANDS 15 +#define ATOM_PP_THERMALCONTROLLER_SISLANDS 16 +#define ATOM_PP_THERMALCONTROLLER_LM96163 17 +#define ATOM_PP_THERMALCONTROLLER_CISLANDS 18 +#define ATOM_PP_THERMALCONTROLLER_KAVERI 19 + + +// Thermal controller 'combo type' to use an external controller for Fan control and an internal controller for thermal. +// We probably should reserve the bit 0x80 for this use. +// To keep the number of these types low we should also use the same code for all ASICs (i.e. do not distinguish RV6xx and RV7xx Internal here). +// The driver can pick the correct internal controller based on the ASIC. + +#define ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL 0x89 // ADT7473 Fan Control + Internal Thermal Controller +#define ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL 0x8D // EMC2103 Fan Control + Internal Thermal Controller + +typedef struct _ATOM_PPLIB_STATE +{ + UCHAR ucNonClockStateIndex; + UCHAR ucClockStateIndices[1]; // variable-sized +} ATOM_PPLIB_STATE; + + +typedef struct _ATOM_PPLIB_FANTABLE +{ + UCHAR ucFanTableFormat; // Change this if the table format changes or version changes so that the other fields are not the same. + UCHAR ucTHyst; // Temperature hysteresis. Integer. + USHORT usTMin; // The temperature, in 0.01 centigrades, below which we just run at a minimal PWM. + USHORT usTMed; // The middle temperature where we change slopes. + USHORT usTHigh; // The high point above TMed for adjusting the second slope. + USHORT usPWMMin; // The minimum PWM value in percent (0.01% increments). + USHORT usPWMMed; // The PWM value (in percent) at TMed. + USHORT usPWMHigh; // The PWM value at THigh. +} ATOM_PPLIB_FANTABLE; + +typedef struct _ATOM_PPLIB_FANTABLE2 +{ + ATOM_PPLIB_FANTABLE basicTable; + USHORT usTMax; // The max temperature +} ATOM_PPLIB_FANTABLE2; + +typedef struct _ATOM_PPLIB_FANTABLE3 +{ + ATOM_PPLIB_FANTABLE2 basicTable2; + UCHAR ucFanControlMode; + USHORT usFanPWMMax; + USHORT usFanOutputSensitivity; +} ATOM_PPLIB_FANTABLE3; + +typedef struct _ATOM_PPLIB_EXTENDEDHEADER +{ + USHORT usSize; + ULONG ulMaxEngineClock; // For Overdrive. + ULONG ulMaxMemoryClock; // For Overdrive. + // Add extra system parameters here, always adjust size to include all fields. + USHORT usVCETableOffset; //points to ATOM_PPLIB_VCE_Table + USHORT usUVDTableOffset; //points to ATOM_PPLIB_UVD_Table + USHORT usSAMUTableOffset; //points to ATOM_PPLIB_SAMU_Table + USHORT usPPMTableOffset; //points to ATOM_PPLIB_PPM_Table + USHORT usACPTableOffset; //points to ATOM_PPLIB_ACP_Table + USHORT usPowerTuneTableOffset; //points to ATOM_PPLIB_POWERTUNE_Table +} ATOM_PPLIB_EXTENDEDHEADER; + +//// ATOM_PPLIB_POWERPLAYTABLE::ulPlatformCaps +#define ATOM_PP_PLATFORM_CAP_BACKBIAS 1 +#define ATOM_PP_PLATFORM_CAP_POWERPLAY 2 +#define ATOM_PP_PLATFORM_CAP_SBIOSPOWERSOURCE 4 +#define ATOM_PP_PLATFORM_CAP_ASPM_L0s 8 +#define ATOM_PP_PLATFORM_CAP_ASPM_L1 16 +#define ATOM_PP_PLATFORM_CAP_HARDWAREDC 32 +#define ATOM_PP_PLATFORM_CAP_GEMINIPRIMARY 64 +#define ATOM_PP_PLATFORM_CAP_STEPVDDC 128 +#define ATOM_PP_PLATFORM_CAP_VOLTAGECONTROL 256 +#define ATOM_PP_PLATFORM_CAP_SIDEPORTCONTROL 512 +#define ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1 1024 +#define ATOM_PP_PLATFORM_CAP_HTLINKCONTROL 2048 +#define ATOM_PP_PLATFORM_CAP_MVDDCONTROL 4096 +#define ATOM_PP_PLATFORM_CAP_GOTO_BOOT_ON_ALERT 0x2000 // Go to boot state on alerts, e.g. on an AC->DC transition. +#define ATOM_PP_PLATFORM_CAP_DONT_WAIT_FOR_VBLANK_ON_ALERT 0x4000 // Do NOT wait for VBLANK during an alert (e.g. AC->DC transition). +#define ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL 0x8000 // Does the driver control VDDCI independently from VDDC. +#define ATOM_PP_PLATFORM_CAP_REGULATOR_HOT 0x00010000 // Enable the 'regulator hot' feature. +#define ATOM_PP_PLATFORM_CAP_BACO 0x00020000 // Does the driver supports BACO state. +#define ATOM_PP_PLATFORM_CAP_NEW_CAC_VOLTAGE 0x00040000 // Does the driver supports new CAC voltage table. +#define ATOM_PP_PLATFORM_CAP_REVERT_GPIO5_POLARITY 0x00080000 // Does the driver supports revert GPIO5 polarity. +#define ATOM_PP_PLATFORM_CAP_OUTPUT_THERMAL2GPIO17 0x00100000 // Does the driver supports thermal2GPIO17. +#define ATOM_PP_PLATFORM_CAP_VRHOT_GPIO_CONFIGURABLE 0x00200000 // Does the driver supports VR HOT GPIO Configurable. +#define ATOM_PP_PLATFORM_CAP_TEMP_INVERSION 0x00400000 // Does the driver supports Temp Inversion feature. +#define ATOM_PP_PLATFORM_CAP_EVV 0x00800000 + +typedef struct _ATOM_PPLIB_POWERPLAYTABLE +{ + ATOM_COMMON_TABLE_HEADER sHeader; + + UCHAR ucDataRevision; + + UCHAR ucNumStates; + UCHAR ucStateEntrySize; + UCHAR ucClockInfoSize; + UCHAR ucNonClockSize; + + // offset from start of this table to array of ucNumStates ATOM_PPLIB_STATE structures + USHORT usStateArrayOffset; + + // offset from start of this table to array of ASIC-specific structures, + // currently ATOM_PPLIB_CLOCK_INFO. + USHORT usClockInfoArrayOffset; + + // offset from start of this table to array of ATOM_PPLIB_NONCLOCK_INFO + USHORT usNonClockInfoArrayOffset; + + USHORT usBackbiasTime; // in microseconds + USHORT usVoltageTime; // in microseconds + USHORT usTableSize; //the size of this structure, or the extended structure + + ULONG ulPlatformCaps; // See ATOM_PPLIB_CAPS_* + + ATOM_PPLIB_THERMALCONTROLLER sThermalController; + + USHORT usBootClockInfoOffset; + USHORT usBootNonClockInfoOffset; + +} ATOM_PPLIB_POWERPLAYTABLE; + +typedef struct _ATOM_PPLIB_POWERPLAYTABLE2 +{ + ATOM_PPLIB_POWERPLAYTABLE basicTable; + UCHAR ucNumCustomThermalPolicy; + USHORT usCustomThermalPolicyArrayOffset; +}ATOM_PPLIB_POWERPLAYTABLE2, *LPATOM_PPLIB_POWERPLAYTABLE2; + +typedef struct _ATOM_PPLIB_POWERPLAYTABLE3 +{ + ATOM_PPLIB_POWERPLAYTABLE2 basicTable2; + USHORT usFormatID; // To be used ONLY by PPGen. + USHORT usFanTableOffset; + USHORT usExtendendedHeaderOffset; +} ATOM_PPLIB_POWERPLAYTABLE3, *LPATOM_PPLIB_POWERPLAYTABLE3; + +typedef struct _ATOM_PPLIB_POWERPLAYTABLE4 +{ + ATOM_PPLIB_POWERPLAYTABLE3 basicTable3; + ULONG ulGoldenPPID; // PPGen use only + ULONG ulGoldenRevision; // PPGen use only + USHORT usVddcDependencyOnSCLKOffset; + USHORT usVddciDependencyOnMCLKOffset; + USHORT usVddcDependencyOnMCLKOffset; + USHORT usMaxClockVoltageOnDCOffset; + USHORT usVddcPhaseShedLimitsTableOffset; // Points to ATOM_PPLIB_PhaseSheddingLimits_Table + USHORT usMvddDependencyOnMCLKOffset; +} ATOM_PPLIB_POWERPLAYTABLE4, *LPATOM_PPLIB_POWERPLAYTABLE4; + +typedef struct _ATOM_PPLIB_POWERPLAYTABLE5 +{ + ATOM_PPLIB_POWERPLAYTABLE4 basicTable4; + ULONG ulTDPLimit; + ULONG ulNearTDPLimit; + ULONG ulSQRampingThreshold; + USHORT usCACLeakageTableOffset; // Points to ATOM_PPLIB_CAC_Leakage_Table + ULONG ulCACLeakage; // The iLeakage for driver calculated CAC leakage table + USHORT usTDPODLimit; + USHORT usLoadLineSlope; // in milliOhms * 100 +} ATOM_PPLIB_POWERPLAYTABLE5, *LPATOM_PPLIB_POWERPLAYTABLE5; + +//// ATOM_PPLIB_NONCLOCK_INFO::usClassification +#define ATOM_PPLIB_CLASSIFICATION_UI_MASK 0x0007 +#define ATOM_PPLIB_CLASSIFICATION_UI_SHIFT 0 +#define ATOM_PPLIB_CLASSIFICATION_UI_NONE 0 +#define ATOM_PPLIB_CLASSIFICATION_UI_BATTERY 1 +#define ATOM_PPLIB_CLASSIFICATION_UI_BALANCED 3 +#define ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE 5 +// 2, 4, 6, 7 are reserved + +#define ATOM_PPLIB_CLASSIFICATION_BOOT 0x0008 +#define ATOM_PPLIB_CLASSIFICATION_THERMAL 0x0010 +#define ATOM_PPLIB_CLASSIFICATION_LIMITEDPOWERSOURCE 0x0020 +#define ATOM_PPLIB_CLASSIFICATION_REST 0x0040 +#define ATOM_PPLIB_CLASSIFICATION_FORCED 0x0080 +#define ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE 0x0100 +#define ATOM_PPLIB_CLASSIFICATION_OVERDRIVETEMPLATE 0x0200 +#define ATOM_PPLIB_CLASSIFICATION_UVDSTATE 0x0400 +#define ATOM_PPLIB_CLASSIFICATION_3DLOW 0x0800 +#define ATOM_PPLIB_CLASSIFICATION_ACPI 0x1000 +#define ATOM_PPLIB_CLASSIFICATION_HD2STATE 0x2000 +#define ATOM_PPLIB_CLASSIFICATION_HDSTATE 0x4000 +#define ATOM_PPLIB_CLASSIFICATION_SDSTATE 0x8000 + +//// ATOM_PPLIB_NONCLOCK_INFO::usClassification2 +#define ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2 0x0001 +#define ATOM_PPLIB_CLASSIFICATION2_ULV 0x0002 +#define ATOM_PPLIB_CLASSIFICATION2_MVC 0x0004 //Multi-View Codec (BD-3D) + +//// ATOM_PPLIB_NONCLOCK_INFO::ulCapsAndSettings +#define ATOM_PPLIB_SINGLE_DISPLAY_ONLY 0x00000001 +#define ATOM_PPLIB_SUPPORTS_VIDEO_PLAYBACK 0x00000002 + +// 0 is 2.5Gb/s, 1 is 5Gb/s +#define ATOM_PPLIB_PCIE_LINK_SPEED_MASK 0x00000004 +#define ATOM_PPLIB_PCIE_LINK_SPEED_SHIFT 2 + +// lanes - 1: 1, 2, 4, 8, 12, 16 permitted by PCIE spec +#define ATOM_PPLIB_PCIE_LINK_WIDTH_MASK 0x000000F8 +#define ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT 3 + +// lookup into reduced refresh-rate table +#define ATOM_PPLIB_LIMITED_REFRESHRATE_VALUE_MASK 0x00000F00 +#define ATOM_PPLIB_LIMITED_REFRESHRATE_VALUE_SHIFT 8 + +#define ATOM_PPLIB_LIMITED_REFRESHRATE_UNLIMITED 0 +#define ATOM_PPLIB_LIMITED_REFRESHRATE_50HZ 1 +// 2-15 TBD as needed. + +#define ATOM_PPLIB_SOFTWARE_DISABLE_LOADBALANCING 0x00001000 +#define ATOM_PPLIB_SOFTWARE_ENABLE_SLEEP_FOR_TIMESTAMPS 0x00002000 + +#define ATOM_PPLIB_DISALLOW_ON_DC 0x00004000 + +#define ATOM_PPLIB_ENABLE_VARIBRIGHT 0x00008000 + +//memory related flags +#define ATOM_PPLIB_SWSTATE_MEMORY_DLL_OFF 0x000010000 + +//M3 Arb //2bits, current 3 sets of parameters in total +#define ATOM_PPLIB_M3ARB_MASK 0x00060000 +#define ATOM_PPLIB_M3ARB_SHIFT 17 + +#define ATOM_PPLIB_ENABLE_DRR 0x00080000 + +// remaining 16 bits are reserved +typedef struct _ATOM_PPLIB_THERMAL_STATE +{ + UCHAR ucMinTemperature; + UCHAR ucMaxTemperature; + UCHAR ucThermalAction; +}ATOM_PPLIB_THERMAL_STATE, *LPATOM_PPLIB_THERMAL_STATE; + +// Contained in an array starting at the offset +// in ATOM_PPLIB_POWERPLAYTABLE::usNonClockInfoArrayOffset. +// referenced from ATOM_PPLIB_STATE_INFO::ucNonClockStateIndex +#define ATOM_PPLIB_NONCLOCKINFO_VER1 12 +#define ATOM_PPLIB_NONCLOCKINFO_VER2 24 +typedef struct _ATOM_PPLIB_NONCLOCK_INFO +{ + USHORT usClassification; + UCHAR ucMinTemperature; + UCHAR ucMaxTemperature; + ULONG ulCapsAndSettings; + UCHAR ucRequiredPower; + USHORT usClassification2; + ULONG ulVCLK; + ULONG ulDCLK; + UCHAR ucUnused[5]; +} ATOM_PPLIB_NONCLOCK_INFO; + +// Contained in an array starting at the offset +// in ATOM_PPLIB_POWERPLAYTABLE::usClockInfoArrayOffset. +// referenced from ATOM_PPLIB_STATE::ucClockStateIndices +typedef struct _ATOM_PPLIB_R600_CLOCK_INFO +{ + USHORT usEngineClockLow; + UCHAR ucEngineClockHigh; + + USHORT usMemoryClockLow; + UCHAR ucMemoryClockHigh; + + USHORT usVDDC; + USHORT usUnused1; + USHORT usUnused2; + + ULONG ulFlags; // ATOM_PPLIB_R600_FLAGS_* + +} ATOM_PPLIB_R600_CLOCK_INFO; + +// ulFlags in ATOM_PPLIB_R600_CLOCK_INFO +#define ATOM_PPLIB_R600_FLAGS_PCIEGEN2 1 +#define ATOM_PPLIB_R600_FLAGS_UVDSAFE 2 +#define ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE 4 +#define ATOM_PPLIB_R600_FLAGS_MEMORY_ODT_OFF 8 +#define ATOM_PPLIB_R600_FLAGS_MEMORY_DLL_OFF 16 +#define ATOM_PPLIB_R600_FLAGS_LOWPOWER 32 // On the RV770 use 'low power' setting (sequencer S0). + +typedef struct _ATOM_PPLIB_RS780_CLOCK_INFO + +{ + USHORT usLowEngineClockLow; // Low Engine clock in MHz (the same way as on the R600). + UCHAR ucLowEngineClockHigh; + USHORT usHighEngineClockLow; // High Engine clock in MHz. + UCHAR ucHighEngineClockHigh; + USHORT usMemoryClockLow; // For now one of the ATOM_PPLIB_RS780_SPMCLK_XXXX constants. + UCHAR ucMemoryClockHigh; // Currentyl unused. + UCHAR ucPadding; // For proper alignment and size. + USHORT usVDDC; // For the 780, use: None, Low, High, Variable + UCHAR ucMaxHTLinkWidth; // From SBIOS - {2, 4, 8, 16} + UCHAR ucMinHTLinkWidth; // From SBIOS - {2, 4, 8, 16}. Effective only if CDLW enabled. Minimum down stream width could + USHORT usHTLinkFreq; // See definition ATOM_PPLIB_RS780_HTLINKFREQ_xxx or in MHz(>=200). + ULONG ulFlags; +} ATOM_PPLIB_RS780_CLOCK_INFO; + +#define ATOM_PPLIB_RS780_VOLTAGE_NONE 0 +#define ATOM_PPLIB_RS780_VOLTAGE_LOW 1 +#define ATOM_PPLIB_RS780_VOLTAGE_HIGH 2 +#define ATOM_PPLIB_RS780_VOLTAGE_VARIABLE 3 + +#define ATOM_PPLIB_RS780_SPMCLK_NONE 0 // We cannot change the side port memory clock, leave it as it is. +#define ATOM_PPLIB_RS780_SPMCLK_LOW 1 +#define ATOM_PPLIB_RS780_SPMCLK_HIGH 2 + +#define ATOM_PPLIB_RS780_HTLINKFREQ_NONE 0 +#define ATOM_PPLIB_RS780_HTLINKFREQ_LOW 1 +#define ATOM_PPLIB_RS780_HTLINKFREQ_HIGH 2 + +typedef struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO +{ + USHORT usEngineClockLow; + UCHAR ucEngineClockHigh; + + USHORT usMemoryClockLow; + UCHAR ucMemoryClockHigh; + + USHORT usVDDC; + USHORT usVDDCI; + USHORT usUnused; + + ULONG ulFlags; // ATOM_PPLIB_R600_FLAGS_* + +} ATOM_PPLIB_EVERGREEN_CLOCK_INFO; + +typedef struct _ATOM_PPLIB_SI_CLOCK_INFO +{ + USHORT usEngineClockLow; + UCHAR ucEngineClockHigh; + + USHORT usMemoryClockLow; + UCHAR ucMemoryClockHigh; + + USHORT usVDDC; + USHORT usVDDCI; + UCHAR ucPCIEGen; + UCHAR ucUnused1; + + ULONG ulFlags; // ATOM_PPLIB_SI_FLAGS_*, no flag is necessary for now + +} ATOM_PPLIB_SI_CLOCK_INFO; + +typedef struct _ATOM_PPLIB_CI_CLOCK_INFO +{ + USHORT usEngineClockLow; + UCHAR ucEngineClockHigh; + + USHORT usMemoryClockLow; + UCHAR ucMemoryClockHigh; + + UCHAR ucPCIEGen; + USHORT usPCIELane; +} ATOM_PPLIB_CI_CLOCK_INFO; + +typedef struct _ATOM_PPLIB_SUMO_CLOCK_INFO{ + USHORT usEngineClockLow; //clockfrequency & 0xFFFF. The unit is in 10khz + UCHAR ucEngineClockHigh; //clockfrequency >> 16. + UCHAR vddcIndex; //2-bit vddc index; + USHORT tdpLimit; + //please initalize to 0 + USHORT rsv1; + //please initialize to 0s + ULONG rsv2[2]; +}ATOM_PPLIB_SUMO_CLOCK_INFO; + +typedef struct _ATOM_PPLIB_STATE_V2 +{ + //number of valid dpm levels in this state; Driver uses it to calculate the whole + //size of the state: sizeof(ATOM_PPLIB_STATE_V2) + (ucNumDPMLevels - 1) * sizeof(UCHAR) + UCHAR ucNumDPMLevels; + + //a index to the array of nonClockInfos + UCHAR nonClockInfoIndex; + /** + * Driver will read the first ucNumDPMLevels in this array + */ + UCHAR clockInfoIndex[1]; +} ATOM_PPLIB_STATE_V2; + +typedef struct _StateArray{ + //how many states we have + UCHAR ucNumEntries; + + ATOM_PPLIB_STATE_V2 states[1]; +}StateArray; + + +typedef struct _ClockInfoArray{ + //how many clock levels we have + UCHAR ucNumEntries; + + //sizeof(ATOM_PPLIB_CLOCK_INFO) + UCHAR ucEntrySize; + + UCHAR clockInfo[1]; +}ClockInfoArray; + +typedef struct _NonClockInfoArray{ + + //how many non-clock levels we have. normally should be same as number of states + UCHAR ucNumEntries; + //sizeof(ATOM_PPLIB_NONCLOCK_INFO) + UCHAR ucEntrySize; + + ATOM_PPLIB_NONCLOCK_INFO nonClockInfo[1]; +}NonClockInfoArray; + +typedef struct _ATOM_PPLIB_Clock_Voltage_Dependency_Record +{ + USHORT usClockLow; + UCHAR ucClockHigh; + USHORT usVoltage; +}ATOM_PPLIB_Clock_Voltage_Dependency_Record; + +typedef struct _ATOM_PPLIB_Clock_Voltage_Dependency_Table +{ + UCHAR ucNumEntries; // Number of entries. + ATOM_PPLIB_Clock_Voltage_Dependency_Record entries[1]; // Dynamically allocate entries. +}ATOM_PPLIB_Clock_Voltage_Dependency_Table; + +typedef struct _ATOM_PPLIB_Clock_Voltage_Limit_Record +{ + USHORT usSclkLow; + UCHAR ucSclkHigh; + USHORT usMclkLow; + UCHAR ucMclkHigh; + USHORT usVddc; + USHORT usVddci; +}ATOM_PPLIB_Clock_Voltage_Limit_Record; + +typedef struct _ATOM_PPLIB_Clock_Voltage_Limit_Table +{ + UCHAR ucNumEntries; // Number of entries. + ATOM_PPLIB_Clock_Voltage_Limit_Record entries[1]; // Dynamically allocate entries. +}ATOM_PPLIB_Clock_Voltage_Limit_Table; + +union _ATOM_PPLIB_CAC_Leakage_Record +{ + struct + { + USHORT usVddc; // We use this field for the "fake" standardized VDDC for power calculations; For CI and newer, we use this as the real VDDC value. in CI we read it as StdVoltageHiSidd + ULONG ulLeakageValue; // For CI and newer we use this as the "fake" standar VDDC value. in CI we read it as StdVoltageLoSidd + + }; + struct + { + USHORT usVddc1; + USHORT usVddc2; + USHORT usVddc3; + }; +}; + +typedef union _ATOM_PPLIB_CAC_Leakage_Record ATOM_PPLIB_CAC_Leakage_Record; + +typedef struct _ATOM_PPLIB_CAC_Leakage_Table +{ + UCHAR ucNumEntries; // Number of entries. + ATOM_PPLIB_CAC_Leakage_Record entries[1]; // Dynamically allocate entries. +}ATOM_PPLIB_CAC_Leakage_Table; + +typedef struct _ATOM_PPLIB_PhaseSheddingLimits_Record +{ + USHORT usVoltage; + USHORT usSclkLow; + UCHAR ucSclkHigh; + USHORT usMclkLow; + UCHAR ucMclkHigh; +}ATOM_PPLIB_PhaseSheddingLimits_Record; + +typedef struct _ATOM_PPLIB_PhaseSheddingLimits_Table +{ + UCHAR ucNumEntries; // Number of entries. + ATOM_PPLIB_PhaseSheddingLimits_Record entries[1]; // Dynamically allocate entries. +}ATOM_PPLIB_PhaseSheddingLimits_Table; + +typedef struct _VCEClockInfo{ + USHORT usEVClkLow; + UCHAR ucEVClkHigh; + USHORT usECClkLow; + UCHAR ucECClkHigh; +}VCEClockInfo; + +typedef struct _VCEClockInfoArray{ + UCHAR ucNumEntries; + VCEClockInfo entries[1]; +}VCEClockInfoArray; + +typedef struct _ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record +{ + USHORT usVoltage; + UCHAR ucVCEClockInfoIndex; +}ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record; + +typedef struct _ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table +{ + UCHAR numEntries; + ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record entries[1]; +}ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table; + +typedef struct _ATOM_PPLIB_VCE_State_Record +{ + UCHAR ucVCEClockInfoIndex; + UCHAR ucClockInfoIndex; //highest 2 bits indicates memory p-states, lower 6bits indicates index to ClockInfoArrary +}ATOM_PPLIB_VCE_State_Record; + +typedef struct _ATOM_PPLIB_VCE_State_Table +{ + UCHAR numEntries; + ATOM_PPLIB_VCE_State_Record entries[1]; +}ATOM_PPLIB_VCE_State_Table; + + +typedef struct _ATOM_PPLIB_VCE_Table +{ + UCHAR revid; +// VCEClockInfoArray array; +// ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table limits; +// ATOM_PPLIB_VCE_State_Table states; +}ATOM_PPLIB_VCE_Table; + + +typedef struct _UVDClockInfo{ + USHORT usVClkLow; + UCHAR ucVClkHigh; + USHORT usDClkLow; + UCHAR ucDClkHigh; +}UVDClockInfo; + +typedef struct _UVDClockInfoArray{ + UCHAR ucNumEntries; + UVDClockInfo entries[1]; +}UVDClockInfoArray; + +typedef struct _ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record +{ + USHORT usVoltage; + UCHAR ucUVDClockInfoIndex; +}ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record; + +typedef struct _ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table +{ + UCHAR numEntries; + ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record entries[1]; +}ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table; + +typedef struct _ATOM_PPLIB_UVD_Table +{ + UCHAR revid; +// UVDClockInfoArray array; +// ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table limits; +}ATOM_PPLIB_UVD_Table; + +typedef struct _ATOM_PPLIB_SAMClk_Voltage_Limit_Record +{ + USHORT usVoltage; + USHORT usSAMClockLow; + UCHAR ucSAMClockHigh; +}ATOM_PPLIB_SAMClk_Voltage_Limit_Record; + +typedef struct _ATOM_PPLIB_SAMClk_Voltage_Limit_Table{ + UCHAR numEntries; + ATOM_PPLIB_SAMClk_Voltage_Limit_Record entries[1]; +}ATOM_PPLIB_SAMClk_Voltage_Limit_Table; + +typedef struct _ATOM_PPLIB_SAMU_Table +{ + UCHAR revid; + ATOM_PPLIB_SAMClk_Voltage_Limit_Table limits; +}ATOM_PPLIB_SAMU_Table; + +typedef struct _ATOM_PPLIB_ACPClk_Voltage_Limit_Record +{ + USHORT usVoltage; + USHORT usACPClockLow; + UCHAR ucACPClockHigh; +}ATOM_PPLIB_ACPClk_Voltage_Limit_Record; + +typedef struct _ATOM_PPLIB_ACPClk_Voltage_Limit_Table{ + UCHAR numEntries; + ATOM_PPLIB_ACPClk_Voltage_Limit_Record entries[1]; +}ATOM_PPLIB_ACPClk_Voltage_Limit_Table; + +typedef struct _ATOM_PPLIB_ACP_Table +{ + UCHAR revid; + ATOM_PPLIB_ACPClk_Voltage_Limit_Table limits; +}ATOM_PPLIB_ACP_Table; + +typedef struct _ATOM_PowerTune_Table{ + USHORT usTDP; + USHORT usConfigurableTDP; + USHORT usTDC; + USHORT usBatteryPowerLimit; + USHORT usSmallPowerLimit; + USHORT usLowCACLeakage; + USHORT usHighCACLeakage; +}ATOM_PowerTune_Table; + +typedef struct _ATOM_PPLIB_POWERTUNE_Table +{ + UCHAR revid; + ATOM_PowerTune_Table power_tune_table; +}ATOM_PPLIB_POWERTUNE_Table; + +typedef struct _ATOM_PPLIB_POWERTUNE_Table_V1 +{ + UCHAR revid; + ATOM_PowerTune_Table power_tune_table; + USHORT usMaximumPowerDeliveryLimit; + USHORT usReserve[7]; +} ATOM_PPLIB_POWERTUNE_Table_V1; + +#define ATOM_PPM_A_A 1 +#define ATOM_PPM_A_I 2 +typedef struct _ATOM_PPLIB_PPM_Table +{ + UCHAR ucRevId; + UCHAR ucPpmDesign; //A+I or A+A + USHORT usCpuCoreNumber; + ULONG ulPlatformTDP; + ULONG ulSmallACPlatformTDP; + ULONG ulPlatformTDC; + ULONG ulSmallACPlatformTDC; + ULONG ulApuTDP; + ULONG ulDGpuTDP; + ULONG ulDGpuUlvPower; + ULONG ulTjmax; +} ATOM_PPLIB_PPM_Table; + +#pragma pack() + +#endif diff --git a/sys/dev/pci/drm/radeon/r100.c b/sys/dev/pci/drm/radeon/r100.c index 9723320499d..9fcac11ad7c 100644 --- a/sys/dev/pci/drm/radeon/r100.c +++ b/sys/dev/pci/drm/radeon/r100.c @@ -1,4 +1,3 @@ -/* $OpenBSD: r100.c,v 1.18 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -41,13 +40,13 @@ #include "rn50_reg_safe.h" /* Firmware Names */ -#define FIRMWARE_R100 "radeon-r100_cp" -#define FIRMWARE_R200 "radeon-r200_cp" -#define FIRMWARE_R300 "radeon-r300_cp" -#define FIRMWARE_R420 "radeon-r420_cp" -#define FIRMWARE_RS690 "radeon-rs690_cp" -#define FIRMWARE_RS600 "radeon-rs600_cp" -#define FIRMWARE_R520 "radeon-r520_cp" +#define FIRMWARE_R100 "radeon/R100_cp.bin" +#define FIRMWARE_R200 "radeon/R200_cp.bin" +#define FIRMWARE_R300 "radeon/R300_cp.bin" +#define FIRMWARE_R420 "radeon/R420_cp.bin" +#define FIRMWARE_RS690 "radeon/RS690_cp.bin" +#define FIRMWARE_RS600 "radeon/RS600_cp.bin" +#define FIRMWARE_R520 "radeon/R520_cp.bin" MODULE_FIRMWARE(FIRMWARE_R100); MODULE_FIRMWARE(FIRMWARE_R200); @@ -137,36 +136,6 @@ void r100_wait_for_vblank(struct radeon_device *rdev, int crtc) } } -/** - * r100_pre_page_flip - pre-pageflip callback. - * - * @rdev: radeon_device pointer - * @crtc: crtc to prepare for pageflip on - * - * Pre-pageflip callback (r1xx-r4xx). - * Enables the pageflip irq (vblank irq). - */ -void r100_pre_page_flip(struct radeon_device *rdev, int crtc) -{ - /* enable the pflip int */ - radeon_irq_kms_pflip_irq_get(rdev, crtc); -} - -/** - * r100_post_page_flip - pos-pageflip callback. - * - * @rdev: radeon_device pointer - * @crtc: crtc to cleanup pageflip on - * - * Post-pageflip callback (r1xx-r4xx). - * Disables the pageflip irq (vblank irq). - */ -void r100_post_page_flip(struct radeon_device *rdev, int crtc) -{ - /* disable the pflip int */ - radeon_irq_kms_pflip_irq_put(rdev, crtc); -} - /** * r100_page_flip - pageflip callback. * @@ -178,9 +147,8 @@ void r100_post_page_flip(struct radeon_device *rdev, int crtc) * During vblank we take the crtc lock and wait for the update_pending * bit to go high, when it does, we release the lock, and allow the * double buffered update to take place. - * Returns the current update pending status. */ -u32 r100_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) +void r100_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; u32 tmp = ((u32)crtc_base) | RADEON_CRTC_OFFSET__OFFSET_LOCK; @@ -202,8 +170,24 @@ u32 r100_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) tmp &= ~RADEON_CRTC_OFFSET__OFFSET_LOCK; WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, tmp); +} + +/** + * r100_page_flip_pending - check if page flip is still pending + * + * @rdev: radeon_device pointer + * @crtc_id: crtc to check + * + * Check if the last pagefilp is still pending (r1xx-r4xx). + * Returns the current update pending status. + */ +bool r100_page_flip_pending(struct radeon_device *rdev, int crtc_id) +{ + struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; + /* Return current update_pending status: */ - return RREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset) & RADEON_CRTC_OFFSET__GUI_TRIG_OFFSET; + return !!(RREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset) & + RADEON_CRTC_OFFSET__GUI_TRIG_OFFSET); } /** @@ -655,6 +639,7 @@ int r100_pci_gart_init(struct radeon_device *rdev) return r; rdev->gart.table_size = rdev->gart.num_gpu_pages * 4; rdev->asic->gart.tlb_flush = &r100_pci_gart_tlb_flush; + rdev->asic->gart.get_page_entry = &r100_pci_gart_get_page_entry; rdev->asic->gart.set_page = &r100_pci_gart_set_page; return radeon_gart_table_ram_alloc(rdev); } @@ -663,7 +648,6 @@ int r100_pci_gart_enable(struct radeon_device *rdev) { uint32_t tmp; - radeon_gart_restore(rdev); /* discard memory request outside of configured range */ tmp = RREG32(RADEON_AIC_CNTL) | RADEON_DIS_OUT_OF_PCI_GART_ACCESS; WREG32(RADEON_AIC_CNTL, tmp); @@ -693,15 +677,16 @@ void r100_pci_gart_disable(struct radeon_device *rdev) WREG32(RADEON_AIC_HI_ADDR, 0); } -int r100_pci_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr) +uint64_t r100_pci_gart_get_page_entry(uint64_t addr, uint32_t flags) { - u32 *gtt = rdev->gart.ptr; + return addr; +} - if (i < 0 || i > rdev->gart.num_gpu_pages) { - return -EINVAL; - } - gtt[i] = cpu_to_le32((u32)addr); - return 0; +void r100_pci_gart_set_page(struct radeon_device *rdev, unsigned i, + uint64_t entry) +{ + u32 *gtt = rdev->gart.ptr; + gtt[i] = cpu_to_le32(lower_32_bits(entry)); } void r100_pci_gart_fini(struct radeon_device *rdev) @@ -738,6 +723,10 @@ int r100_irq_set(struct radeon_device *rdev) tmp |= RADEON_FP2_DETECT_MASK; } WREG32(RADEON_GEN_INT_CNTL, tmp); + + /* read back to post the write */ + RREG32(RADEON_GEN_INT_CNTL); + return 0; } @@ -790,7 +779,7 @@ int r100_irq_process(struct radeon_device *rdev) wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[0])) - radeon_crtc_handle_flip(rdev, 0); + radeon_crtc_handle_vblank(rdev, 0); } if (status & RADEON_CRTC2_VBLANK_STAT) { if (rdev->irq.crtc_vblank_int[1]) { @@ -799,7 +788,7 @@ int r100_irq_process(struct radeon_device *rdev) wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[1])) - radeon_crtc_handle_flip(rdev, 1); + radeon_crtc_handle_vblank(rdev, 1); } if (status & RADEON_FP_DETECT_STAT) { queue_hotplug = true; @@ -812,7 +801,7 @@ int r100_irq_process(struct radeon_device *rdev) status = r100_irq_ack(rdev); } if (queue_hotplug) - task_add(systq, &rdev->hotplug_task); + schedule_delayed_work(&rdev->hotplug_work, 0); if (rdev->msi_enabled) { switch (rdev->family) { case CHIP_RS400: @@ -837,6 +826,20 @@ u32 r100_get_vblank_counter(struct radeon_device *rdev, int crtc) return RREG32(RADEON_CRTC2_CRNT_FRAME); } +/** + * r100_ring_hdp_flush - flush Host Data Path via the ring buffer + * rdev: radeon device structure + * ring: ring buffer struct for emitting packets + */ +static void r100_ring_hdp_flush(struct radeon_device *rdev, struct radeon_ring *ring) +{ + radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0)); + radeon_ring_write(ring, rdev->config.r100.hdp_cntl | + RADEON_HDP_READ_BUFFER_INVALIDATE); + radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0)); + radeon_ring_write(ring, rdev->config.r100.hdp_cntl); +} + /* Who ever call radeon_fence_emit should call ring_lock and ask * for enough space (today caller are ib schedule and buffer move) */ void r100_fence_ring_emit(struct radeon_device *rdev, @@ -853,11 +856,7 @@ void r100_fence_ring_emit(struct radeon_device *rdev, /* Wait until IDLE & CLEAN */ radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0)); radeon_ring_write(ring, RADEON_WAIT_2D_IDLECLEAN | RADEON_WAIT_3D_IDLECLEAN); - radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0)); - radeon_ring_write(ring, rdev->config.r100.hdp_cntl | - RADEON_HDP_READ_BUFFER_INVALIDATE); - radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0)); - radeon_ring_write(ring, rdev->config.r100.hdp_cntl); + r100_ring_hdp_flush(rdev, ring); /* Emit fence sequence & fire IRQ */ radeon_ring_write(ring, PACKET0(rdev->fence_drv[fence->ring].scratch_reg, 0)); radeon_ring_write(ring, fence->seq); @@ -865,22 +864,24 @@ void r100_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(ring, RADEON_SW_INT_FIRE); } -void r100_semaphore_ring_emit(struct radeon_device *rdev, +bool r100_semaphore_ring_emit(struct radeon_device *rdev, struct radeon_ring *ring, struct radeon_semaphore *semaphore, bool emit_wait) { /* Unused on older asics, since we don't have semaphores or multiple rings */ BUG(); + return false; } -int r100_copy_blit(struct radeon_device *rdev, - uint64_t src_offset, - uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence) +struct radeon_fence *r100_copy_blit(struct radeon_device *rdev, + uint64_t src_offset, + uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv) { struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; + struct radeon_fence *fence; uint32_t cur_pages; uint32_t stride_bytes = RADEON_GPU_PAGE_SIZE; uint32_t pitch; @@ -901,7 +902,7 @@ int r100_copy_blit(struct radeon_device *rdev, r = radeon_ring_lock(rdev, ring, ndw); if (r) { DRM_ERROR("radeon: moving bo (%d) asking for %u dw.\n", r, ndw); - return -EINVAL; + return ERR_PTR(-EINVAL); } while (num_gpu_pages > 0) { cur_pages = num_gpu_pages; @@ -941,11 +942,13 @@ int r100_copy_blit(struct radeon_device *rdev, RADEON_WAIT_2D_IDLECLEAN | RADEON_WAIT_HOST_IDLECLEAN | RADEON_WAIT_DMA_GUI_IDLE); - if (fence) { - r = radeon_fence_emit(rdev, fence, RADEON_RING_TYPE_GFX_INDEX); + r = radeon_fence_emit(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + radeon_ring_unlock_undo(rdev, ring); + return ERR_PTR(r); } - radeon_ring_unlock_commit(rdev, ring); - return r; + radeon_ring_unlock_commit(rdev, ring, false); + return fence; } static int r100_cp_wait_for_idle(struct radeon_device *rdev) @@ -977,7 +980,7 @@ void r100_ring_start(struct radeon_device *rdev, struct radeon_ring *ring) RADEON_ISYNC_ANY3D_IDLE2D | RADEON_ISYNC_WAIT_IDLEGUI | RADEON_ISYNC_CPSCRATCH_IDLEGUI); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); } @@ -985,7 +988,7 @@ void r100_ring_start(struct radeon_device *rdev, struct radeon_ring *ring) static int r100_cp_init_microcode(struct radeon_device *rdev) { const char *fw_name = NULL; - int err = 0; + int err; DRM_DEBUG_KMS("\n"); @@ -1030,24 +1033,51 @@ static int r100_cp_init_microcode(struct radeon_device *rdev) fw_name = FIRMWARE_R520; } - err = loadfirmware(fw_name, &rdev->me_fw, &rdev->me_fw_size); + err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); if (err) { - DRM_ERROR("radeon_cp: Failed to load firmware \"%s\"\n", + printk(KERN_ERR "radeon_cp: Failed to load firmware \"%s\"\n", fw_name); - err = -err; - rdev->me_fw = NULL; - } else if (rdev->me_fw_size % 8) { - DRM_ERROR( + } else if (rdev->me_fw->size % 8) { + printk(KERN_ERR "radeon_cp: Bogus length %zu in firmware \"%s\"\n", - rdev->me_fw_size, fw_name); + rdev->me_fw->size, fw_name); err = -EINVAL; - free(rdev->me_fw, M_DEVBUF, 0); + release_firmware(rdev->me_fw); rdev->me_fw = NULL; } - return err; } +u32 r100_gfx_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 rptr; + + if (rdev->wb.enabled) + rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]); + else + rptr = RREG32(RADEON_CP_RB_RPTR); + + return rptr; +} + +u32 r100_gfx_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 wptr; + + wptr = RREG32(RADEON_CP_RB_WPTR); + + return wptr; +} + +void r100_gfx_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + WREG32(RADEON_CP_RB_WPTR, ring->wptr); + (void)RREG32(RADEON_CP_RB_WPTR); +} + static void r100_cp_load_microcode(struct radeon_device *rdev) { const __be32 *fw_data; @@ -1059,8 +1089,8 @@ static void r100_cp_load_microcode(struct radeon_device *rdev) } if (rdev->me_fw) { - size = rdev->me_fw_size / 4; - fw_data = (const __be32 *)&rdev->me_fw[0]; + size = rdev->me_fw->size / 4; + fw_data = (const __be32 *)&rdev->me_fw->data[0]; WREG32(RADEON_CP_ME_RAM_ADDR, 0); for (i = 0; i < size; i += 2) { WREG32(RADEON_CP_ME_RAM_DATAH, @@ -1096,12 +1126,11 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size) } /* Align ring size */ - rb_bufsz = drm_order(ring_size / 8); + rb_bufsz = order_base_2(ring_size / 8); ring_size = (1 << (rb_bufsz + 1)) * 4; r100_cp_load_microcode(rdev); r = radeon_ring_init(rdev, ring, ring_size, RADEON_WB_CP_RPTR_OFFSET, - RADEON_CP_RB_RPTR, RADEON_CP_RB_WPTR, - 0, 0x7fffff, RADEON_CP_PACKET2); + RADEON_CP_PACKET2); if (r) { return r; } @@ -1162,7 +1191,6 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size) WREG32(RADEON_CP_RB_CNTL, tmp); udelay(10); - ring->rptr = RREG32(RADEON_CP_RB_RPTR); /* Set cp mode to bus mastering & enable cp*/ WREG32(RADEON_CP_CSQ_MODE, REG_SET(RADEON_INDIRECT2_START, indirect2_start) | @@ -1172,9 +1200,7 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size) WREG32(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIBM_INDBM); /* at this point everything should be setup correctly to enable master */ -#ifdef notyet pci_set_master(rdev->pdev); -#endif radeon_ring_start(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); @@ -1233,28 +1259,28 @@ int r100_reloc_pitch_offset(struct radeon_cs_parser *p, int r; u32 tile_flags = 0; u32 tmp; - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; u32 value; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } value = radeon_get_ib_value(p, idx); tmp = value & 0x003fffff; - tmp += (((u32)reloc->lobj.gpu_offset) >> 10); + tmp += (((u32)reloc->gpu_offset) >> 10); if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) + if (reloc->tiling_flags & RADEON_TILING_MACRO) tile_flags |= RADEON_DST_TILE_MACRO; - if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) { + if (reloc->tiling_flags & RADEON_TILING_MICRO) { if (reg == RADEON_SRC_PITCH_OFFSET) { DRM_ERROR("Cannot src blit from microtiled surface\n"); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return -EINVAL; } tile_flags |= RADEON_DST_TILE_MICRO; @@ -1272,7 +1298,7 @@ int r100_packet3_load_vbpntr(struct radeon_cs_parser *p, int idx) { unsigned c, i; - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; struct r100_cs_track *track; int r = 0; volatile uint32_t *ib; @@ -1284,46 +1310,46 @@ int r100_packet3_load_vbpntr(struct radeon_cs_parser *p, if (c > 16) { DRM_ERROR("Only 16 vertex buffers are allowed %d\n", pkt->opcode); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return -EINVAL; } track->num_arrays = c; for (i = 0; i < (c - 1); i+=2, idx+=3) { - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for packet3 %d\n", pkt->opcode); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } idx_value = radeon_get_ib_value(p, idx); - ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->lobj.gpu_offset); + ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->gpu_offset); track->arrays[i + 0].esize = idx_value >> 8; track->arrays[i + 0].robj = reloc->robj; track->arrays[i + 0].esize &= 0x7F; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for packet3 %d\n", pkt->opcode); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } - ib[idx+2] = radeon_get_ib_value(p, idx + 2) + ((u32)reloc->lobj.gpu_offset); + ib[idx+2] = radeon_get_ib_value(p, idx + 2) + ((u32)reloc->gpu_offset); track->arrays[i + 1].robj = reloc->robj; track->arrays[i + 1].esize = idx_value >> 24; track->arrays[i + 1].esize &= 0x7F; } if (c & 1) { - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for packet3 %d\n", pkt->opcode); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } idx_value = radeon_get_ib_value(p, idx); - ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->lobj.gpu_offset); + ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->gpu_offset); track->arrays[i + 0].robj = reloc->robj; track->arrays[i + 0].esize = idx_value >> 8; track->arrays[i + 0].esize &= 0x7F; @@ -1376,67 +1402,6 @@ int r100_cs_parse_packet0(struct radeon_cs_parser *p, return 0; } -void r100_cs_dump_packet(struct radeon_cs_parser *p, - struct radeon_cs_packet *pkt) -{ - volatile uint32_t *ib; - unsigned i; - unsigned idx; - - ib = p->ib.ptr; - idx = pkt->idx; - for (i = 0; i <= (pkt->count + 1); i++, idx++) { - DRM_INFO("ib[%d]=0x%08X\n", idx, ib[idx]); - } -} - -/** - * r100_cs_packet_parse() - parse cp packet and point ib index to next packet - * @parser: parser structure holding parsing context. - * @pkt: where to store packet informations - * - * Assume that chunk_ib_index is properly set. Will return -EINVAL - * if packet is bigger than remaining ib size. or if packets is unknown. - **/ -int r100_cs_packet_parse(struct radeon_cs_parser *p, - struct radeon_cs_packet *pkt, - unsigned idx) -{ - struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx]; - uint32_t header; - - if (idx >= ib_chunk->length_dw) { - DRM_ERROR("Can not parse packet at %d after CS end %d !\n", - idx, ib_chunk->length_dw); - return -EINVAL; - } - header = radeon_get_ib_value(p, idx); - pkt->idx = idx; - pkt->type = CP_PACKET_GET_TYPE(header); - pkt->count = CP_PACKET_GET_COUNT(header); - switch (pkt->type) { - case PACKET_TYPE0: - pkt->reg = CP_PACKET0_GET_REG(header); - pkt->one_reg_wr = CP_PACKET0_GET_ONE_REG_WR(header); - break; - case PACKET_TYPE3: - pkt->opcode = CP_PACKET3_GET_OPCODE(header); - break; - case PACKET_TYPE2: - pkt->count = -1; - break; - default: - DRM_ERROR("Unknown packet type %d at %d !\n", pkt->type, idx); - return -EINVAL; - } - if ((pkt->count + 1 + pkt->idx) >= ib_chunk->length_dw) { - DRM_ERROR("Packet (%d:%d:%d) end after CS buffer (%d) !\n", - pkt->idx, pkt->type, pkt->count, ib_chunk->length_dw); - return -EINVAL; - } - return 0; -} - /** * r100_cs_packet_next_vline() - parse userspace VLINE packet * @parser: parser structure holding parsing context. @@ -1453,7 +1418,6 @@ int r100_cs_packet_parse(struct radeon_cs_parser *p, */ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p) { - struct drm_mode_object *obj; struct drm_crtc *crtc; struct radeon_crtc *radeon_crtc; struct radeon_cs_packet p3reloc, waitreloc; @@ -1465,7 +1429,7 @@ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p) ib = p->ib.ptr; /* parse the wait until */ - r = r100_cs_packet_parse(p, &waitreloc, p->idx); + r = radeon_cs_packet_parse(p, &waitreloc, p->idx); if (r) return r; @@ -1482,7 +1446,7 @@ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p) } /* jump over the NOP */ - r = r100_cs_packet_parse(p, &p3reloc, p->idx + waitreloc.count + 2); + r = radeon_cs_packet_parse(p, &p3reloc, p->idx + waitreloc.count + 2); if (r) return r; @@ -1492,13 +1456,12 @@ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p) header = radeon_get_ib_value(p, h_idx); crtc_id = radeon_get_ib_value(p, h_idx + 5); - reg = CP_PACKET0_GET_REG(header); - obj = drm_mode_object_find(p->rdev->ddev, crtc_id, DRM_MODE_OBJECT_CRTC); - if (!obj) { + reg = R100_CP_PACKET0_GET_REG(header); + crtc = drm_crtc_find(p->rdev->ddev, crtc_id); + if (!crtc) { DRM_ERROR("cannot find crtc %d\n", crtc_id); - return -EINVAL; + return -ENOENT; } - crtc = obj_to_crtc(obj); radeon_crtc = to_radeon_crtc(crtc); crtc_id = radeon_crtc->crtc_id; @@ -1527,54 +1490,6 @@ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p) return 0; } -/** - * r100_cs_packet_next_reloc() - parse next packet which should be reloc packet3 - * @parser: parser structure holding parsing context. - * @data: pointer to relocation data - * @offset_start: starting offset - * @offset_mask: offset mask (to align start offset on) - * @reloc: reloc informations - * - * Check next packet is relocation packet3, do bo validation and compute - * GPU offset using the provided start. - **/ -int r100_cs_packet_next_reloc(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc) -{ - struct radeon_cs_chunk *relocs_chunk; - struct radeon_cs_packet p3reloc; - unsigned idx; - int r; - - if (p->chunk_relocs_idx == -1) { - DRM_ERROR("No relocation chunk !\n"); - return -EINVAL; - } - *cs_reloc = NULL; - relocs_chunk = &p->chunks[p->chunk_relocs_idx]; - r = r100_cs_packet_parse(p, &p3reloc, p->idx); - if (r) { - return r; - } - p->idx += p3reloc.count + 2; - if (p3reloc.type != PACKET_TYPE3 || p3reloc.opcode != PACKET3_NOP) { - DRM_ERROR("No packet3 for relocation for packet at %d.\n", - p3reloc.idx); - r100_cs_dump_packet(p, &p3reloc); - return -EINVAL; - } - idx = radeon_get_ib_value(p, p3reloc.idx + 1); - if (idx >= relocs_chunk->length_dw) { - DRM_ERROR("Relocs at %d after relocations chunk end %d !\n", - idx, relocs_chunk->length_dw); - r100_cs_dump_packet(p, &p3reloc); - return -EINVAL; - } - /* FIXME: we assume reloc size is 4 dwords */ - *cs_reloc = p->relocs_ptr[(idx / 4)]; - return 0; -} - static int r100_get_vtx_size(uint32_t vtx_fmt) { int vtx_size; @@ -1632,7 +1547,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p, struct radeon_cs_packet *pkt, unsigned idx, unsigned reg) { - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; struct r100_cs_track *track; volatile uint32_t *ib; uint32_t tmp; @@ -1652,7 +1567,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p, if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } break; @@ -1665,53 +1580,53 @@ static int r100_packet0_check(struct radeon_cs_parser *p, return r; break; case RADEON_RB3D_DEPTHOFFSET: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->zb.robj = reloc->robj; track->zb.offset = idx_value; track->zb_dirty = true; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case RADEON_RB3D_COLOROFFSET: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->cb[0].robj = reloc->robj; track->cb[0].offset = idx_value; track->cb_dirty = true; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case RADEON_PP_TXOFFSET_0: case RADEON_PP_TXOFFSET_1: case RADEON_PP_TXOFFSET_2: i = (reg - RADEON_PP_TXOFFSET_0) / 24; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) + if (reloc->tiling_flags & RADEON_TILING_MACRO) tile_flags |= RADEON_TXO_MACRO_TILE; - if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) + if (reloc->tiling_flags & RADEON_TILING_MICRO) tile_flags |= RADEON_TXO_MICRO_TILE_X2; tmp = idx_value & ~(0x7 << 2); tmp |= tile_flags; - ib[idx] = tmp + ((u32)reloc->lobj.gpu_offset); + ib[idx] = tmp + ((u32)reloc->gpu_offset); } else - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); track->textures[i].robj = reloc->robj; track->tex_dirty = true; break; @@ -1721,15 +1636,15 @@ static int r100_packet0_check(struct radeon_cs_parser *p, case RADEON_PP_CUBIC_OFFSET_T0_3: case RADEON_PP_CUBIC_OFFSET_T0_4: i = (reg - RADEON_PP_CUBIC_OFFSET_T0_0) / 4; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->textures[0].cube_info[i].offset = idx_value; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); track->textures[0].cube_info[i].robj = reloc->robj; track->tex_dirty = true; break; @@ -1739,15 +1654,15 @@ static int r100_packet0_check(struct radeon_cs_parser *p, case RADEON_PP_CUBIC_OFFSET_T1_3: case RADEON_PP_CUBIC_OFFSET_T1_4: i = (reg - RADEON_PP_CUBIC_OFFSET_T1_0) / 4; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->textures[1].cube_info[i].offset = idx_value; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); track->textures[1].cube_info[i].robj = reloc->robj; track->tex_dirty = true; break; @@ -1757,15 +1672,15 @@ static int r100_packet0_check(struct radeon_cs_parser *p, case RADEON_PP_CUBIC_OFFSET_T2_3: case RADEON_PP_CUBIC_OFFSET_T2_4: i = (reg - RADEON_PP_CUBIC_OFFSET_T2_0) / 4; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->textures[2].cube_info[i].offset = idx_value; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); track->textures[2].cube_info[i].robj = reloc->robj; track->tex_dirty = true; break; @@ -1775,17 +1690,17 @@ static int r100_packet0_check(struct radeon_cs_parser *p, track->zb_dirty = true; break; case RADEON_RB3D_COLORPITCH: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) + if (reloc->tiling_flags & RADEON_TILING_MACRO) tile_flags |= RADEON_COLOR_TILE_ENABLE; - if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) + if (reloc->tiling_flags & RADEON_TILING_MICRO) tile_flags |= RADEON_COLOR_MICROTILE_ENABLE; tmp = idx_value & ~(0x7 << 16); @@ -1846,14 +1761,14 @@ static int r100_packet0_check(struct radeon_cs_parser *p, track->zb_dirty = true; break; case RADEON_RB3D_ZPASS_ADDR: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case RADEON_PP_CNTL: { @@ -1991,7 +1906,7 @@ int r100_cs_track_check_pkt3_indx_buffer(struct radeon_cs_parser *p, static int r100_packet3_check(struct radeon_cs_parser *p, struct radeon_cs_packet *pkt) { - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; struct r100_cs_track *track; unsigned idx; volatile uint32_t *ib; @@ -2007,13 +1922,13 @@ static int r100_packet3_check(struct radeon_cs_parser *p, return r; break; case PACKET3_INDX_BUFFER: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for packet3 %d\n", pkt->opcode); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } - ib[idx+1] = radeon_get_ib_value(p, idx+1) + ((u32)reloc->lobj.gpu_offset); + ib[idx+1] = radeon_get_ib_value(p, idx+1) + ((u32)reloc->gpu_offset); r = r100_cs_track_check_pkt3_indx_buffer(p, pkt, reloc->robj); if (r) { return r; @@ -2021,13 +1936,13 @@ static int r100_packet3_check(struct radeon_cs_parser *p, break; case 0x23: /* 3D_RNDR_GEN_INDX_PRIM on r100/r200 */ - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for packet3 %d\n", pkt->opcode); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } - ib[idx] = radeon_get_ib_value(p, idx) + ((u32)reloc->lobj.gpu_offset); + ib[idx] = radeon_get_ib_value(p, idx) + ((u32)reloc->gpu_offset); track->num_arrays = 1; track->vtx_size = r100_get_vtx_size(radeon_get_ib_value(p, idx + 2)); @@ -2121,38 +2036,37 @@ int r100_cs_parse(struct radeon_cs_parser *p) r100_cs_track_clear(p->rdev, track); p->track = track; do { - r = r100_cs_packet_parse(p, &pkt, p->idx); + r = radeon_cs_packet_parse(p, &pkt, p->idx); if (r) { return r; } p->idx += pkt.count + 2; switch (pkt.type) { - case PACKET_TYPE0: - if (p->rdev->family >= CHIP_R200) - r = r100_cs_parse_packet0(p, &pkt, - p->rdev->config.r100.reg_safe_bm, - p->rdev->config.r100.reg_safe_bm_size, - &r200_packet0_check); - else - r = r100_cs_parse_packet0(p, &pkt, - p->rdev->config.r100.reg_safe_bm, - p->rdev->config.r100.reg_safe_bm_size, - &r100_packet0_check); - break; - case PACKET_TYPE2: - break; - case PACKET_TYPE3: - r = r100_packet3_check(p, &pkt); - break; - default: - DRM_ERROR("Unknown packet type %d !\n", - pkt.type); - return -EINVAL; + case RADEON_PACKET_TYPE0: + if (p->rdev->family >= CHIP_R200) + r = r100_cs_parse_packet0(p, &pkt, + p->rdev->config.r100.reg_safe_bm, + p->rdev->config.r100.reg_safe_bm_size, + &r200_packet0_check); + else + r = r100_cs_parse_packet0(p, &pkt, + p->rdev->config.r100.reg_safe_bm, + p->rdev->config.r100.reg_safe_bm_size, + &r100_packet0_check); + break; + case RADEON_PACKET_TYPE2: + break; + case RADEON_PACKET_TYPE3: + r = r100_packet3_check(p, &pkt); + break; + default: + DRM_ERROR("Unknown packet type %d !\n", + pkt.type); + return -EINVAL; } - if (r) { + if (r) return r; - } - } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw); + } while (p->idx < p->chunk_ib->length_dw); return 0; } @@ -2604,11 +2518,9 @@ bool r100_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) rbbm_status = RREG32(R_000E40_RBBM_STATUS); if (!G_000E40_GUI_ACTIVE(rbbm_status)) { - radeon_ring_lockup_update(ring); + radeon_ring_lockup_update(rdev, ring); return false; } - /* force CP activities */ - radeon_ring_force_activity(rdev, ring); return radeon_ring_test_lockup(rdev, ring); } @@ -2634,9 +2546,7 @@ void r100_bm_disable(struct radeon_device *rdev) WREG32(R_000030_BUS_CNTL, (tmp & 0xFFFFFFFF) | 0x00000040); tmp = RREG32(RADEON_BUS_CNTL); mdelay(1); -#ifdef notyet pci_clear_master(rdev->pdev); -#endif mdelay(1); } @@ -2661,9 +2571,7 @@ int r100_asic_reset(struct radeon_device *rdev) WREG32(RADEON_CP_RB_WPTR, 0); WREG32(RADEON_CP_RB_CNTL, tmp); /* save PCI state */ -#ifdef notyet pci_save_state(rdev->pdev); -#endif /* disable bus mastering */ r100_bm_disable(rdev); WREG32(R_0000F0_RBBM_SOFT_RESET, S_0000F0_SOFT_RESET_SE(1) | @@ -2685,9 +2593,7 @@ int r100_asic_reset(struct radeon_device *rdev) status = RREG32(R_000E40_RBBM_STATUS); dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status); /* restore PCI & busmastering */ -#ifdef notyet pci_restore_state(rdev->pdev); -#endif r100_enable_bm(rdev); /* Check if GPU is idle */ if (G_000E40_SE_BUSY(status) || G_000E40_RE_BUSY(status) || @@ -2825,7 +2731,7 @@ static void r100_vram_get_type(struct radeon_device *rdev) static u32 r100_get_accessible_vram(struct radeon_device *rdev) { u32 aper_size; - pcireg_t reg; + u8 byte; aper_size = RREG32(RADEON_CONFIG_APER_SIZE); @@ -2844,8 +2750,8 @@ static u32 r100_get_accessible_vram(struct radeon_device *rdev) * check if it's a multifunction card by reading the PCI config * header type... Limit those to one aperture size */ - reg = pci_conf_read(rdev->pc, rdev->pa_tag, PCI_BHLC_REG); - if (PCI_HDRTYPE_MULTIFN(reg)) { + pci_read_config_byte(rdev->pdev, 0xe, &byte); + if (byte & 0x80) { DRM_INFO("Generation 1 PCI interface in multifunction mode\n"); DRM_INFO("Limiting VRAM to one aperture\n"); return aper_size; @@ -2970,21 +2876,28 @@ static void r100_pll_errata_after_data(struct radeon_device *rdev) uint32_t r100_pll_rreg(struct radeon_device *rdev, uint32_t reg) { + unsigned long flags; uint32_t data; + spin_lock_irqsave(&rdev->pll_idx_lock, flags); WREG8(RADEON_CLOCK_CNTL_INDEX, reg & 0x3f); r100_pll_errata_after_index(rdev); data = RREG32(RADEON_CLOCK_CNTL_DATA); r100_pll_errata_after_data(rdev); + spin_unlock_irqrestore(&rdev->pll_idx_lock, flags); return data; } void r100_pll_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v) { + unsigned long flags; + + spin_lock_irqsave(&rdev->pll_idx_lock, flags); WREG8(RADEON_CLOCK_CNTL_INDEX, ((reg & 0x3f) | RADEON_PLL_WR_EN)); r100_pll_errata_after_index(rdev); WREG32(RADEON_CLOCK_CNTL_DATA, v); r100_pll_errata_after_data(rdev); + spin_unlock_irqrestore(&rdev->pll_idx_lock, flags); } static void r100_set_safe_registers(struct radeon_device *rdev) @@ -3299,6 +3212,12 @@ void r100_bandwidth_update(struct radeon_device *rdev) uint32_t pixel_bytes1 = 0; uint32_t pixel_bytes2 = 0; + /* Guess line buffer size to be 8192 pixels */ + u32 lb_size = 8192; + + if (!rdev->mode_info.mode_config_initialized) + return; + radeon_update_display_priority(rdev); if (rdev->mode_info.crtcs[0]->base.enabled) { @@ -3710,6 +3629,13 @@ void r100_bandwidth_update(struct radeon_device *rdev) DRM_DEBUG_KMS("GRPH2_BUFFER_CNTL from to %x\n", (unsigned int)RREG32(RADEON_GRPH2_BUFFER_CNTL)); } + + /* Save number of lines the linebuffer leads before the scanout */ + if (mode1) + rdev->mode_info.crtcs[0]->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode1->crtc_hdisplay); + + if (mode2) + rdev->mode_info.crtcs[1]->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode2->crtc_hdisplay); } int r100_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) @@ -3733,7 +3659,7 @@ int r100_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) } radeon_ring_write(ring, PACKET0(scratch, 0)); radeon_ring_write(ring, 0xDEADBEEF); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); for (i = 0; i < rdev->usec_timeout; i++) { tmp = RREG32(scratch); if (tmp == 0xDEADBEEF) { @@ -3795,7 +3721,7 @@ int r100_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) ib.ptr[6] = PACKET2(0); ib.ptr[7] = PACKET2(0); ib.length_dw = 8; - r = radeon_ib_schedule(rdev, &ib, NULL); + r = radeon_ib_schedule(rdev, &ib, NULL, false); if (r) { DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); goto free_ib; @@ -3911,7 +3837,7 @@ static void r100_mc_program(struct radeon_device *rdev) WREG32(R_00014C_MC_AGP_LOCATION, S_00014C_MC_AGP_START(rdev->mc.gtt_start >> 16) | S_00014C_MC_AGP_TOP(rdev->mc.gtt_end >> 16)); - WREG32(R_000170_AGP_BASE, (u32)(rdev->mc.agp_base)); + WREG32(R_000170_AGP_BASE, lower_32_bits(rdev->mc.agp_base)); if (rdev->family > CHIP_RV200) WREG32(R_00015C_AGP_BASE_2, upper_32_bits(rdev->mc.agp_base) & 0xff); @@ -4032,6 +3958,7 @@ int r100_resume(struct radeon_device *rdev) int r100_suspend(struct radeon_device *rdev) { + radeon_pm_suspend(rdev); r100_cp_disable(rdev); radeon_wb_disable(rdev); r100_irq_disable(rdev); @@ -4042,6 +3969,7 @@ int r100_suspend(struct radeon_device *rdev) void r100_fini(struct radeon_device *rdev) { + radeon_pm_fini(rdev); r100_cp_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); @@ -4148,6 +4076,9 @@ int r100_init(struct radeon_device *rdev) } r100_set_safe_registers(rdev); + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->accel_working = true; r = r100_startup(rdev); if (r) { @@ -4164,42 +4095,30 @@ int r100_init(struct radeon_device *rdev) return 0; } -uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg, - bool always_indirect) +uint32_t r100_mm_rreg_slow(struct radeon_device *rdev, uint32_t reg) { - if (reg < rdev->rmmio_size && !always_indirect) - return bus_space_read_4(rdev->memt, rdev->rmmio, reg); - - else { - unsigned long flags; - uint32_t ret; - - spin_lock_irqsave(&rdev->mmio_idx_lock, flags); - bus_space_write_4(rdev->memt, rdev->rmmio, - RADEON_MM_INDEX, reg); - ret = bus_space_read_4(rdev->memt, rdev->rmmio, - RADEON_MM_DATA); - spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); - - return ret; - } + unsigned long flags; + uint32_t ret; + + spin_lock_irqsave(&rdev->mmio_idx_lock, flags); + bus_space_write_4(rdev->memt, rdev->rmmio_bsh, + RADEON_MM_INDEX, reg); + ret = bus_space_read_4(rdev->memt, rdev->rmmio_bsh, + RADEON_MM_DATA); + spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); + return ret; } -void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v, - bool always_indirect) +void r100_mm_wreg_slow(struct radeon_device *rdev, uint32_t reg, uint32_t v) { - if (reg < rdev->rmmio_size && !always_indirect) - bus_space_write_4(rdev->memt, rdev->rmmio, reg, v); - else { - unsigned long flags; - - spin_lock_irqsave(&rdev->mmio_idx_lock, flags); - bus_space_write_4(rdev->memt, rdev->rmmio, - RADEON_MM_INDEX, reg); - bus_space_write_4(rdev->memt, rdev->rmmio, - RADEON_MM_DATA, v); - spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); - } + unsigned long flags; + + spin_lock_irqsave(&rdev->mmio_idx_lock, flags); + bus_space_write_4(rdev->memt, rdev->rmmio_bsh, + RADEON_MM_INDEX, reg); + bus_space_write_4(rdev->memt, rdev->rmmio_bsh, + RADEON_MM_DATA, v); + spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); } u32 r100_io_rreg(struct radeon_device *rdev, u32 reg) diff --git a/sys/dev/pci/drm/radeon/r100_track.h b/sys/dev/pci/drm/radeon/r100_track.h index 5d0ae292c6a..eb40888bdfc 100644 --- a/sys/dev/pci/drm/radeon/r100_track.h +++ b/sys/dev/pci/drm/radeon/r100_track.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r100_track.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ #define R100_TRACK_MAX_TEXTURE 3 #define R200_TRACK_MAX_TEXTURE 6 @@ -82,10 +81,6 @@ struct r100_cs_track { int r100_cs_track_check(struct radeon_device *rdev, struct r100_cs_track *track); void r100_cs_track_clear(struct radeon_device *rdev, struct r100_cs_track *track); -int r100_cs_packet_next_reloc(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc); -void r100_cs_dump_packet(struct radeon_cs_parser *p, - struct radeon_cs_packet *pkt); int r100_cs_packet_parse_vline(struct radeon_cs_parser *p); diff --git a/sys/dev/pci/drm/radeon/r100d.h b/sys/dev/pci/drm/radeon/r100d.h index 31a783f1e75..f0f8ee69f48 100644 --- a/sys/dev/pci/drm/radeon/r100d.h +++ b/sys/dev/pci/drm/radeon/r100d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r100d.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -65,17 +64,6 @@ REG_SET(PACKET3_IT_OPCODE, (op)) | \ REG_SET(PACKET3_COUNT, (n))) -#define PACKET_TYPE0 0 -#define PACKET_TYPE1 1 -#define PACKET_TYPE2 2 -#define PACKET_TYPE3 3 - -#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) -#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) -#define CP_PACKET0_GET_REG(h) (((h) & 0x1FFF) << 2) -#define CP_PACKET0_GET_ONE_REG_WR(h) (((h) >> 15) & 1) -#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) - /* Registers */ #define R_0000F0_RBBM_SOFT_RESET 0x0000F0 #define S_0000F0_SOFT_RESET_CP(x) (((x) & 0x1) << 0) diff --git a/sys/dev/pci/drm/radeon/r200.c b/sys/dev/pci/drm/radeon/r200.c index ea04034724b..8267fd0f4be 100644 --- a/sys/dev/pci/drm/radeon/r200.c +++ b/sys/dev/pci/drm/radeon/r200.c @@ -1,4 +1,3 @@ -/* $OpenBSD: r200.c,v 1.5 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -81,13 +80,14 @@ static int r200_get_vtx_size_0(uint32_t vtx_fmt_0) return vtx_size; } -int r200_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, - uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence) +struct radeon_fence *r200_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, + uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv) { struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; + struct radeon_fence *fence; uint32_t size; uint32_t cur_size; int i, num_loops; @@ -99,7 +99,7 @@ int r200_copy_dma(struct radeon_device *rdev, r = radeon_ring_lock(rdev, ring, num_loops * 4 + 64); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); - return r; + return ERR_PTR(r); } /* Must wait for 2D idle & clean before DMA or hangs might happen */ radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0)); @@ -119,11 +119,13 @@ int r200_copy_dma(struct radeon_device *rdev, } radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0)); radeon_ring_write(ring, RADEON_WAIT_DMA_GUI_IDLE); - if (fence) { - r = radeon_fence_emit(rdev, fence, RADEON_RING_TYPE_GFX_INDEX); + r = radeon_fence_emit(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + radeon_ring_unlock_undo(rdev, ring); + return ERR_PTR(r); } - radeon_ring_unlock_commit(rdev, ring); - return r; + radeon_ring_unlock_commit(rdev, ring, false); + return fence; } @@ -144,7 +146,7 @@ int r200_packet0_check(struct radeon_cs_parser *p, struct radeon_cs_packet *pkt, unsigned idx, unsigned reg) { - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; struct r100_cs_track *track; volatile uint32_t *ib; uint32_t tmp; @@ -163,7 +165,7 @@ int r200_packet0_check(struct radeon_cs_parser *p, if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } break; @@ -176,30 +178,30 @@ int r200_packet0_check(struct radeon_cs_parser *p, return r; break; case RADEON_RB3D_DEPTHOFFSET: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->zb.robj = reloc->robj; track->zb.offset = idx_value; track->zb_dirty = true; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case RADEON_RB3D_COLOROFFSET: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->cb[0].robj = reloc->robj; track->cb[0].offset = idx_value; track->cb_dirty = true; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case R200_PP_TXOFFSET_0: case R200_PP_TXOFFSET_1: @@ -208,24 +210,24 @@ int r200_packet0_check(struct radeon_cs_parser *p, case R200_PP_TXOFFSET_4: case R200_PP_TXOFFSET_5: i = (reg - R200_PP_TXOFFSET_0) / 24; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) + if (reloc->tiling_flags & RADEON_TILING_MACRO) tile_flags |= R200_TXO_MACRO_TILE; - if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) + if (reloc->tiling_flags & RADEON_TILING_MICRO) tile_flags |= R200_TXO_MICRO_TILE; tmp = idx_value & ~(0x7 << 2); tmp |= tile_flags; - ib[idx] = tmp + ((u32)reloc->lobj.gpu_offset); + ib[idx] = tmp + ((u32)reloc->gpu_offset); } else - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); track->textures[i].robj = reloc->robj; track->tex_dirty = true; break; @@ -261,15 +263,15 @@ int r200_packet0_check(struct radeon_cs_parser *p, case R200_PP_CUBIC_OFFSET_F5_5: i = (reg - R200_PP_TXOFFSET_0) / 24; face = (reg - ((i * 24) + R200_PP_TXOFFSET_0)) / 4; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->textures[i].cube_info[face - 1].offset = idx_value; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); track->textures[i].cube_info[face - 1].robj = reloc->robj; track->tex_dirty = true; break; @@ -279,18 +281,18 @@ int r200_packet0_check(struct radeon_cs_parser *p, track->zb_dirty = true; break; case RADEON_RB3D_COLORPITCH: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) + if (reloc->tiling_flags & RADEON_TILING_MACRO) tile_flags |= RADEON_COLOR_TILE_ENABLE; - if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) + if (reloc->tiling_flags & RADEON_TILING_MICRO) tile_flags |= RADEON_COLOR_MICROTILE_ENABLE; tmp = idx_value & ~(0x7 << 16); @@ -356,14 +358,14 @@ int r200_packet0_check(struct radeon_cs_parser *p, track->zb_dirty = true; break; case RADEON_RB3D_ZPASS_ADDR: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case RADEON_PP_CNTL: { diff --git a/sys/dev/pci/drm/radeon/r300.c b/sys/dev/pci/drm/radeon/r300.c index b34f87e7141..4ebb653cf65 100644 --- a/sys/dev/pci/drm/radeon/r300.c +++ b/sys/dev/pci/drm/radeon/r300.c @@ -1,4 +1,3 @@ -/* $OpenBSD: r300.c,v 1.9 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -48,6 +47,31 @@ * tell. (Jerome Glisse) */ +/* + * Indirect registers accessor + */ +uint32_t rv370_pcie_rreg(struct radeon_device *rdev, uint32_t reg) +{ + unsigned long flags; + uint32_t r; + + spin_lock_irqsave(&rdev->pcie_idx_lock, flags); + WREG32(RADEON_PCIE_INDEX, ((reg) & rdev->pcie_reg_mask)); + r = RREG32(RADEON_PCIE_DATA); + spin_unlock_irqrestore(&rdev->pcie_idx_lock, flags); + return r; +} + +void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->pcie_idx_lock, flags); + WREG32(RADEON_PCIE_INDEX, ((reg) & rdev->pcie_reg_mask)); + WREG32(RADEON_PCIE_DATA, (v)); + spin_unlock_irqrestore(&rdev->pcie_idx_lock, flags); +} + /* * rv370,rv380 PCIE GART */ @@ -68,25 +92,32 @@ void rv370_pcie_gart_tlb_flush(struct radeon_device *rdev) mb(); } +#define R300_PTE_UNSNOOPED (1 << 0) #define R300_PTE_WRITEABLE (1 << 2) #define R300_PTE_READABLE (1 << 3) -int rv370_pcie_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr) +uint64_t rv370_pcie_gart_get_page_entry(uint64_t addr, uint32_t flags) { - volatile uint32_t *ptr = rdev->gart.ptr; - - if (i < 0 || i > rdev->gart.num_gpu_pages) { - return -EINVAL; - } addr = (lower_32_bits(addr) >> 8) | - ((upper_32_bits(addr) & 0xff) << 24) | - R300_PTE_WRITEABLE | R300_PTE_READABLE; + ((upper_32_bits(addr) & 0xff) << 24); + if (flags & RADEON_GART_PAGE_READ) + addr |= R300_PTE_READABLE; + if (flags & RADEON_GART_PAGE_WRITE) + addr |= R300_PTE_WRITEABLE; + if (!(flags & RADEON_GART_PAGE_SNOOP)) + addr |= R300_PTE_UNSNOOPED; + return addr; +} + +void rv370_pcie_gart_set_page(struct radeon_device *rdev, unsigned i, + uint64_t entry) +{ + void __iomem *ptr = rdev->gart.ptr; + /* on x86 we want this to be CPU endian, on powerpc * on powerpc without HW swappers, it'll get swapped on way * into VRAM - so no need for cpu_to_le32 on VRAM tables */ - ptr += i; - *ptr = (uint32_t)addr; - return 0; + writel(entry, ((void __iomem *)ptr) + (i * 4)); } int rv370_pcie_gart_init(struct radeon_device *rdev) @@ -106,6 +137,7 @@ int rv370_pcie_gart_init(struct radeon_device *rdev) DRM_ERROR("Failed to register debugfs file for PCIE gart !\n"); rdev->gart.table_size = rdev->gart.num_gpu_pages * 4; rdev->asic->gart.tlb_flush = &rv370_pcie_gart_tlb_flush; + rdev->asic->gart.get_page_entry = &rv370_pcie_gart_get_page_entry; rdev->asic->gart.set_page = &rv370_pcie_gart_set_page; return radeon_gart_table_vram_alloc(rdev); } @@ -123,7 +155,6 @@ int rv370_pcie_gart_enable(struct radeon_device *rdev) r = radeon_gart_table_vram_pin(rdev); if (r) return r; - radeon_gart_restore(rdev); /* discard memory request outside of configured range */ tmp = RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD; WREG32_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp); @@ -293,7 +324,7 @@ void r300_ring_start(struct radeon_device *rdev, struct radeon_ring *ring) radeon_ring_write(ring, R300_GEOMETRY_ROUND_NEAREST | R300_COLOR_ROUND_NEAREST); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); } static void r300_errata(struct radeon_device *rdev) @@ -398,9 +429,7 @@ int r300_asic_reset(struct radeon_device *rdev) WREG32(RADEON_CP_RB_WPTR, 0); WREG32(RADEON_CP_RB_CNTL, tmp); /* save PCI state */ -#ifdef notyet pci_save_state(rdev->pdev); -#endif /* disable bus mastering */ r100_bm_disable(rdev); WREG32(R_0000F0_RBBM_SOFT_RESET, S_0000F0_SOFT_RESET_VAP(1) | @@ -424,9 +453,7 @@ int r300_asic_reset(struct radeon_device *rdev) status = RREG32(R_000E40_RBBM_STATUS); dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status); /* restore PCI & busmastering */ -#ifdef notyet pci_restore_state(rdev->pdev); -#endif r100_enable_bm(rdev); /* Check if GPU is idle */ if (G_000E40_GA_BUSY(status) || G_000E40_VAP_BUSY(status)) { @@ -600,7 +627,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p, struct radeon_cs_packet *pkt, unsigned idx, unsigned reg) { - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; struct r100_cs_track *track; volatile uint32_t *ib; uint32_t tmp, tile_flags = 0; @@ -619,7 +646,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p, if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } break; @@ -634,30 +661,30 @@ static int r300_packet0_check(struct radeon_cs_parser *p, case R300_RB3D_COLOROFFSET2: case R300_RB3D_COLOROFFSET3: i = (reg - R300_RB3D_COLOROFFSET0) >> 2; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->cb[i].robj = reloc->robj; track->cb[i].offset = idx_value; track->cb_dirty = true; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case R300_ZB_DEPTHOFFSET: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->zb.robj = reloc->robj; track->zb.offset = idx_value; track->zb_dirty = true; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case R300_TX_OFFSET_0: case R300_TX_OFFSET_0+4: @@ -676,26 +703,26 @@ static int r300_packet0_check(struct radeon_cs_parser *p, case R300_TX_OFFSET_0+56: case R300_TX_OFFSET_0+60: i = (reg - R300_TX_OFFSET_0) >> 2; - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } if (p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS) { ib[idx] = (idx_value & 31) | /* keep the 1st 5 bits */ - ((idx_value & ~31) + (u32)reloc->lobj.gpu_offset); + ((idx_value & ~31) + (u32)reloc->gpu_offset); } else { - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) + if (reloc->tiling_flags & RADEON_TILING_MACRO) tile_flags |= R300_TXO_MACRO_TILE; - if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) + if (reloc->tiling_flags & RADEON_TILING_MICRO) tile_flags |= R300_TXO_MICRO_TILE; - else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO_SQUARE) + else if (reloc->tiling_flags & RADEON_TILING_MICRO_SQUARE) tile_flags |= R300_TXO_MICRO_TILE_SQUARE; - tmp = idx_value + ((u32)reloc->lobj.gpu_offset); + tmp = idx_value + ((u32)reloc->gpu_offset); tmp |= tile_flags; ib[idx] = tmp; } @@ -749,19 +776,19 @@ static int r300_packet0_check(struct radeon_cs_parser *p, /* RB3D_COLORPITCH2 */ /* RB3D_COLORPITCH3 */ if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) + if (reloc->tiling_flags & RADEON_TILING_MACRO) tile_flags |= R300_COLOR_TILE_ENABLE; - if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) + if (reloc->tiling_flags & RADEON_TILING_MICRO) tile_flags |= R300_COLOR_MICROTILE_ENABLE; - else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO_SQUARE) + else if (reloc->tiling_flags & RADEON_TILING_MICRO_SQUARE) tile_flags |= R300_COLOR_MICROTILE_SQUARE_ENABLE; tmp = idx_value & ~(0x7 << 16); @@ -834,19 +861,19 @@ static int r300_packet0_check(struct radeon_cs_parser *p, case 0x4F24: /* ZB_DEPTHPITCH */ if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) + if (reloc->tiling_flags & RADEON_TILING_MACRO) tile_flags |= R300_DEPTHMACROTILE_ENABLE; - if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) + if (reloc->tiling_flags & RADEON_TILING_MICRO) tile_flags |= R300_DEPTHMICROTILE_TILED; - else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO_SQUARE) + else if (reloc->tiling_flags & RADEON_TILING_MICRO_SQUARE) tile_flags |= R300_DEPTHMICROTILE_TILED_SQUARE; tmp = idx_value & ~(0x7 << 16); @@ -1049,14 +1076,14 @@ static int r300_packet0_check(struct radeon_cs_parser *p, track->tex_dirty = true; break; case R300_ZB_ZPASS_ADDR: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case 0x4e0c: /* RB3D_COLOR_CHANNEL_MASK */ @@ -1091,17 +1118,17 @@ static int r300_packet0_check(struct radeon_cs_parser *p, track->cb_dirty = true; break; case R300_RB3D_AARESOLVE_OFFSET: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for ib[%d]=0x%04X\n", idx, reg); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } track->aa.robj = reloc->robj; track->aa.offset = idx_value; track->aa_dirty = true; - ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset); + ib[idx] = idx_value + ((u32)reloc->gpu_offset); break; case R300_RB3D_AARESOLVE_PITCH: track->aa.pitch = idx_value & 0x3FFE; @@ -1144,7 +1171,7 @@ fail: static int r300_packet3_check(struct radeon_cs_parser *p, struct radeon_cs_packet *pkt) { - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; struct r100_cs_track *track; volatile uint32_t *ib; unsigned idx; @@ -1160,13 +1187,13 @@ static int r300_packet3_check(struct radeon_cs_parser *p, return r; break; case PACKET3_INDX_BUFFER: - r = r100_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { DRM_ERROR("No reloc for packet3 %d\n", pkt->opcode); - r100_cs_dump_packet(p, pkt); + radeon_cs_dump_packet(p, pkt); return r; } - ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->lobj.gpu_offset); + ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->gpu_offset); r = r100_cs_track_check_pkt3_indx_buffer(p, pkt, reloc->robj); if (r) { return r; @@ -1261,21 +1288,21 @@ int r300_cs_parse(struct radeon_cs_parser *p) r100_cs_track_clear(p->rdev, track); p->track = track; do { - r = r100_cs_packet_parse(p, &pkt, p->idx); + r = radeon_cs_packet_parse(p, &pkt, p->idx); if (r) { return r; } p->idx += pkt.count + 2; switch (pkt.type) { - case PACKET_TYPE0: + case RADEON_PACKET_TYPE0: r = r100_cs_parse_packet0(p, &pkt, p->rdev->config.r300.reg_safe_bm, p->rdev->config.r300.reg_safe_bm_size, &r300_packet0_check); break; - case PACKET_TYPE2: + case RADEON_PACKET_TYPE2: break; - case PACKET_TYPE3: + case RADEON_PACKET_TYPE3: r = r300_packet3_check(p, &pkt); break; default: @@ -1285,7 +1312,7 @@ int r300_cs_parse(struct radeon_cs_parser *p) if (r) { return r; } - } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw); + } while (p->idx < p->chunk_ib->length_dw); return 0; } @@ -1444,6 +1471,7 @@ int r300_resume(struct radeon_device *rdev) int r300_suspend(struct radeon_device *rdev) { + radeon_pm_suspend(rdev); r100_cp_disable(rdev); radeon_wb_disable(rdev); r100_irq_disable(rdev); @@ -1456,6 +1484,7 @@ int r300_suspend(struct radeon_device *rdev) void r300_fini(struct radeon_device *rdev) { + radeon_pm_fini(rdev); r100_cp_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); @@ -1542,10 +1571,13 @@ int r300_init(struct radeon_device *rdev) } r300_set_reg_safe(rdev); + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->accel_working = true; r = r300_startup(rdev); if (r) { - /* Somethings want wront with the accel init stop accel */ + /* Something went wrong with the accel init, so stop accel */ dev_err(rdev->dev, "Disabling GPU acceleration\n"); r100_cp_fini(rdev); radeon_wb_fini(rdev); diff --git a/sys/dev/pci/drm/radeon/r300_reg.h b/sys/dev/pci/drm/radeon/r300_reg.h index 9e6bdf9384c..00c0d2ba22d 100644 --- a/sys/dev/pci/drm/radeon/r300_reg.h +++ b/sys/dev/pci/drm/radeon/r300_reg.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r300_reg.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2005 Nicolai Haehnle et al. * Copyright 2008 Advanced Micro Devices, Inc. diff --git a/sys/dev/pci/drm/radeon/r300d.h b/sys/dev/pci/drm/radeon/r300d.h index 0ba74ea4b6c..ff229a00d27 100644 --- a/sys/dev/pci/drm/radeon/r300d.h +++ b/sys/dev/pci/drm/radeon/r300d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r300d.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -66,17 +65,6 @@ REG_SET(PACKET3_IT_OPCODE, (op)) | \ REG_SET(PACKET3_COUNT, (n))) -#define PACKET_TYPE0 0 -#define PACKET_TYPE1 1 -#define PACKET_TYPE2 2 -#define PACKET_TYPE3 3 - -#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) -#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) -#define CP_PACKET0_GET_REG(h) (((h) & 0x1FFF) << 2) -#define CP_PACKET0_GET_ONE_REG_WR(h) (((h) >> 15) & 1) -#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) - /* Registers */ #define R_000148_MC_FB_LOCATION 0x000148 #define S_000148_MC_FB_START(x) (((x) & 0xFFFF) << 0) diff --git a/sys/dev/pci/drm/radeon/r420.c b/sys/dev/pci/drm/radeon/r420.c index 3c231602683..1089aebee66 100644 --- a/sys/dev/pci/drm/radeon/r420.c +++ b/sys/dev/pci/drm/radeon/r420.c @@ -1,4 +1,3 @@ -/* $OpenBSD: r420.c,v 1.7 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -159,18 +158,25 @@ void r420_pipes_init(struct radeon_device *rdev) u32 r420_mc_rreg(struct radeon_device *rdev, u32 reg) { + unsigned long flags; u32 r; + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(R_0001F8_MC_IND_INDEX, S_0001F8_MC_IND_ADDR(reg)); r = RREG32(R_0001FC_MC_IND_DATA); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); return r; } void r420_mc_wreg(struct radeon_device *rdev, u32 reg, u32 v) { + unsigned long flags; + + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(R_0001F8_MC_IND_INDEX, S_0001F8_MC_IND_ADDR(reg) | S_0001F8_MC_IND_WR_EN(1)); WREG32(R_0001FC_MC_IND_DATA, v); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); } static void r420_debugfs(struct radeon_device *rdev) @@ -211,7 +217,7 @@ static void r420_cp_errata_init(struct radeon_device *rdev) radeon_ring_write(ring, PACKET0(R300_CP_RESYNC_ADDR, 1)); radeon_ring_write(ring, rdev->config.r300.resync_scratch); radeon_ring_write(ring, 0xDEADBEEF); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); } static void r420_cp_errata_fini(struct radeon_device *rdev) @@ -224,7 +230,7 @@ static void r420_cp_errata_fini(struct radeon_device *rdev) radeon_ring_lock(rdev, ring, 8); radeon_ring_write(ring, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); radeon_ring_write(ring, R300_RB3D_DC_FINISH); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); radeon_scratch_free(rdev, rdev->config.r300.resync_scratch); } @@ -327,6 +333,7 @@ int r420_resume(struct radeon_device *rdev) int r420_suspend(struct radeon_device *rdev) { + radeon_pm_suspend(rdev); r420_cp_errata_fini(rdev); r100_cp_disable(rdev); radeon_wb_disable(rdev); @@ -340,6 +347,7 @@ int r420_suspend(struct radeon_device *rdev) void r420_fini(struct radeon_device *rdev) { + radeon_pm_fini(rdev); r100_cp_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); @@ -436,6 +444,9 @@ int r420_init(struct radeon_device *rdev) } r420_set_reg_safe(rdev); + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->accel_working = true; r = r420_startup(rdev); if (r) { diff --git a/sys/dev/pci/drm/radeon/r420d.h b/sys/dev/pci/drm/radeon/r420d.h index 74f79636f0d..fc78d31a0b4 100644 --- a/sys/dev/pci/drm/radeon/r420d.h +++ b/sys/dev/pci/drm/radeon/r420d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r420d.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/r500_reg.h b/sys/dev/pci/drm/radeon/r500_reg.h index eead3d85392..136b7bc7cd2 100644 --- a/sys/dev/pci/drm/radeon/r500_reg.h +++ b/sys/dev/pci/drm/radeon/r500_reg.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r500_reg.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -356,6 +355,7 @@ # define AVIVO_D1CRTC_V_BLANK (1 << 0) #define AVIVO_D1CRTC_STATUS_POSITION 0x60a0 #define AVIVO_D1CRTC_FRAME_COUNT 0x60a4 +#define AVIVO_D1CRTC_STATUS_HV_COUNT 0x60ac #define AVIVO_D1CRTC_STEREO_CONTROL 0x60c4 #define AVIVO_D1MODE_MASTER_UPDATE_LOCK 0x60e0 @@ -402,6 +402,7 @@ * block and vice versa. This applies to GRPH, CUR, etc. */ #define AVIVO_D1GRPH_LUT_SEL 0x6108 +# define AVIVO_LUT_10BIT_BYPASS_EN (1 << 8) #define AVIVO_D1GRPH_PRIMARY_SURFACE_ADDRESS 0x6110 #define R700_D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x6914 #define R700_D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x6114 diff --git a/sys/dev/pci/drm/radeon/r520.c b/sys/dev/pci/drm/radeon/r520.c index 9da1f5d3320..3f51ef2028b 100644 --- a/sys/dev/pci/drm/radeon/r520.c +++ b/sys/dev/pci/drm/radeon/r520.c @@ -1,4 +1,3 @@ -/* $OpenBSD: r520.c,v 1.6 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -313,6 +312,9 @@ int r520_init(struct radeon_device *rdev) return r; rv515_set_safe_registers(rdev); + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->accel_working = true; r = r520_startup(rdev); if (r) { diff --git a/sys/dev/pci/drm/radeon/r520d.h b/sys/dev/pci/drm/radeon/r520d.h index d008fdffc47..61af61f644b 100644 --- a/sys/dev/pci/drm/radeon/r520d.h +++ b/sys/dev/pci/drm/radeon/r520d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r520d.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/r600.c b/sys/dev/pci/drm/radeon/r600.c index d64a6fd0bab..05ca3ffbb65 100644 --- a/sys/dev/pci/drm/radeon/r600.c +++ b/sys/dev/pci/drm/radeon/r600.c @@ -1,4 +1,3 @@ -/* $OpenBSD: r600.c,v 1.21 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -30,22 +29,12 @@ #include #include "radeon.h" #include "radeon_asic.h" +#include "radeon_audio.h" #include "radeon_mode.h" #include "r600d.h" #include "atom.h" #include "avivod.h" - -#define PFP_UCODE_SIZE 576 -#define PM4_UCODE_SIZE 1792 -#define RLC_UCODE_SIZE 768 -#define R700_PFP_UCODE_SIZE 848 -#define R700_PM4_UCODE_SIZE 1360 -#define R700_RLC_UCODE_SIZE 1024 -#define EVERGREEN_PFP_UCODE_SIZE 1120 -#define EVERGREEN_PM4_UCODE_SIZE 1376 -#define EVERGREEN_RLC_UCODE_SIZE 768 -#define CAYMAN_RLC_UCODE_SIZE 1024 -#define ARUBA_RLC_UCODE_SIZE 1536 +#include "radeon_ucode.h" /* Firmware Names */ MODULE_FIRMWARE("radeon/R600_pfp.bin"); @@ -64,24 +53,32 @@ MODULE_FIRMWARE("radeon/RS780_pfp.bin"); MODULE_FIRMWARE("radeon/RS780_me.bin"); MODULE_FIRMWARE("radeon/RV770_pfp.bin"); MODULE_FIRMWARE("radeon/RV770_me.bin"); +MODULE_FIRMWARE("radeon/RV770_smc.bin"); MODULE_FIRMWARE("radeon/RV730_pfp.bin"); MODULE_FIRMWARE("radeon/RV730_me.bin"); +MODULE_FIRMWARE("radeon/RV730_smc.bin"); +MODULE_FIRMWARE("radeon/RV740_smc.bin"); MODULE_FIRMWARE("radeon/RV710_pfp.bin"); MODULE_FIRMWARE("radeon/RV710_me.bin"); +MODULE_FIRMWARE("radeon/RV710_smc.bin"); MODULE_FIRMWARE("radeon/R600_rlc.bin"); MODULE_FIRMWARE("radeon/R700_rlc.bin"); MODULE_FIRMWARE("radeon/CEDAR_pfp.bin"); MODULE_FIRMWARE("radeon/CEDAR_me.bin"); MODULE_FIRMWARE("radeon/CEDAR_rlc.bin"); +MODULE_FIRMWARE("radeon/CEDAR_smc.bin"); MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin"); MODULE_FIRMWARE("radeon/REDWOOD_me.bin"); MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin"); +MODULE_FIRMWARE("radeon/REDWOOD_smc.bin"); MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin"); MODULE_FIRMWARE("radeon/JUNIPER_me.bin"); MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin"); +MODULE_FIRMWARE("radeon/JUNIPER_smc.bin"); MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin"); MODULE_FIRMWARE("radeon/CYPRESS_me.bin"); MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin"); +MODULE_FIRMWARE("radeon/CYPRESS_smc.bin"); MODULE_FIRMWARE("radeon/PALM_pfp.bin"); MODULE_FIRMWARE("radeon/PALM_me.bin"); MODULE_FIRMWARE("radeon/SUMO_rlc.bin"); @@ -90,6 +87,12 @@ MODULE_FIRMWARE("radeon/SUMO_me.bin"); MODULE_FIRMWARE("radeon/SUMO2_pfp.bin"); MODULE_FIRMWARE("radeon/SUMO2_me.bin"); +static const u32 crtc_offsets[2] = +{ + 0, + AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL +}; + int r600_debugfs_mc_info_init(struct radeon_device *rdev); /* r600,rv610,rv630,rv620,rv635,rv670 */ @@ -98,6 +101,240 @@ static void r600_gpu_init(struct radeon_device *rdev); void r600_fini(struct radeon_device *rdev); void r600_irq_disable(struct radeon_device *rdev); static void r600_pcie_gen2_enable(struct radeon_device *rdev); +extern int evergreen_rlc_resume(struct radeon_device *rdev); +extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev); + +/* + * Indirect registers accessor + */ +u32 r600_rcu_rreg(struct radeon_device *rdev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->rcu_idx_lock, flags); + WREG32(R600_RCU_INDEX, ((reg) & 0x1fff)); + r = RREG32(R600_RCU_DATA); + spin_unlock_irqrestore(&rdev->rcu_idx_lock, flags); + return r; +} + +void r600_rcu_wreg(struct radeon_device *rdev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->rcu_idx_lock, flags); + WREG32(R600_RCU_INDEX, ((reg) & 0x1fff)); + WREG32(R600_RCU_DATA, (v)); + spin_unlock_irqrestore(&rdev->rcu_idx_lock, flags); +} + +u32 r600_uvd_ctx_rreg(struct radeon_device *rdev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->uvd_idx_lock, flags); + WREG32(R600_UVD_CTX_INDEX, ((reg) & 0x1ff)); + r = RREG32(R600_UVD_CTX_DATA); + spin_unlock_irqrestore(&rdev->uvd_idx_lock, flags); + return r; +} + +void r600_uvd_ctx_wreg(struct radeon_device *rdev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->uvd_idx_lock, flags); + WREG32(R600_UVD_CTX_INDEX, ((reg) & 0x1ff)); + WREG32(R600_UVD_CTX_DATA, (v)); + spin_unlock_irqrestore(&rdev->uvd_idx_lock, flags); +} + +/** + * r600_get_allowed_info_register - fetch the register for the info ioctl + * + * @rdev: radeon_device pointer + * @reg: register offset in bytes + * @val: register value + * + * Returns 0 for success or -EINVAL for an invalid register + * + */ +int r600_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val) +{ + switch (reg) { + case GRBM_STATUS: + case GRBM_STATUS2: + case R_000E50_SRBM_STATUS: + case DMA_STATUS_REG: + case UVD_STATUS: + *val = RREG32(reg); + return 0; + default: + return -EINVAL; + } +} + +/** + * r600_get_xclk - get the xclk + * + * @rdev: radeon_device pointer + * + * Returns the reference clock used by the gfx engine + * (r6xx, IGPs, APUs). + */ +u32 r600_get_xclk(struct radeon_device *rdev) +{ + return rdev->clock.spll.reference_freq; +} + +int r600_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) +{ + unsigned fb_div = 0, ref_div, vclk_div = 0, dclk_div = 0; + int r; + + /* bypass vclk and dclk with bclk */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), + ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); + + /* assert BYPASS_EN, deassert UPLL_RESET, UPLL_SLEEP and UPLL_CTLREQ */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~( + UPLL_RESET_MASK | UPLL_SLEEP_MASK | UPLL_CTLREQ_MASK)); + + if (rdev->family >= CHIP_RS780) + WREG32_P(GFX_MACRO_BYPASS_CNTL, UPLL_BYPASS_CNTL, + ~UPLL_BYPASS_CNTL); + + if (!vclk || !dclk) { + /* keep the Bypass mode, put PLL to sleep */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); + return 0; + } + + if (rdev->clock.spll.reference_freq == 10000) + ref_div = 34; + else + ref_div = 4; + + r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000, + ref_div + 1, 0xFFF, 2, 30, ~0, + &fb_div, &vclk_div, &dclk_div); + if (r) + return r; + + if (rdev->family >= CHIP_RV670 && rdev->family < CHIP_RS780) + fb_div >>= 1; + else + fb_div |= 1; + + r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); + if (r) + return r; + + /* assert PLL_RESET */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); + + /* For RS780 we have to choose ref clk */ + if (rdev->family >= CHIP_RS780) + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REFCLK_SRC_SEL_MASK, + ~UPLL_REFCLK_SRC_SEL_MASK); + + /* set the required fb, ref and post divder values */ + WREG32_P(CG_UPLL_FUNC_CNTL, + UPLL_FB_DIV(fb_div) | + UPLL_REF_DIV(ref_div), + ~(UPLL_FB_DIV_MASK | UPLL_REF_DIV_MASK)); + WREG32_P(CG_UPLL_FUNC_CNTL_2, + UPLL_SW_HILEN(vclk_div >> 1) | + UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) | + UPLL_SW_HILEN2(dclk_div >> 1) | + UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)) | + UPLL_DIVEN_MASK | UPLL_DIVEN2_MASK, + ~UPLL_SW_MASK); + + /* give the PLL some time to settle */ + mdelay(15); + + /* deassert PLL_RESET */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); + + mdelay(15); + + /* deassert BYPASS EN */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); + + if (rdev->family >= CHIP_RS780) + WREG32_P(GFX_MACRO_BYPASS_CNTL, 0, ~UPLL_BYPASS_CNTL); + + r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); + if (r) + return r; + + /* switch VCLK and DCLK selection */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), + ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); + + mdelay(100); + + return 0; +} + +void dce3_program_fmt(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + int bpc = 0; + u32 tmp = 0; + enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; + + if (connector) { + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + bpc = radeon_get_monitor_bpc(connector); + dither = radeon_connector->dither; + } + + /* LVDS FMT is set up by atom */ + if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) + return; + + /* not needed for analog */ + if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || + (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) + return; + + if (bpc == 0) + return; + + switch (bpc) { + case 6: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= FMT_SPATIAL_DITHER_EN; + else + tmp |= FMT_TRUNCATE_EN; + break; + case 8: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= (FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH); + else + tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH); + break; + case 10: + default: + /* not needed */ + break; + } + + WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); +} /* get temperature in millidegrees */ int rv6xx_get_temp(struct radeon_device *rdev) @@ -830,7 +1067,7 @@ void r600_pcie_gart_tlb_flush(struct radeon_device *rdev) /* flush hdp cache so updates hit vram */ if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) && !(rdev->flags & RADEON_IS_AGP)) { - volatile uint32_t *ptr = rdev->gart.ptr; + void __iomem *ptr = (void *)rdev->gart.ptr; u32 tmp; /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read @@ -839,7 +1076,7 @@ void r600_pcie_gart_tlb_flush(struct radeon_device *rdev) * method for them. */ WREG32(HDP_DEBUG1, 0); - tmp = *ptr; + tmp = readl((void __iomem *)ptr); } else WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); @@ -889,7 +1126,6 @@ static int r600_pcie_gart_enable(struct radeon_device *rdev) r = radeon_gart_table_vram_pin(rdev); if (r) return r; - radeon_gart_restore(rdev); /* Setup L2 cache */ WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | @@ -914,6 +1150,8 @@ static int r600_pcie_gart_enable(struct radeon_device *rdev) WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp); WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp); WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp); + WREG32(MC_VM_L1_TLB_MCB_RD_UVD_CNTL, tmp); + WREG32(MC_VM_L1_TLB_MCB_WR_UVD_CNTL, tmp); WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE); WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE); WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); @@ -964,6 +1202,8 @@ static void r600_pcie_gart_disable(struct radeon_device *rdev) WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp); WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp); WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp); + WREG32(MC_VM_L1_TLB_MCB_RD_UVD_CNTL, tmp); + WREG32(MC_VM_L1_TLB_MCB_WR_UVD_CNTL, tmp); radeon_gart_table_vram_unpin(rdev); } @@ -1023,6 +1263,31 @@ int r600_mc_wait_for_idle(struct radeon_device *rdev) return -1; } +uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg) +{ + unsigned long flags; + uint32_t r; + + spin_lock_irqsave(&rdev->mc_idx_lock, flags); + WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg)); + r = RREG32(R_0028FC_MC_DATA); + WREG32(R_0028F8_MC_INDEX, ~C_0028F8_MC_IND_ADDR); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); + return r; +} + +void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v) +{ + unsigned long flags; + + spin_lock_irqsave(&rdev->mc_idx_lock, flags); + WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg) | + S_0028F8_MC_IND_WR_EN(1)); + WREG32(R_0028FC_MC_DATA, v); + WREG32(R_0028F8_MC_INDEX, 0x7F); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); +} + static void r600_mc_program(struct radeon_device *rdev) { struct rv515_mc_save save; @@ -1122,7 +1387,7 @@ static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc } if (rdev->flags & RADEON_IS_AGP) { size_bf = mc->gtt_start; - size_af = 0xFFFFFFFF - mc->gtt_end; + size_af = mc->mc_mask - mc->gtt_end; if (size_bf > size_af) { if (mc->mc_vram_size > size_bf) { dev_warn(rdev->dev, "limiting VRAM\n"); @@ -1158,6 +1423,8 @@ static int r600_mc_init(struct radeon_device *rdev) { u32 tmp; int chansize, numchan; + uint32_t h_addr, l_addr; + unsigned long long k8_addr; /* Get VRAM informations */ rdev->mc.vram_is_ddr = true; @@ -1198,7 +1465,30 @@ static int r600_mc_init(struct radeon_device *rdev) if (rdev->flags & RADEON_IS_IGP) { rs690_pm_info(rdev); rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev); + + if (rdev->family == CHIP_RS780 || rdev->family == CHIP_RS880) { + /* Use K8 direct mapping for fast fb access. */ + rdev->fastfb_working = false; + h_addr = G_000012_K8_ADDR_EXT(RREG32_MC(R_000012_MC_MISC_UMA_CNTL)); + l_addr = RREG32_MC(R_000011_K8_FB_LOCATION); + k8_addr = ((unsigned long long)h_addr) << 32 | l_addr; +#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE) + if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL) +#endif + { + /* FastFB shall be used with UMA memory. Here it is simply disabled when sideport + * memory is present. + */ + if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) { + DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n", + (unsigned long long)rdev->mc.aper_base, k8_addr); + rdev->mc.aper_base = (resource_size_t)k8_addr; + rdev->fastfb_working = true; + } + } + } } + radeon_update_bandwidth_info(rdev); return 0; } @@ -1210,7 +1500,7 @@ int r600_vram_scratch_init(struct radeon_device *rdev) if (rdev->vram_scratch.robj == NULL) { r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM, - NULL, &rdev->vram_scratch.robj); + 0, NULL, NULL, &rdev->vram_scratch.robj); if (r) { return r; } @@ -1250,207 +1540,381 @@ void r600_vram_scratch_fini(struct radeon_device *rdev) radeon_bo_unref(&rdev->vram_scratch.robj); } -/* We doesn't check that the GPU really needs a reset we simply do the - * reset, it's up to the caller to determine if the GPU needs one. We - * might add an helper function to check that. - */ -static void r600_gpu_soft_reset_gfx(struct radeon_device *rdev) +void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung) { - u32 grbm_busy_mask = S_008010_VC_BUSY(1) | S_008010_VGT_BUSY_NO_DMA(1) | - S_008010_VGT_BUSY(1) | S_008010_TA03_BUSY(1) | - S_008010_TC_BUSY(1) | S_008010_SX_BUSY(1) | - S_008010_SH_BUSY(1) | S_008010_SPI03_BUSY(1) | - S_008010_SMX_BUSY(1) | S_008010_SC_BUSY(1) | - S_008010_PA_BUSY(1) | S_008010_DB03_BUSY(1) | - S_008010_CR_BUSY(1) | S_008010_CB03_BUSY(1) | - S_008010_GUI_ACTIVE(1); - u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) | S_008014_SPI1_BUSY(1) | - S_008014_SPI2_BUSY(1) | S_008014_SPI3_BUSY(1) | - S_008014_TA0_BUSY(1) | S_008014_TA1_BUSY(1) | - S_008014_TA2_BUSY(1) | S_008014_TA3_BUSY(1) | - S_008014_DB0_BUSY(1) | S_008014_DB1_BUSY(1) | - S_008014_DB2_BUSY(1) | S_008014_DB3_BUSY(1) | - S_008014_CB0_BUSY(1) | S_008014_CB1_BUSY(1) | - S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1); - u32 tmp; + u32 tmp = RREG32(R600_BIOS_3_SCRATCH); - if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) - return; + if (hung) + tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG; + else + tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG; + + WREG32(R600_BIOS_3_SCRATCH, tmp); +} +static void r600_print_gpu_status_regs(struct radeon_device *rdev) +{ dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n", - RREG32(R_008010_GRBM_STATUS)); + RREG32(R_008010_GRBM_STATUS)); dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n", - RREG32(R_008014_GRBM_STATUS2)); + RREG32(R_008014_GRBM_STATUS2)); dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n", - RREG32(R_000E50_SRBM_STATUS)); + RREG32(R_000E50_SRBM_STATUS)); dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", - RREG32(CP_STALLED_STAT1)); + RREG32(CP_STALLED_STAT1)); dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", - RREG32(CP_STALLED_STAT2)); + RREG32(CP_STALLED_STAT2)); dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", - RREG32(CP_BUSY_STAT)); + RREG32(CP_BUSY_STAT)); dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", - RREG32(CP_STAT)); + RREG32(CP_STAT)); + dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", + RREG32(DMA_STATUS_REG)); +} - /* Disable CP parsing/prefetching */ - WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1)); +static bool r600_is_display_hung(struct radeon_device *rdev) +{ + u32 crtc_hung = 0; + u32 crtc_status[2]; + u32 i, j, tmp; + + for (i = 0; i < rdev->num_crtc; i++) { + if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) { + crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]); + crtc_hung |= (1 << i); + } + } - /* Check if any of the rendering block is busy and reset it */ - if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) || - (RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) { - tmp = S_008020_SOFT_RESET_CR(1) | - S_008020_SOFT_RESET_DB(1) | - S_008020_SOFT_RESET_CB(1) | - S_008020_SOFT_RESET_PA(1) | - S_008020_SOFT_RESET_SC(1) | - S_008020_SOFT_RESET_SMX(1) | - S_008020_SOFT_RESET_SPI(1) | - S_008020_SOFT_RESET_SX(1) | - S_008020_SOFT_RESET_SH(1) | - S_008020_SOFT_RESET_TC(1) | - S_008020_SOFT_RESET_TA(1) | - S_008020_SOFT_RESET_VC(1) | - S_008020_SOFT_RESET_VGT(1); - dev_info(rdev->dev, " R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp); - WREG32(R_008020_GRBM_SOFT_RESET, tmp); - RREG32(R_008020_GRBM_SOFT_RESET); - mdelay(15); - WREG32(R_008020_GRBM_SOFT_RESET, 0); + for (j = 0; j < 10; j++) { + for (i = 0; i < rdev->num_crtc; i++) { + if (crtc_hung & (1 << i)) { + tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]); + if (tmp != crtc_status[i]) + crtc_hung &= ~(1 << i); + } + } + if (crtc_hung == 0) + return false; + udelay(100); } - /* Reset CP (we always reset CP) */ - tmp = S_008020_SOFT_RESET_CP(1); - dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp); - WREG32(R_008020_GRBM_SOFT_RESET, tmp); - RREG32(R_008020_GRBM_SOFT_RESET); - mdelay(15); - WREG32(R_008020_GRBM_SOFT_RESET, 0); - dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n", - RREG32(R_008010_GRBM_STATUS)); - dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n", - RREG32(R_008014_GRBM_STATUS2)); - dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n", - RREG32(R_000E50_SRBM_STATUS)); - dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", - RREG32(CP_STALLED_STAT1)); - dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", - RREG32(CP_STALLED_STAT2)); - dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", - RREG32(CP_BUSY_STAT)); - dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", - RREG32(CP_STAT)); + return true; +} + +u32 r600_gpu_check_soft_reset(struct radeon_device *rdev) +{ + u32 reset_mask = 0; + u32 tmp; + + /* GRBM_STATUS */ + tmp = RREG32(R_008010_GRBM_STATUS); + if (rdev->family >= CHIP_RV770) { + if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) | + G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) | + G_008010_TA_BUSY(tmp) | G_008010_VGT_BUSY(tmp) | + G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) | + G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp)) + reset_mask |= RADEON_RESET_GFX; + } else { + if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) | + G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) | + G_008010_TA03_BUSY(tmp) | G_008010_VGT_BUSY(tmp) | + G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) | + G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp)) + reset_mask |= RADEON_RESET_GFX; + } + + if (G_008010_CF_RQ_PENDING(tmp) | G_008010_PF_RQ_PENDING(tmp) | + G_008010_CP_BUSY(tmp) | G_008010_CP_COHERENCY_BUSY(tmp)) + reset_mask |= RADEON_RESET_CP; + + if (G_008010_GRBM_EE_BUSY(tmp)) + reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; + + /* DMA_STATUS_REG */ + tmp = RREG32(DMA_STATUS_REG); + if (!(tmp & DMA_IDLE)) + reset_mask |= RADEON_RESET_DMA; + + /* SRBM_STATUS */ + tmp = RREG32(R_000E50_SRBM_STATUS); + if (G_000E50_RLC_RQ_PENDING(tmp) | G_000E50_RLC_BUSY(tmp)) + reset_mask |= RADEON_RESET_RLC; + + if (G_000E50_IH_BUSY(tmp)) + reset_mask |= RADEON_RESET_IH; + + if (G_000E50_SEM_BUSY(tmp)) + reset_mask |= RADEON_RESET_SEM; + + if (G_000E50_GRBM_RQ_PENDING(tmp)) + reset_mask |= RADEON_RESET_GRBM; + + if (G_000E50_VMC_BUSY(tmp)) + reset_mask |= RADEON_RESET_VMC; + + if (G_000E50_MCB_BUSY(tmp) | G_000E50_MCDZ_BUSY(tmp) | + G_000E50_MCDY_BUSY(tmp) | G_000E50_MCDX_BUSY(tmp) | + G_000E50_MCDW_BUSY(tmp)) + reset_mask |= RADEON_RESET_MC; + + if (r600_is_display_hung(rdev)) + reset_mask |= RADEON_RESET_DISPLAY; + /* Skip MC reset as it's mostly likely not hung, just busy */ + if (reset_mask & RADEON_RESET_MC) { + DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); + reset_mask &= ~RADEON_RESET_MC; + } + + return reset_mask; } -static void r600_gpu_soft_reset_dma(struct radeon_device *rdev) +static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) { + struct rv515_mc_save save; + u32 grbm_soft_reset = 0, srbm_soft_reset = 0; u32 tmp; - if (RREG32(DMA_STATUS_REG) & DMA_IDLE) + if (reset_mask == 0) return; - dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", - RREG32(DMA_STATUS_REG)); + dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); - /* Disable DMA */ - tmp = RREG32(DMA_RB_CNTL); - tmp &= ~DMA_RB_ENABLE; - WREG32(DMA_RB_CNTL, tmp); + r600_print_gpu_status_regs(rdev); - /* Reset dma */ + /* Disable CP parsing/prefetching */ if (rdev->family >= CHIP_RV770) - WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA); + WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1)); else - WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA); - RREG32(SRBM_SOFT_RESET); + WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1)); + + /* disable the RLC */ + WREG32(RLC_CNTL, 0); + + if (reset_mask & RADEON_RESET_DMA) { + /* Disable DMA */ + tmp = RREG32(DMA_RB_CNTL); + tmp &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL, tmp); + } + + mdelay(50); + + rv515_mc_stop(rdev, &save); + if (r600_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); + } + + if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { + if (rdev->family >= CHIP_RV770) + grbm_soft_reset |= S_008020_SOFT_RESET_DB(1) | + S_008020_SOFT_RESET_CB(1) | + S_008020_SOFT_RESET_PA(1) | + S_008020_SOFT_RESET_SC(1) | + S_008020_SOFT_RESET_SPI(1) | + S_008020_SOFT_RESET_SX(1) | + S_008020_SOFT_RESET_SH(1) | + S_008020_SOFT_RESET_TC(1) | + S_008020_SOFT_RESET_TA(1) | + S_008020_SOFT_RESET_VC(1) | + S_008020_SOFT_RESET_VGT(1); + else + grbm_soft_reset |= S_008020_SOFT_RESET_CR(1) | + S_008020_SOFT_RESET_DB(1) | + S_008020_SOFT_RESET_CB(1) | + S_008020_SOFT_RESET_PA(1) | + S_008020_SOFT_RESET_SC(1) | + S_008020_SOFT_RESET_SMX(1) | + S_008020_SOFT_RESET_SPI(1) | + S_008020_SOFT_RESET_SX(1) | + S_008020_SOFT_RESET_SH(1) | + S_008020_SOFT_RESET_TC(1) | + S_008020_SOFT_RESET_TA(1) | + S_008020_SOFT_RESET_VC(1) | + S_008020_SOFT_RESET_VGT(1); + } + + if (reset_mask & RADEON_RESET_CP) { + grbm_soft_reset |= S_008020_SOFT_RESET_CP(1) | + S_008020_SOFT_RESET_VGT(1); + + srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1); + } + + if (reset_mask & RADEON_RESET_DMA) { + if (rdev->family >= CHIP_RV770) + srbm_soft_reset |= RV770_SOFT_RESET_DMA; + else + srbm_soft_reset |= SOFT_RESET_DMA; + } + + if (reset_mask & RADEON_RESET_RLC) + srbm_soft_reset |= S_000E60_SOFT_RESET_RLC(1); + + if (reset_mask & RADEON_RESET_SEM) + srbm_soft_reset |= S_000E60_SOFT_RESET_SEM(1); + + if (reset_mask & RADEON_RESET_IH) + srbm_soft_reset |= S_000E60_SOFT_RESET_IH(1); + + if (reset_mask & RADEON_RESET_GRBM) + srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1); + + if (!(rdev->flags & RADEON_IS_IGP)) { + if (reset_mask & RADEON_RESET_MC) + srbm_soft_reset |= S_000E60_SOFT_RESET_MC(1); + } + + if (reset_mask & RADEON_RESET_VMC) + srbm_soft_reset |= S_000E60_SOFT_RESET_VMC(1); + + if (grbm_soft_reset) { + tmp = RREG32(R_008020_GRBM_SOFT_RESET); + tmp |= grbm_soft_reset; + dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(R_008020_GRBM_SOFT_RESET, tmp); + tmp = RREG32(R_008020_GRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~grbm_soft_reset; + WREG32(R_008020_GRBM_SOFT_RESET, tmp); + tmp = RREG32(R_008020_GRBM_SOFT_RESET); + } + + if (srbm_soft_reset) { + tmp = RREG32(SRBM_SOFT_RESET); + tmp |= srbm_soft_reset; + dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~srbm_soft_reset; + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); + } + + /* Wait a little for things to settle down */ + mdelay(1); + + rv515_mc_resume(rdev, &save); udelay(50); - WREG32(SRBM_SOFT_RESET, 0); - dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", - RREG32(DMA_STATUS_REG)); + r600_print_gpu_status_regs(rdev); } -static int r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) +static void r600_gpu_pci_config_reset(struct radeon_device *rdev) { struct rv515_mc_save save; + u32 tmp, i; - if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) - reset_mask &= ~(RADEON_RESET_GFX | RADEON_RESET_COMPUTE); + dev_info(rdev->dev, "GPU pci config reset\n"); - if (RREG32(DMA_STATUS_REG) & DMA_IDLE) - reset_mask &= ~RADEON_RESET_DMA; + /* disable dpm? */ - if (reset_mask == 0) - return 0; + /* Disable CP parsing/prefetching */ + if (rdev->family >= CHIP_RV770) + WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1)); + else + WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1)); - dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); + /* disable the RLC */ + WREG32(RLC_CNTL, 0); + + /* Disable DMA */ + tmp = RREG32(DMA_RB_CNTL); + tmp &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL, tmp); + + mdelay(50); + /* set mclk/sclk to bypass */ + if (rdev->family >= CHIP_RV770) + rv770_set_clk_bypass_mode(rdev); + /* disable BM */ + pci_clear_master(rdev->pdev); + /* disable mem access */ rv515_mc_stop(rdev, &save); if (r600_mc_wait_for_idle(rdev)) { dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); } - if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) - r600_gpu_soft_reset_gfx(rdev); + /* BIF reset workaround. Not sure if this is needed on 6xx */ + tmp = RREG32(BUS_CNTL); + tmp |= VGA_COHE_SPEC_TIMER_DIS; + WREG32(BUS_CNTL, tmp); - if (reset_mask & RADEON_RESET_DMA) - r600_gpu_soft_reset_dma(rdev); + tmp = RREG32(BIF_SCRATCH0); - /* Wait a little for things to settle down */ + /* reset */ + radeon_pci_config_reset(rdev); mdelay(1); - rv515_mc_resume(rdev, &save); - return 0; + /* BIF reset workaround. Not sure if this is needed on 6xx */ + tmp = SOFT_RESET_BIF; + WREG32(SRBM_SOFT_RESET, tmp); + mdelay(1); + WREG32(SRBM_SOFT_RESET, 0); + + /* wait for asic to come out of reset */ + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) + break; + udelay(1); + } } -bool r600_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +int r600_asic_reset(struct radeon_device *rdev) { - u32 srbm_status; - u32 grbm_status; - u32 grbm_status2; - - srbm_status = RREG32(R_000E50_SRBM_STATUS); - grbm_status = RREG32(R_008010_GRBM_STATUS); - grbm_status2 = RREG32(R_008014_GRBM_STATUS2); - if (!G_008010_GUI_ACTIVE(grbm_status)) { - radeon_ring_lockup_update(ring); - return false; - } - /* force CP activities */ - radeon_ring_force_activity(rdev, ring); - return radeon_ring_test_lockup(rdev, ring); + u32 reset_mask; + + reset_mask = r600_gpu_check_soft_reset(rdev); + + if (reset_mask) + r600_set_bios_scratch_engine_hung(rdev, true); + + /* try soft reset */ + r600_gpu_soft_reset(rdev, reset_mask); + + reset_mask = r600_gpu_check_soft_reset(rdev); + + /* try pci config reset */ + if (reset_mask && radeon_hard_reset) + r600_gpu_pci_config_reset(rdev); + + reset_mask = r600_gpu_check_soft_reset(rdev); + + if (!reset_mask) + r600_set_bios_scratch_engine_hung(rdev, false); + + return 0; } /** - * r600_dma_is_lockup - Check if the DMA engine is locked up + * r600_gfx_is_lockup - Check if the GFX engine is locked up * * @rdev: radeon_device pointer * @ring: radeon_ring structure holding ring information * - * Check if the async DMA engine is locked up (r6xx-evergreen). + * Check if the GFX engine is locked up. * Returns true if the engine appears to be locked up, false if not. */ -bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) { - u32 dma_status_reg; + u32 reset_mask = r600_gpu_check_soft_reset(rdev); - dma_status_reg = RREG32(DMA_STATUS_REG); - if (dma_status_reg & DMA_IDLE) { - radeon_ring_lockup_update(ring); + if (!(reset_mask & (RADEON_RESET_GFX | + RADEON_RESET_COMPUTE | + RADEON_RESET_CP))) { + radeon_ring_lockup_update(rdev, ring); return false; } - /* force ring activities */ - radeon_ring_force_activity(rdev, ring); return radeon_ring_test_lockup(rdev, ring); } -int r600_asic_reset(struct radeon_device *rdev) -{ - return r600_gpu_soft_reset(rdev, (RADEON_RESET_GFX | - RADEON_RESET_COMPUTE | - RADEON_RESET_DMA)); -} - u32 r6xx_remap_render_backend(struct radeon_device *rdev, u32 tiling_pipe_num, u32 max_rb_num, @@ -1510,7 +1974,6 @@ static void r600_gpu_init(struct radeon_device *rdev) { u32 tiling_config; u32 ramcfg; - u32 cc_rb_backend_disable; u32 cc_gc_shader_pipe_config; u32 tmp; int i, j; @@ -1637,26 +2100,20 @@ static void r600_gpu_init(struct radeon_device *rdev) } tiling_config |= BANK_SWAPS(1); - cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000; - tmp = R6XX_MAX_BACKENDS - - r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK); - if (tmp < rdev->config.r600.max_backends) { - rdev->config.r600.max_backends = tmp; - } - cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00; - tmp = R6XX_MAX_PIPES - - r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK); - if (tmp < rdev->config.r600.max_pipes) { - rdev->config.r600.max_pipes = tmp; - } - tmp = R6XX_MAX_SIMDS - + tmp = rdev->config.r600.max_simds - r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK); - if (tmp < rdev->config.r600.max_simds) { - rdev->config.r600.max_simds = tmp; - } + rdev->config.r600.active_simds = tmp; disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK; + tmp = 0; + for (i = 0; i < rdev->config.r600.max_backends; i++) + tmp |= (1 << i); + /* if all the backends are disabled, fix it up here */ + if ((disabled_rb_mask & tmp) == tmp) { + for (i = 0; i < rdev->config.r600.max_backends; i++) + disabled_rb_mask &= ~(1 << i); + } tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT; tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends, R6XX_MAX_BACKENDS, disabled_rb_mask); @@ -1921,20 +2378,27 @@ static void r600_gpu_init(struct radeon_device *rdev) */ u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg) { + unsigned long flags; u32 r; + spin_lock_irqsave(&rdev->pciep_idx_lock, flags); WREG32(PCIE_PORT_INDEX, ((reg) & 0xff)); (void)RREG32(PCIE_PORT_INDEX); r = RREG32(PCIE_PORT_DATA); + spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); return r; } void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) { + unsigned long flags; + + spin_lock_irqsave(&rdev->pciep_idx_lock, flags); WREG32(PCIE_PORT_INDEX, ((reg) & 0xff)); (void)RREG32(PCIE_PORT_INDEX); WREG32(PCIE_PORT_DATA, (v)); (void)RREG32(PCIE_PORT_DATA); + spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); } /* @@ -1942,7 +2406,8 @@ void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) */ void r600_cp_stop(struct radeon_device *rdev) { - radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1)); WREG32(SCRATCH_UMSK, 0); rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; @@ -1952,92 +2417,103 @@ int r600_init_microcode(struct radeon_device *rdev) { const char *chip_name; const char *rlc_chip_name; - size_t pfp_req_size, me_req_size, rlc_req_size; + const char *smc_chip_name = "RV770"; + size_t pfp_req_size, me_req_size, rlc_req_size, smc_req_size = 0; char fw_name[30]; int err; DRM_DEBUG("\n"); -#if 0 - pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0); - err = IS_ERR(pdev); - if (err) { - printk(KERN_ERR "radeon_cp: Failed to register firmware\n"); - return -EINVAL; - } -#endif - switch (rdev->family) { case CHIP_R600: - chip_name = "r600"; - rlc_chip_name = "r600"; + chip_name = "R600"; + rlc_chip_name = "R600"; break; case CHIP_RV610: - chip_name = "rv610"; - rlc_chip_name = "r600"; + chip_name = "RV610"; + rlc_chip_name = "R600"; break; case CHIP_RV630: - chip_name = "rv630"; - rlc_chip_name = "r600"; + chip_name = "RV630"; + rlc_chip_name = "R600"; break; case CHIP_RV620: - chip_name = "rv620"; - rlc_chip_name = "r600"; + chip_name = "RV620"; + rlc_chip_name = "R600"; break; case CHIP_RV635: - chip_name = "rv635"; - rlc_chip_name = "r600"; + chip_name = "RV635"; + rlc_chip_name = "R600"; break; case CHIP_RV670: - chip_name = "rv670"; - rlc_chip_name = "r600"; + chip_name = "RV670"; + rlc_chip_name = "R600"; break; case CHIP_RS780: case CHIP_RS880: - chip_name = "rs780"; - rlc_chip_name = "r600"; + chip_name = "RS780"; + rlc_chip_name = "R600"; break; case CHIP_RV770: - chip_name = "rv770"; - rlc_chip_name = "r700"; + chip_name = "RV770"; + rlc_chip_name = "R700"; + smc_chip_name = "RV770"; + smc_req_size = roundup2(RV770_SMC_UCODE_SIZE, 4); break; case CHIP_RV730: - case CHIP_RV740: - chip_name = "rv730"; - rlc_chip_name = "r700"; + chip_name = "RV730"; + rlc_chip_name = "R700"; + smc_chip_name = "RV730"; + smc_req_size = roundup2(RV730_SMC_UCODE_SIZE, 4); break; case CHIP_RV710: - chip_name = "rv710"; - rlc_chip_name = "r700"; + chip_name = "RV710"; + rlc_chip_name = "R700"; + smc_chip_name = "RV710"; + smc_req_size = roundup2(RV710_SMC_UCODE_SIZE, 4); + break; + case CHIP_RV740: + chip_name = "RV730"; + rlc_chip_name = "R700"; + smc_chip_name = "RV740"; + smc_req_size = roundup2(RV740_SMC_UCODE_SIZE, 4); break; case CHIP_CEDAR: - chip_name = "cedar"; - rlc_chip_name = "cedar"; + chip_name = "CEDAR"; + rlc_chip_name = "CEDAR"; + smc_chip_name = "CEDAR"; + smc_req_size = roundup2(CEDAR_SMC_UCODE_SIZE, 4); break; case CHIP_REDWOOD: - chip_name = "redwood"; - rlc_chip_name = "redwood"; + chip_name = "REDWOOD"; + rlc_chip_name = "REDWOOD"; + smc_chip_name = "REDWOOD"; + smc_req_size = roundup2(REDWOOD_SMC_UCODE_SIZE, 4); break; case CHIP_JUNIPER: - chip_name = "juniper"; - rlc_chip_name = "juniper"; + chip_name = "JUNIPER"; + rlc_chip_name = "JUNIPER"; + smc_chip_name = "JUNIPER"; + smc_req_size = roundup2(JUNIPER_SMC_UCODE_SIZE, 4); break; case CHIP_CYPRESS: case CHIP_HEMLOCK: - chip_name = "cypress"; - rlc_chip_name = "cypress"; + chip_name = "CYPRESS"; + rlc_chip_name = "CYPRESS"; + smc_chip_name = "CYPRESS"; + smc_req_size = roundup2(CYPRESS_SMC_UCODE_SIZE, 4); break; case CHIP_PALM: - chip_name = "palm"; - rlc_chip_name = "sumo"; + chip_name = "PALM"; + rlc_chip_name = "SUMO"; break; case CHIP_SUMO: - chip_name = "sumo"; - rlc_chip_name = "sumo"; + chip_name = "SUMO"; + rlc_chip_name = "SUMO"; break; case CHIP_SUMO2: - chip_name = "sumo2"; - rlc_chip_name = "sumo"; + chip_name = "SUMO2"; + rlc_chip_name = "SUMO"; break; default: BUG(); } @@ -2051,69 +2527,113 @@ int r600_init_microcode(struct radeon_device *rdev) me_req_size = R700_PM4_UCODE_SIZE * 4; rlc_req_size = R700_RLC_UCODE_SIZE * 4; } else { - pfp_req_size = PFP_UCODE_SIZE * 4; - me_req_size = PM4_UCODE_SIZE * 12; - rlc_req_size = RLC_UCODE_SIZE * 4; + pfp_req_size = R600_PFP_UCODE_SIZE * 4; + me_req_size = R600_PM4_UCODE_SIZE * 12; + rlc_req_size = R600_RLC_UCODE_SIZE * 4; } DRM_INFO("Loading %s Microcode\n", chip_name); - snprintf(fw_name, sizeof(fw_name), "radeon-%s_pfp", chip_name); - err = loadfirmware(fw_name, &rdev->pfp_fw, &rdev->pfp_fw_size); + snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); + err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); if (err) goto out; - if (rdev->pfp_fw_size != pfp_req_size) { - DRM_ERROR( + if (rdev->pfp_fw->size != pfp_req_size) { + printk(KERN_ERR "r600_cp: Bogus length %zu in firmware \"%s\"\n", - rdev->pfp_fw_size, fw_name); + rdev->pfp_fw->size, fw_name); err = -EINVAL; goto out; } - snprintf(fw_name, sizeof(fw_name), "radeon-%s_me", chip_name); - err = loadfirmware(fw_name, &rdev->me_fw, &rdev->me_fw_size); + snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); + err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); if (err) goto out; - if (rdev->me_fw_size != me_req_size) { - DRM_ERROR( + if (rdev->me_fw->size != me_req_size) { + printk(KERN_ERR "r600_cp: Bogus length %zu in firmware \"%s\"\n", - rdev->me_fw_size, fw_name); + rdev->me_fw->size, fw_name); err = -EINVAL; } - snprintf(fw_name, sizeof(fw_name), "radeon-%s_rlc", rlc_chip_name); - err = loadfirmware(fw_name, &rdev->rlc_fw, &rdev->rlc_fw_size); + snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name); + err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); if (err) goto out; - if (rdev->rlc_fw_size != rlc_req_size) { - DRM_ERROR( + if (rdev->rlc_fw->size != rlc_req_size) { + printk(KERN_ERR "r600_rlc: Bogus length %zu in firmware \"%s\"\n", - rdev->rlc_fw_size, fw_name); + rdev->rlc_fw->size, fw_name); err = -EINVAL; } + if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_HEMLOCK)) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", smc_chip_name); + err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); + if (err) { + printk(KERN_ERR + "smc: error loading firmware \"%s\"\n", + fw_name); + release_firmware(rdev->smc_fw); + rdev->smc_fw = NULL; + err = 0; + } else if (rdev->smc_fw->size != smc_req_size) { + printk(KERN_ERR + "smc: Bogus length %zu in firmware \"%s\"\n", + rdev->smc_fw->size, fw_name); + err = -EINVAL; + } + } + out: if (err) { if (err != -EINVAL) printk(KERN_ERR "r600_cp: Failed to load firmware \"%s\"\n", fw_name); - if (rdev->pfp_fw) { - free(rdev->pfp_fw, M_DEVBUF, 0); - rdev->pfp_fw = NULL; - } - if (rdev->me_fw) { - free(rdev->me_fw, M_DEVBUF, 0); - rdev->me_fw = NULL; - } - if (rdev->rlc_fw) { - free(rdev->rlc_fw, M_DEVBUF, 0); - rdev->rlc_fw = NULL; - } + release_firmware(rdev->pfp_fw); + rdev->pfp_fw = NULL; + release_firmware(rdev->me_fw); + rdev->me_fw = NULL; + release_firmware(rdev->rlc_fw); + rdev->rlc_fw = NULL; + release_firmware(rdev->smc_fw); + rdev->smc_fw = NULL; } return err; } +u32 r600_gfx_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 rptr; + + if (rdev->wb.enabled) + rptr = rdev->wb.wb[ring->rptr_offs/4]; + else + rptr = RREG32(R600_CP_RB_RPTR); + + return rptr; +} + +u32 r600_gfx_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 wptr; + + wptr = RREG32(R600_CP_RB_WPTR); + + return wptr; +} + +void r600_gfx_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + WREG32(R600_CP_RB_WPTR, ring->wptr); + (void)RREG32(R600_CP_RB_WPTR); +} + static int r600_cp_load_microcode(struct radeon_device *rdev) { const __be32 *fw_data; @@ -2138,15 +2658,15 @@ static int r600_cp_load_microcode(struct radeon_device *rdev) WREG32(CP_ME_RAM_WADDR, 0); - fw_data = (const __be32 *)rdev->me_fw; + fw_data = (const __be32 *)rdev->me_fw->data; WREG32(CP_ME_RAM_WADDR, 0); - for (i = 0; i < PM4_UCODE_SIZE * 3; i++) + for (i = 0; i < R600_PM4_UCODE_SIZE * 3; i++) WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); - fw_data = (const __be32 *)rdev->pfp_fw; + fw_data = (const __be32 *)rdev->pfp_fw->data; WREG32(CP_PFP_UCODE_ADDR, 0); - for (i = 0; i < PFP_UCODE_SIZE; i++) + for (i = 0; i < R600_PFP_UCODE_SIZE; i++) WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); @@ -2179,7 +2699,7 @@ int r600_cp_start(struct radeon_device *rdev) radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); radeon_ring_write(ring, 0); radeon_ring_write(ring, 0); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); cp_me = 0xff; WREG32(R_0086D8_CP_ME_CNTL, cp_me); @@ -2200,8 +2720,8 @@ int r600_cp_resume(struct radeon_device *rdev) WREG32(GRBM_SOFT_RESET, 0); /* Set ring buffer size */ - rb_bufsz = drm_order(ring->ring_size / 8); - tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; + rb_bufsz = order_base_2(ring->ring_size / 8); + tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; #ifdef __BIG_ENDIAN tmp |= BUF_SWAP_32BIT; #endif @@ -2236,8 +2756,6 @@ int r600_cp_resume(struct radeon_device *rdev) WREG32(CP_RB_BASE, ring->gpu_addr >> 8); WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); - ring->rptr = RREG32(CP_RB_RPTR); - r600_cp_start(rdev); ring->ready = true; r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); @@ -2245,6 +2763,10 @@ int r600_cp_resume(struct radeon_device *rdev) ring->ready = false; return r; } + + if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); + return 0; } @@ -2254,7 +2776,7 @@ void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsign int r; /* Align ring size */ - rb_bufsz = drm_order(ring_size / 8); + rb_bufsz = order_base_2(ring_size / 8); ring_size = (1 << (rb_bufsz + 1)) * 4; ring->ring_size = ring_size; ring->align_mask = 16 - 1; @@ -2276,133 +2798,6 @@ void r600_cp_fini(struct radeon_device *rdev) radeon_scratch_free(rdev, ring->rptr_save_reg); } -/* - * DMA - * Starting with R600, the GPU has an asynchronous - * DMA engine. The programming model is very similar - * to the 3D engine (ring buffer, IBs, etc.), but the - * DMA controller has it's own packet format that is - * different form the PM4 format used by the 3D engine. - * It supports copying data, writing embedded data, - * solid fills, and a number of other things. It also - * has support for tiling/detiling of buffers. - */ -/** - * r600_dma_stop - stop the async dma engine - * - * @rdev: radeon_device pointer - * - * Stop the async dma engine (r6xx-evergreen). - */ -void r600_dma_stop(struct radeon_device *rdev) -{ - u32 rb_cntl = RREG32(DMA_RB_CNTL); - - radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); - - rb_cntl &= ~DMA_RB_ENABLE; - WREG32(DMA_RB_CNTL, rb_cntl); - - rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false; -} - -/** - * r600_dma_resume - setup and start the async dma engine - * - * @rdev: radeon_device pointer - * - * Set up the DMA ring buffer and enable it. (r6xx-evergreen). - * Returns 0 for success, error for failure. - */ -int r600_dma_resume(struct radeon_device *rdev) -{ - struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; - u32 rb_cntl, dma_cntl, ib_cntl; - u32 rb_bufsz; - int r; - - /* Reset dma */ - if (rdev->family >= CHIP_RV770) - WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA); - else - WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA); - RREG32(SRBM_SOFT_RESET); - udelay(50); - WREG32(SRBM_SOFT_RESET, 0); - - WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0); - WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0); - - /* Set ring buffer size in dwords */ - rb_bufsz = drm_order(ring->ring_size / 4); - rb_cntl = rb_bufsz << 1; -#ifdef __BIG_ENDIAN - rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE; -#endif - WREG32(DMA_RB_CNTL, rb_cntl); - - /* Initialize the ring buffer's read and write pointers */ - WREG32(DMA_RB_RPTR, 0); - WREG32(DMA_RB_WPTR, 0); - - /* set the wb address whether it's enabled or not */ - WREG32(DMA_RB_RPTR_ADDR_HI, - upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF); - WREG32(DMA_RB_RPTR_ADDR_LO, - ((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC)); - - if (rdev->wb.enabled) - rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE; - - WREG32(DMA_RB_BASE, ring->gpu_addr >> 8); - - /* enable DMA IBs */ - ib_cntl = DMA_IB_ENABLE; -#ifdef __BIG_ENDIAN - ib_cntl |= DMA_IB_SWAP_ENABLE; -#endif - WREG32(DMA_IB_CNTL, ib_cntl); - - dma_cntl = RREG32(DMA_CNTL); - dma_cntl &= ~CTXEMPTY_INT_ENABLE; - WREG32(DMA_CNTL, dma_cntl); - - if (rdev->family >= CHIP_RV770) - WREG32(DMA_MODE, 1); - - ring->wptr = 0; - WREG32(DMA_RB_WPTR, ring->wptr << 2); - - ring->rptr = RREG32(DMA_RB_RPTR) >> 2; - - WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE); - - ring->ready = true; - - r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring); - if (r) { - ring->ready = false; - return r; - } - - radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); - - return 0; -} - -/** - * r600_dma_fini - tear down the async dma engine - * - * @rdev: radeon_device pointer - * - * Stop the async dma engine and free the ring (r6xx-evergreen). - */ -void r600_dma_fini(struct radeon_device *rdev) -{ - r600_dma_stop(rdev); - radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]); -} - /* * GPU scratch registers helpers function. */ @@ -2440,7 +2835,7 @@ int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2)); radeon_ring_write(ring, 0xDEADBEEF); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); for (i = 0; i < rdev->usec_timeout; i++) { tmp = RREG32(scratch); if (tmp == 0xDEADBEEF) @@ -2458,60 +2853,6 @@ int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) return r; } -/** - * r600_dma_ring_test - simple async dma engine test - * - * @rdev: radeon_device pointer - * @ring: radeon_ring structure holding ring information - * - * Test the DMA engine by writing using it to write an - * value to memory. (r6xx-SI). - * Returns 0 for success, error for failure. - */ -int r600_dma_ring_test(struct radeon_device *rdev, - struct radeon_ring *ring) -{ - unsigned i; - int r; - volatile uint32_t *ptr = rdev->vram_scratch.ptr; - u32 tmp; - - if (!ptr) { - DRM_ERROR("invalid vram scratch pointer\n"); - return -EINVAL; - } - - tmp = 0xCAFEDEAD; - *ptr = tmp; - - r = radeon_ring_lock(rdev, ring, 4); - if (r) { - DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r); - return r; - } - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1)); - radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff); - radeon_ring_write(ring, 0xDEADBEEF); - radeon_ring_unlock_commit(rdev, ring); - - for (i = 0; i < rdev->usec_timeout; i++) { - tmp = *ptr; - if (tmp == 0xDEADBEEF) - break; - DRM_UDELAY(1); - } - - if (i < rdev->usec_timeout) { - DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); - } else { - DRM_ERROR("radeon: ring %d test failed (0x%08X)\n", - ring->idx, tmp); - r = -EINVAL; - } - return r; -} - /* * CP fences/semaphores */ @@ -2537,7 +2878,7 @@ void r600_fence_ring_emit(struct radeon_device *rdev, /* EVENT_WRITE_EOP - flush caches, send int */ radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5)); - radeon_ring_write(ring, addr & 0xffffffff); + radeon_ring_write(ring, lower_32_bits(addr)); radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); radeon_ring_write(ring, fence->seq); radeon_ring_write(ring, 0); @@ -2564,7 +2905,18 @@ void r600_fence_ring_emit(struct radeon_device *rdev, } } -void r600_semaphore_ring_emit(struct radeon_device *rdev, +/** + * r600_semaphore_ring_emit - emit a semaphore on the CP ring + * + * @rdev: radeon_device pointer + * @ring: radeon ring buffer object + * @semaphore: radeon semaphore object + * @emit_wait: Is this a sempahore wait? + * + * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP + * from running ahead of semaphore waits. + */ +bool r600_semaphore_ring_emit(struct radeon_device *rdev, struct radeon_ring *ring, struct radeon_semaphore *semaphore, bool emit_wait) @@ -2576,84 +2928,21 @@ void r600_semaphore_ring_emit(struct radeon_device *rdev, sel |= PACKET3_SEM_WAIT_ON_SIGNAL; radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1)); - radeon_ring_write(ring, addr & 0xffffffff); + radeon_ring_write(ring, lower_32_bits(addr)); radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel); -} - -/* - * DMA fences/semaphores - */ - -/** - * r600_dma_fence_ring_emit - emit a fence on the DMA ring - * - * @rdev: radeon_device pointer - * @fence: radeon fence object - * - * Add a DMA fence packet to the ring to write - * the fence seq number and DMA trap packet to generate - * an interrupt if needed (r6xx-r7xx). - */ -void r600_dma_fence_ring_emit(struct radeon_device *rdev, - struct radeon_fence *fence) -{ - struct radeon_ring *ring = &rdev->ring[fence->ring]; - u64 addr = rdev->fence_drv[fence->ring].gpu_addr; - - /* write the fence */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0)); - radeon_ring_write(ring, addr & 0xfffffffc); - radeon_ring_write(ring, (upper_32_bits(addr) & 0xff)); - radeon_ring_write(ring, lower_32_bits(fence->seq)); - /* generate an interrupt */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0)); -} -/** - * r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring - * - * @rdev: radeon_device pointer - * @ring: radeon_ring structure holding ring information - * @semaphore: radeon semaphore object - * @emit_wait: wait or signal semaphore - * - * Add a DMA semaphore packet to the ring wait on or signal - * other rings (r6xx-SI). - */ -void r600_dma_semaphore_ring_emit(struct radeon_device *rdev, - struct radeon_ring *ring, - struct radeon_semaphore *semaphore, - bool emit_wait) -{ - u64 addr = semaphore->gpu_addr; - u32 s = emit_wait ? 0 : 1; - - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0)); - radeon_ring_write(ring, addr & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(addr) & 0xff); -} - -int r600_copy_blit(struct radeon_device *rdev, - uint64_t src_offset, - uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence) -{ - struct radeon_semaphore *sem = NULL; - struct radeon_sa_bo *vb = NULL; - int r; - - r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem); - if (r) { - return r; + /* PFP_SYNC_ME packet only exists on 7xx+, only enable it on eg+ */ + if (emit_wait && (rdev->family >= CHIP_CEDAR)) { + /* Prevent the PFP from running ahead of the semaphore wait */ + radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); + radeon_ring_write(ring, 0x0); } - r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb); - r600_blit_done_copy(rdev, fence, vb, sem); - return 0; + + return true; } /** - * r600_copy_dma - copy pages using the DMA engine + * r600_copy_cpdma - copy pages using the CP DMA engine * * @rdev: radeon_device pointer * @src_offset: src GPU address @@ -2661,69 +2950,72 @@ int r600_copy_blit(struct radeon_device *rdev, * @num_gpu_pages: number of GPU pages to xfer * @fence: radeon fence object * - * Copy GPU paging using the DMA engine (r6xx). + * Copy GPU paging using the CP DMA engine (r6xx+). * Used by the radeon ttm implementation to move pages if * registered as the asic copy callback. */ -int r600_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence) +struct radeon_fence *r600_copy_cpdma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv) { - struct radeon_semaphore *sem = NULL; - int ring_index = rdev->asic->copy.dma_ring_index; + struct radeon_fence *fence; + struct radeon_sync sync; + int ring_index = rdev->asic->copy.blit_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; - u32 size_in_dw, cur_size_in_dw; + u32 size_in_bytes, cur_size_in_bytes, tmp; int i, num_loops; int r = 0; - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return r; - } + radeon_sync_create(&sync); - size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; - num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE); - r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8); + size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); + num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); + r = radeon_ring_lock(rdev, ring, num_loops * 6 + 24); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); - return r; + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); } - if (radeon_fence_need_sync(*fence, ring->idx)) { - radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring, - ring->idx); - radeon_fence_note_sync(*fence, ring->idx); - } else { - radeon_semaphore_free(rdev, &sem, NULL); - } + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); + radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); + radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); + radeon_ring_write(ring, WAIT_3D_IDLE_bit); for (i = 0; i < num_loops; i++) { - cur_size_in_dw = size_in_dw; - if (cur_size_in_dw > 0xFFFE) - cur_size_in_dw = 0xFFFE; - size_in_dw -= cur_size_in_dw; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw)); - radeon_ring_write(ring, dst_offset & 0xfffffffc); - radeon_ring_write(ring, src_offset & 0xfffffffc); - radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) | - (upper_32_bits(src_offset) & 0xff))); - src_offset += cur_size_in_dw * 4; - dst_offset += cur_size_in_dw * 4; - } - - r = radeon_fence_emit(rdev, fence, ring->idx); + cur_size_in_bytes = size_in_bytes; + if (cur_size_in_bytes > 0x1fffff) + cur_size_in_bytes = 0x1fffff; + size_in_bytes -= cur_size_in_bytes; + tmp = upper_32_bits(src_offset) & 0xff; + if (size_in_bytes == 0) + tmp |= PACKET3_CP_DMA_CP_SYNC; + radeon_ring_write(ring, PACKET3(PACKET3_CP_DMA, 4)); + radeon_ring_write(ring, lower_32_bits(src_offset)); + radeon_ring_write(ring, tmp); + radeon_ring_write(ring, lower_32_bits(dst_offset)); + radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); + radeon_ring_write(ring, cur_size_in_bytes); + src_offset += cur_size_in_bytes; + dst_offset += cur_size_in_bytes; + } + radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); + radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); + radeon_ring_write(ring, WAIT_CP_DMA_IDLE_bit); + + r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); - return r; + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); } - radeon_ring_unlock_commit(rdev, ring); - radeon_semaphore_free(rdev, &sem, *fence); + radeon_ring_unlock_commit(rdev, ring, false); + radeon_sync_free(rdev, &sync, fence); - return r; + return fence; } int r600_set_surface_reg(struct radeon_device *rdev, int reg, @@ -2747,20 +3039,13 @@ static int r600_startup(struct radeon_device *rdev) /* enable pcie gen2 link */ r600_pcie_gen2_enable(rdev); - r600_mc_program(rdev); - - if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { - r = r600_init_microcode(rdev); - if (r) { - DRM_ERROR("Failed to load firmware!\n"); - return r; - } - } - + /* scratch needs to be initialized before MC */ r = r600_vram_scratch_init(rdev); if (r) return r; + r600_mc_program(rdev); + if (rdev->flags & RADEON_IS_AGP) { r600_agp_enable(rdev); } else { @@ -2769,12 +3054,6 @@ static int r600_startup(struct radeon_device *rdev) return r; } r600_gpu_init(rdev); - r = r600_blit_init(rdev); - if (r) { - r600_blit_fini(rdev); - rdev->asic->copy.copy = NULL; - dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); - } /* allocate wb buffer */ r = radeon_wb_init(rdev); @@ -2787,10 +3066,16 @@ static int r600_startup(struct radeon_device *rdev) return r; } - r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); - if (r) { - dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); - return r; + if (rdev->has_uvd) { + r = uvd_v1_0_resume(rdev); + if (!r) { + r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r); + } + } + if (r) + rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; } /* Enable IRQ */ @@ -2810,15 +3095,7 @@ static int r600_startup(struct radeon_device *rdev) ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, - R600_CP_RB_RPTR, R600_CP_RB_WPTR, - 0, 0xfffff, RADEON_CP_PACKET2); - if (r) - return r; - - ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; - r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, - DMA_RB_RPTR, DMA_RB_WPTR, - 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); + RADEON_CP_PACKET2); if (r) return r; @@ -2829,9 +3106,17 @@ static int r600_startup(struct radeon_device *rdev) if (r) return r; - r = r600_dma_resume(rdev); - if (r) - return r; + if (rdev->has_uvd) { + ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + if (ring->ring_size) { + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + RADEON_CP_PACKET2); + if (!r) + r = uvd_v1_0_init(rdev); + if (r) + DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); + } + } r = radeon_ib_pool_init(rdev); if (r) { @@ -2839,7 +3124,7 @@ static int r600_startup(struct radeon_device *rdev) return r; } - r = r600_audio_init(rdev); + r = radeon_audio_init(rdev); if (r) { DRM_ERROR("radeon: audio init failed\n"); return r; @@ -2873,6 +3158,9 @@ int r600_resume(struct radeon_device *rdev) /* post card */ atom_asic_init(rdev->mode_info.atom_context); + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_resume(rdev); + rdev->accel_working = true; r = r600_startup(rdev); if (r) { @@ -2886,9 +3174,13 @@ int r600_resume(struct radeon_device *rdev) int r600_suspend(struct radeon_device *rdev) { - r600_audio_fini(rdev); + radeon_pm_suspend(rdev); + radeon_audio_fini(rdev); r600_cp_stop(rdev); - r600_dma_stop(rdev); + if (rdev->has_uvd) { + uvd_v1_0_fini(rdev); + radeon_uvd_suspend(rdev); + } r600_irq_suspend(rdev); radeon_wb_disable(rdev); r600_pcie_gart_disable(rdev); @@ -2954,11 +3246,27 @@ int r600_init(struct radeon_device *rdev) if (r) return r; + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { + r = r600_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); - rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; - r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); + if (rdev->has_uvd) { + r = radeon_uvd_init(rdev); + if (!r) { + rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; + r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096); + } + } rdev->ih.ring_obj = NULL; r600_ih_ring_init(rdev, 64 * 1024); @@ -2972,7 +3280,6 @@ int r600_init(struct radeon_device *rdev) if (r) { dev_err(rdev->dev, "disabling GPU acceleration\n"); r600_cp_fini(rdev); - r600_dma_fini(rdev); r600_irq_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); @@ -2986,11 +3293,14 @@ int r600_init(struct radeon_device *rdev) void r600_fini(struct radeon_device *rdev) { - r600_audio_fini(rdev); - r600_blit_fini(rdev); + radeon_pm_fini(rdev); + radeon_audio_fini(rdev); r600_cp_fini(rdev); - r600_dma_fini(rdev); r600_irq_fini(rdev); + if (rdev->has_uvd) { + uvd_v1_0_fini(rdev); + radeon_uvd_fini(rdev); + } radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); radeon_irq_kms_fini(rdev); @@ -3062,7 +3372,7 @@ int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); ib.ptr[2] = 0xDEADBEEF; ib.length_dw = 3; - r = radeon_ib_schedule(rdev, &ib, NULL); + r = radeon_ib_schedule(rdev, &ib, NULL, false); if (r) { DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); goto free_ib; @@ -3092,104 +3402,6 @@ free_scratch: return r; } -/** - * r600_dma_ib_test - test an IB on the DMA engine - * - * @rdev: radeon_device pointer - * @ring: radeon_ring structure holding ring information - * - * Test a simple IB in the DMA ring (r6xx-SI). - * Returns 0 on success, error on failure. - */ -int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) -{ - struct radeon_ib ib; - unsigned i; - int r; - volatile uint32_t *ptr = rdev->vram_scratch.ptr; - u32 tmp = 0; - - if (!ptr) { - DRM_ERROR("invalid vram scratch pointer\n"); - return -EINVAL; - } - - tmp = 0xCAFEDEAD; - *ptr = tmp; - - r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); - if (r) { - DRM_ERROR("radeon: failed to get ib (%d).\n", r); - return r; - } - - ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1); - ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc; - ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff; - ib.ptr[3] = 0xDEADBEEF; - ib.length_dw = 4; - - r = radeon_ib_schedule(rdev, &ib, NULL); - if (r) { - radeon_ib_free(rdev, &ib); - DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); - return r; - } - r = radeon_fence_wait(ib.fence, false); - if (r) { - DRM_ERROR("radeon: fence wait failed (%d).\n", r); - return r; - } - for (i = 0; i < rdev->usec_timeout; i++) { - tmp = *ptr; - if (tmp == 0xDEADBEEF) - break; - DRM_UDELAY(1); - } - if (i < rdev->usec_timeout) { - DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); - } else { - DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp); - r = -EINVAL; - } - radeon_ib_free(rdev, &ib); - return r; -} - -/** - * r600_dma_ring_ib_execute - Schedule an IB on the DMA engine - * - * @rdev: radeon_device pointer - * @ib: IB object to schedule - * - * Schedule an IB in the DMA ring (r6xx-r7xx). - */ -void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) -{ - struct radeon_ring *ring = &rdev->ring[ib->ring]; - - if (rdev->wb.enabled) { - u32 next_rptr = ring->wptr + 4; - while ((next_rptr & 7) != 5) - next_rptr++; - next_rptr += 3; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1)); - radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff); - radeon_ring_write(ring, next_rptr); - } - - /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. - * Pad as necessary with NOPs. - */ - while ((ring->wptr & 7) != 5) - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0)); - radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); - radeon_ring_write(ring, (ib->length_dw << 16) | (upper_32_bits(ib->gpu_addr) & 0xFF)); - -} - /* * Interrupts * @@ -3206,7 +3418,7 @@ void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size) u32 rb_bufsz; /* Align ring size */ - rb_bufsz = drm_order(ring_size / 4); + rb_bufsz = order_base_2(ring_size / 4); ring_size = (1 << rb_bufsz) * 4; rdev->ih.ring_size = ring_size; rdev->ih.ptr_mask = rdev->ih.ring_size - 1; @@ -3221,8 +3433,8 @@ int r600_ih_ring_alloc(struct radeon_device *rdev) if (rdev->ih.ring_obj == NULL) { r = radeon_bo_create(rdev, rdev->ih.ring_size, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_GTT, - NULL, &rdev->ih.ring_obj); + RADEON_GEM_DOMAIN_GTT, 0, + NULL, NULL, &rdev->ih.ring_obj); if (r) { DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r); return r; @@ -3286,7 +3498,7 @@ static void r600_rlc_start(struct radeon_device *rdev) WREG32(RLC_CNTL, RLC_ENABLE); } -static int r600_rlc_init(struct radeon_device *rdev) +static int r600_rlc_resume(struct radeon_device *rdev) { u32 i; const __be32 *fw_data; @@ -3298,45 +3510,22 @@ static int r600_rlc_init(struct radeon_device *rdev) WREG32(RLC_HB_CNTL, 0); - if (rdev->family == CHIP_ARUBA) { - WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); - WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); - } - if (rdev->family <= CHIP_CAYMAN) { - WREG32(RLC_HB_BASE, 0); - WREG32(RLC_HB_RPTR, 0); - WREG32(RLC_HB_WPTR, 0); - } - if (rdev->family <= CHIP_CAICOS) { - WREG32(RLC_HB_WPTR_LSB_ADDR, 0); - WREG32(RLC_HB_WPTR_MSB_ADDR, 0); - } + WREG32(RLC_HB_BASE, 0); + WREG32(RLC_HB_RPTR, 0); + WREG32(RLC_HB_WPTR, 0); + WREG32(RLC_HB_WPTR_LSB_ADDR, 0); + WREG32(RLC_HB_WPTR_MSB_ADDR, 0); WREG32(RLC_MC_CNTL, 0); WREG32(RLC_UCODE_CNTL, 0); - fw_data = (const __be32 *)rdev->rlc_fw; - if (rdev->family >= CHIP_ARUBA) { - for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) { - WREG32(RLC_UCODE_ADDR, i); - WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); - } - } else if (rdev->family >= CHIP_CAYMAN) { - for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) { - WREG32(RLC_UCODE_ADDR, i); - WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); - } - } else if (rdev->family >= CHIP_CEDAR) { - for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) { - WREG32(RLC_UCODE_ADDR, i); - WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); - } - } else if (rdev->family >= CHIP_RV770) { + fw_data = (const __be32 *)rdev->rlc_fw->data; + if (rdev->family >= CHIP_RV770) { for (i = 0; i < R700_RLC_UCODE_SIZE; i++) { WREG32(RLC_UCODE_ADDR, i); WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); } } else { - for (i = 0; i < RLC_UCODE_SIZE; i++) { + for (i = 0; i < R600_RLC_UCODE_SIZE; i++) { WREG32(RLC_UCODE_ADDR, i); WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); } @@ -3444,7 +3633,10 @@ int r600_irq_init(struct radeon_device *rdev) r600_disable_interrupts(rdev); /* init rlc */ - ret = r600_rlc_init(rdev); + if (rdev->family >= CHIP_CEDAR) + ret = evergreen_rlc_resume(rdev); + else + ret = r600_rlc_resume(rdev); if (ret) { r600_ih_ring_fini(rdev); return ret; @@ -3463,7 +3655,7 @@ int r600_irq_init(struct radeon_device *rdev) WREG32(INTERRUPT_CNTL, interrupt_cntl); WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); - rb_bufsz = drm_order(rdev->ih.ring_size / 4); + rb_bufsz = order_base_2(rdev->ih.ring_size / 4); ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | IH_WPTR_OVERFLOW_CLEAR | @@ -3496,9 +3688,7 @@ int r600_irq_init(struct radeon_device *rdev) r600_disable_interrupt_state(rdev); /* at this point everything should be setup correctly to enable master */ -#ifdef notyet pci_set_master(rdev->pdev); -#endif /* enable irqs */ r600_enable_interrupts(rdev); @@ -3525,8 +3715,8 @@ int r600_irq_set(struct radeon_device *rdev) u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0; u32 grbm_int_cntl = 0; u32 hdmi0, hdmi1; - u32 d1grph = 0, d2grph = 0; u32 dma_cntl; + u32 thermal_int = 0; if (!rdev->irq.installed) { WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); @@ -3561,8 +3751,21 @@ int r600_irq_set(struct radeon_device *rdev) hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK; hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK; } + dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; + if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) { + thermal_int = RREG32(CG_THERMAL_INT) & + ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + } else if (rdev->family >= CHIP_RV770) { + thermal_int = RREG32(RV770_CG_THERMAL_INT) & + ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + } + if (rdev->irq.dpm_thermal) { + DRM_DEBUG("dpm thermal\n"); + thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; + } + if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { DRM_DEBUG("r600_irq_set: sw int\n"); cp_int_cntl |= RB_INT_ENABLE; @@ -3620,8 +3823,8 @@ int r600_irq_set(struct radeon_device *rdev) WREG32(CP_INT_CNTL, cp_int_cntl); WREG32(DMA_CNTL, dma_cntl); WREG32(DxMODE_INT_MASK, mode_int); - WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph); - WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph); + WREG32(D1GRPH_INTERRUPT_CONTROL, DxGRPH_PFLIP_INT_MASK); + WREG32(D2GRPH_INTERRUPT_CONTROL, DxGRPH_PFLIP_INT_MASK); WREG32(GRBM_INT_CNTL, grbm_int_cntl); if (ASIC_IS_DCE3(rdev)) { WREG32(DC_HPD1_INT_CONTROL, hpd1); @@ -3644,6 +3847,14 @@ int r600_irq_set(struct radeon_device *rdev) WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0); WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1); } + if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) { + WREG32(CG_THERMAL_INT, thermal_int); + } else if (rdev->family >= CHIP_RV770) { + WREG32(RV770_CG_THERMAL_INT, thermal_int); + } + + /* posting read */ + RREG32(R_000E50_SRBM_STATUS); return 0; } @@ -3730,7 +3941,7 @@ static void r600_irq_ack(struct radeon_device *rdev) WREG32(DC_HPD5_INT_CONTROL, tmp); } if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) { - tmp = RREG32(DC_HPD5_INT_CONTROL); + tmp = RREG32(DC_HPD6_INT_CONTROL); tmp |= DC_HPDx_INT_ACK; WREG32(DC_HPD6_INT_CONTROL, tmp); } @@ -3783,12 +3994,13 @@ static u32 r600_get_ih_wptr(struct radeon_device *rdev) wptr = RREG32(IH_RB_WPTR); if (wptr & RB_OVERFLOW) { + wptr &= ~RB_OVERFLOW; /* When a ring buffer overflow happen start parsing interrupt * from the last not overwritten vector (wptr + 16). Hopefully * this should allow us to catchup. */ - dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", - wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); + dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; tmp = RREG32(IH_RB_CNTL); tmp |= IH_WPTR_OVERFLOW_CLEAR; @@ -3835,6 +4047,7 @@ int r600_irq_process(struct radeon_device *rdev) u32 ring_index; bool queue_hotplug = false; bool queue_hdmi = false; + bool queue_thermal = false; if (!rdev->ih.enabled || rdev->shutdown) return IRQ_NONE; @@ -3845,8 +4058,6 @@ int r600_irq_process(struct radeon_device *rdev) wptr = r600_get_ih_wptr(rdev); - if (wptr == rdev->ih.rptr) - return IRQ_NONE; restart_ih: /* is somebody else already processing irqs? */ if (atomic_xchg(&rdev->ih.lock, 1)) @@ -3871,23 +4082,27 @@ restart_ih: case 1: /* D1 vblank/vline */ switch (src_data) { case 0: /* D1 vblank */ - if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[0]) { - drm_handle_vblank(rdev->ddev, 0); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[0])) - radeon_crtc_handle_flip(rdev, 0); - rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D1 vblank\n"); + if (!(rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[0]) { + drm_handle_vblank(rdev->ddev, 0); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[0])) + radeon_crtc_handle_vblank(rdev, 0); + rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D1 vblank\n"); + break; case 1: /* D1 vline */ - if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) { - rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT; - DRM_DEBUG("IH: D1 vline\n"); - } + if (!(rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)) + DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT; + DRM_DEBUG("IH: D1 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3897,72 +4112,93 @@ restart_ih: case 5: /* D2 vblank/vline */ switch (src_data) { case 0: /* D2 vblank */ - if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[1]) { - drm_handle_vblank(rdev->ddev, 1); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[1])) - radeon_crtc_handle_flip(rdev, 1); - rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D2 vblank\n"); + if (!(rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[1]) { + drm_handle_vblank(rdev->ddev, 1); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[1])) + radeon_crtc_handle_vblank(rdev, 1); + rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D2 vblank\n"); + break; case 1: /* D1 vline */ - if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) { - rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT; - DRM_DEBUG("IH: D2 vline\n"); - } + if (!(rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)) + DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT; + DRM_DEBUG("IH: D2 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); break; } break; + case 9: /* D1 pflip */ + DRM_DEBUG("IH: D1 flip\n"); + if (radeon_use_pflipirq > 0) + radeon_crtc_handle_flip(rdev, 0); + break; + case 11: /* D2 pflip */ + DRM_DEBUG("IH: D2 flip\n"); + if (radeon_use_pflipirq > 0) + radeon_crtc_handle_flip(rdev, 1); + break; case 19: /* HPD/DAC hotplug */ switch (src_data) { case 0: - if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) { - rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD1\n"); - } + if (!(rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT)) + DRM_DEBUG("IH: HPD1 - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD1\n"); break; case 1: - if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) { - rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD2\n"); - } + if (!(rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT)) + DRM_DEBUG("IH: HPD2 - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD2\n"); break; case 4: - if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) { - rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD3\n"); - } + if (!(rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT)) + DRM_DEBUG("IH: HPD3 - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD3\n"); break; case 5: - if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) { - rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD4\n"); - } + if (!(rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT)) + DRM_DEBUG("IH: HPD4 - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD4\n"); break; case 10: - if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) { - rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD5\n"); - } + if (!(rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT)) + DRM_DEBUG("IH: HPD5 - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD5\n"); break; case 12: - if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) { - rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD6\n"); - } + if (!(rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT)) + DRM_DEBUG("IH: HPD6 - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD6\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3972,24 +4208,32 @@ restart_ih: case 21: /* hdmi */ switch (src_data) { case 4: - if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) { - rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG; - queue_hdmi = true; - DRM_DEBUG("IH: HDMI0\n"); - } + if (!(rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG)) + DRM_DEBUG("IH: HDMI0 - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG; + queue_hdmi = true; + DRM_DEBUG("IH: HDMI0\n"); + break; case 5: - if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) { - rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG; - queue_hdmi = true; - DRM_DEBUG("IH: HDMI1\n"); - } + if (!(rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG)) + DRM_DEBUG("IH: HDMI1 - IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG; + queue_hdmi = true; + DRM_DEBUG("IH: HDMI1\n"); + break; default: DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); break; } break; + case 124: /* UVD */ + DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); + radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); + break; case 176: /* CP_INT in ring buffer */ case 177: /* CP_INT in IB1 */ case 178: /* CP_INT in IB2 */ @@ -4004,6 +4248,16 @@ restart_ih: DRM_DEBUG("IH: DMA trap\n"); radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); break; + case 230: /* thermal low to high */ + DRM_DEBUG("IH: thermal low to high\n"); + rdev->pm.dpm.thermal.high_to_low = false; + queue_thermal = true; + break; + case 231: /* thermal high to low */ + DRM_DEBUG("IH: thermal high to low\n"); + rdev->pm.dpm.thermal.high_to_low = true; + queue_thermal = true; + break; case 233: /* GUI IDLE */ DRM_DEBUG("IH: GUI idle\n"); break; @@ -4015,13 +4269,15 @@ restart_ih: /* wptr/rptr are in bytes! */ rptr += 16; rptr &= rdev->ih.ptr_mask; + WREG32(IH_RB_RPTR, rptr); } if (queue_hotplug) - task_add(systq, &rdev->hotplug_task); + schedule_delayed_work(&rdev->hotplug_work, 0); if (queue_hdmi) - task_add(systq, &rdev->audio_task); + schedule_work(&rdev->audio_work); + if (queue_thermal && rdev->pm.dpm_enabled) + schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; - WREG32(IH_RB_RPTR, rdev->ih.rptr); atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ @@ -4063,16 +4319,15 @@ int r600_debugfs_mc_info_init(struct radeon_device *rdev) } /** - * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl + * r600_mmio_hdp_flush - flush Host Data Path cache via MMIO * rdev: radeon device structure - * bo: buffer object struct which userspace is waiting for idle * - * Some R6XX/R7XX doesn't seems to take into account HDP flush performed - * through ring buffer, this leads to corruption in rendering, see - * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we - * directly perform HDP flush by writing register through MMIO. + * Some R6XX/R7XX don't seem to take into account HDP flushes performed + * through the ring buffer. This leads to corruption in rendering, see + * http://bugzilla.kernel.org/show_bug.cgi?id=15186 . To avoid this, we + * directly perform the HDP flush by writing the register through MMIO. */ -void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo) +void r600_mmio_hdp_flush(struct radeon_device *rdev) { /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL. @@ -4081,18 +4336,18 @@ void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo) */ if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) && rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) { - volatile uint32_t *ptr = rdev->vram_scratch.ptr; + void __iomem *ptr = (void *)rdev->vram_scratch.ptr; u32 tmp; WREG32(HDP_DEBUG1, 0); - tmp = *ptr; + tmp = readl((void __iomem *)ptr); } else WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); } void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes) { - u32 link_width_cntl, mask, target_reg; + u32 link_width_cntl, mask; if (rdev->flags & RADEON_IS_IGP) return; @@ -4104,7 +4359,7 @@ void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes) if (ASIC_IS_X2(rdev)) return; - /* FIXME wait for idle */ + radeon_gui_idle(rdev); switch (lanes) { case 0: @@ -4123,53 +4378,24 @@ void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes) mask = RADEON_PCIE_LC_LINK_WIDTH_X8; break; case 12: + /* not actually supported */ mask = RADEON_PCIE_LC_LINK_WIDTH_X12; break; case 16: - default: mask = RADEON_PCIE_LC_LINK_WIDTH_X16; break; - } - - link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL); - - if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) == - (mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT)) - return; - - if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS) + default: + DRM_ERROR("invalid pcie lane request: %d\n", lanes); return; + } - link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK | - RADEON_PCIE_LC_RECONFIG_NOW | - R600_PCIE_LC_RENEGOTIATE_EN | - R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE); - link_width_cntl |= mask; - - WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); - - /* some northbridges can renegotiate the link rather than requiring - * a complete re-config. - * e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.) - */ - if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT) - link_width_cntl |= R600_PCIE_LC_RENEGOTIATE_EN | R600_PCIE_LC_UPCONFIGURE_SUPPORT; - else - link_width_cntl |= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE; - - WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl | - RADEON_PCIE_LC_RECONFIG_NOW)); - - if (rdev->family >= CHIP_RV770) - target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX; - else - target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX; - - /* wait for lane set to complete */ - link_width_cntl = RREG32(target_reg); - while (link_width_cntl == 0xffffffff) - link_width_cntl = RREG32(target_reg); + link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL); + link_width_cntl &= ~RADEON_PCIE_LC_LINK_WIDTH_MASK; + link_width_cntl |= mask << RADEON_PCIE_LC_LINK_WIDTH_SHIFT; + link_width_cntl |= (RADEON_PCIE_LC_RECONFIG_NOW | + R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE); + WREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); } int r600_get_pcie_lanes(struct radeon_device *rdev) @@ -4186,13 +4412,11 @@ int r600_get_pcie_lanes(struct radeon_device *rdev) if (ASIC_IS_X2(rdev)) return 0; - /* FIXME wait for idle */ + radeon_gui_idle(rdev); - link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL); + link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL); switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) { - case RADEON_PCIE_LC_LINK_WIDTH_X0: - return 0; case RADEON_PCIE_LC_LINK_WIDTH_X1: return 1; case RADEON_PCIE_LC_LINK_WIDTH_X2: @@ -4201,6 +4425,10 @@ int r600_get_pcie_lanes(struct radeon_device *rdev) return 4; case RADEON_PCIE_LC_LINK_WIDTH_X8: return 8; + case RADEON_PCIE_LC_LINK_WIDTH_X12: + /* not actually supported */ + return 12; + case RADEON_PCIE_LC_LINK_WIDTH_X0: case RADEON_PCIE_LC_LINK_WIDTH_X16: default: return 16; @@ -4212,7 +4440,6 @@ static void r600_pcie_gen2_enable(struct radeon_device *rdev) u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp; u16 link_cntl2; u32 mask; - int ret; if (radeon_pcie_gen2 == 0) return; @@ -4231,14 +4458,13 @@ static void r600_pcie_gen2_enable(struct radeon_device *rdev) if (rdev->family <= CHIP_R600) return; - ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); - if (ret != 0) + if (drm_pcie_get_speed_cap_mask(rdev->ddev, &mask)) return; - if (!(mask & DRM_PCIE_SPEED_50)) + if (!(mask & (DRM_PCIE_SPEED_50|DRM_PCIE_SPEED_80))) return; - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); if (speed_cntl & LC_CURRENT_DATA_RATE) { DRM_INFO("PCIE gen 2 link speeds already enabled\n"); return; @@ -4251,23 +4477,23 @@ static void r600_pcie_gen2_enable(struct radeon_device *rdev) (rdev->family == CHIP_RV620) || (rdev->family == CHIP_RV635)) { /* advertise upconfig capability */ - link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); + link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); link_width_cntl &= ~LC_UPCONFIGURE_DIS; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); - link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) { lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT; link_width_cntl &= ~(LC_LINK_WIDTH_MASK | LC_RECONFIG_ARC_MISSING_ESCAPE); link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); } else { link_width_cntl |= LC_UPCONFIGURE_DIS; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); } } - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) && (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { @@ -4288,7 +4514,7 @@ static void r600_pcie_gen2_enable(struct radeon_device *rdev) speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK; speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE; speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); tmp = RREG32(0x541c); WREG32(0x541c, tmp | 0x8); @@ -4302,39 +4528,39 @@ static void r600_pcie_gen2_enable(struct radeon_device *rdev) if ((rdev->family == CHIP_RV670) || (rdev->family == CHIP_RV620) || (rdev->family == CHIP_RV635)) { - training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL); + training_cntl = RREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL); training_cntl &= ~LC_POINT_7_PLUS_EN; - WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl); + WREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL, training_cntl); } else { - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); } - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl |= LC_GEN2_EN_STRAP; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); } else { - link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); + link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ if (1) link_width_cntl |= LC_UPCONFIGURE_DIS; else link_width_cntl &= ~LC_UPCONFIGURE_DIS; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); } } /** - * r600_get_gpu_clock - return GPU clock counter snapshot + * r600_get_gpu_clock_counter - return GPU clock counter snapshot * * @rdev: radeon_device pointer * * Fetches a GPU clock counter snapshot (R6xx-cayman). * Returns the 64 bit clock counter snapshot. */ -uint64_t r600_get_gpu_clock(struct radeon_device *rdev) +uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev) { uint64_t clock; diff --git a/sys/dev/pci/drm/radeon/r600_audio.c b/sys/dev/pci/drm/radeon/r600_audio.c deleted file mode 100644 index b7aecc05ea3..00000000000 --- a/sys/dev/pci/drm/radeon/r600_audio.c +++ /dev/null @@ -1,255 +0,0 @@ -/* $OpenBSD: r600_audio.c,v 1.5 2018/04/20 21:12:50 naddy Exp $ */ -/* - * Copyright 2008 Advanced Micro Devices, Inc. - * Copyright 2008 Red Hat Inc. - * Copyright 2009 Christian König. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Christian König - */ -#include -#include "radeon.h" -#include "radeon_reg.h" -#include "radeon_asic.h" -#include "atom.h" - -/* - * check if enc_priv stores radeon_encoder_atom_dig - */ -static bool radeon_dig_encoder(struct drm_encoder *encoder) -{ - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - switch (radeon_encoder->encoder_id) { - case ENCODER_OBJECT_ID_INTERNAL_LVDS: - case ENCODER_OBJECT_ID_INTERNAL_TMDS1: - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1: - case ENCODER_OBJECT_ID_INTERNAL_LVTM1: - case ENCODER_OBJECT_ID_INTERNAL_DVO1: - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: - case ENCODER_OBJECT_ID_INTERNAL_DDI: - case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: - case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: - case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: - return true; - } - return false; -} - -/* - * check if the chipset is supported - */ -static int r600_audio_chipset_supported(struct radeon_device *rdev) -{ - return (rdev->family >= CHIP_R600 && !ASIC_IS_DCE6(rdev)) - || rdev->family == CHIP_RS600 - || rdev->family == CHIP_RS690 - || rdev->family == CHIP_RS740; -} - -struct r600_audio r600_audio_status(struct radeon_device *rdev) -{ - struct r600_audio status; - uint32_t value; - - value = RREG32(R600_AUDIO_RATE_BPS_CHANNEL); - - /* number of channels */ - status.channels = (value & 0x7) + 1; - - /* bits per sample */ - switch ((value & 0xF0) >> 4) { - case 0x0: - status.bits_per_sample = 8; - break; - case 0x1: - status.bits_per_sample = 16; - break; - case 0x2: - status.bits_per_sample = 20; - break; - case 0x3: - status.bits_per_sample = 24; - break; - case 0x4: - status.bits_per_sample = 32; - break; - default: - dev_err(rdev->dev, "Unknown bits per sample 0x%x, using 16\n", - (int)value); - status.bits_per_sample = 16; - } - - /* current sampling rate in HZ */ - if (value & 0x4000) - status.rate = 44100; - else - status.rate = 48000; - status.rate *= ((value >> 11) & 0x7) + 1; - status.rate /= ((value >> 8) & 0x7) + 1; - - value = RREG32(R600_AUDIO_STATUS_BITS); - - /* iec 60958 status bits */ - status.status_bits = value & 0xff; - - /* iec 60958 category code */ - status.category_code = (value >> 8) & 0xff; - - return status; -} - -/* - * update all hdmi interfaces with current audio parameters - */ -void r600_audio_update_hdmi(void *arg1) -{ - struct radeon_device *rdev = arg1; - struct drm_device *dev = rdev->ddev; - struct r600_audio audio_status = r600_audio_status(rdev); - struct drm_encoder *encoder; - bool changed = false; - - if (rdev->audio_status.channels != audio_status.channels || - rdev->audio_status.rate != audio_status.rate || - rdev->audio_status.bits_per_sample != audio_status.bits_per_sample || - rdev->audio_status.status_bits != audio_status.status_bits || - rdev->audio_status.category_code != audio_status.category_code) { - rdev->audio_status = audio_status; - changed = true; - } - - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { - if (!radeon_dig_encoder(encoder)) - continue; - if (changed || r600_hdmi_buffer_status_changed(encoder)) - r600_hdmi_update_audio_settings(encoder); - } -} - -/* - * turn on/off audio engine - */ -static void r600_audio_engine_enable(struct radeon_device *rdev, bool enable) -{ - u32 value = 0; - DRM_INFO("%s audio support\n", enable ? "Enabling" : "Disabling"); - if (ASIC_IS_DCE4(rdev)) { - if (enable) { - value |= 0x81000000; /* Required to enable audio */ - value |= 0x0e1000f0; /* fglrx sets that too */ - } - WREG32(EVERGREEN_AUDIO_ENABLE, value); - } else { - WREG32_P(R600_AUDIO_ENABLE, - enable ? 0x81000000 : 0x0, ~0x81000000); - } - rdev->audio_enabled = enable; -} - -/* - * initialize the audio vars - */ -int r600_audio_init(struct radeon_device *rdev) -{ - if (!radeon_audio || !r600_audio_chipset_supported(rdev)) - return 0; - - r600_audio_engine_enable(rdev, true); - - rdev->audio_status.channels = -1; - rdev->audio_status.rate = -1; - rdev->audio_status.bits_per_sample = -1; - rdev->audio_status.status_bits = 0; - rdev->audio_status.category_code = 0; - - return 0; -} - -/* - * atach the audio codec to the clock source of the encoder - */ -void r600_audio_set_clock(struct drm_encoder *encoder, int clock) -{ - struct drm_device *dev = encoder->dev; - struct radeon_device *rdev = dev->dev_private; - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); - int base_rate = 48000; - - switch (radeon_encoder->encoder_id) { - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1: - case ENCODER_OBJECT_ID_INTERNAL_LVTM1: - WREG32_P(R600_AUDIO_TIMING, 0, ~0x301); - break; - case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: - case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: - case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: - WREG32_P(R600_AUDIO_TIMING, 0x100, ~0x301); - break; - default: - dev_err(rdev->dev, "Unsupported encoder type 0x%02X\n", - radeon_encoder->encoder_id); - return; - } - - if (ASIC_IS_DCE4(rdev)) { - /* TODO: other PLLs? */ - WREG32(EVERGREEN_AUDIO_PLL1_MUL, base_rate * 10); - WREG32(EVERGREEN_AUDIO_PLL1_DIV, clock * 10); - WREG32(EVERGREEN_AUDIO_PLL1_UNK, 0x00000071); - - /* Select DTO source */ - WREG32(0x5ac, radeon_crtc->crtc_id); - } else { - switch (dig->dig_encoder) { - case 0: - WREG32(R600_AUDIO_PLL1_MUL, base_rate * 50); - WREG32(R600_AUDIO_PLL1_DIV, clock * 100); - WREG32(R600_AUDIO_CLK_SRCSEL, 0); - break; - - case 1: - WREG32(R600_AUDIO_PLL2_MUL, base_rate * 50); - WREG32(R600_AUDIO_PLL2_DIV, clock * 100); - WREG32(R600_AUDIO_CLK_SRCSEL, 1); - break; - default: - dev_err(rdev->dev, - "Unsupported DIG on encoder 0x%02X\n", - radeon_encoder->encoder_id); - return; - } - } -} - -/* - * release the audio timer - * TODO: How to do this correctly on SMP systems? - */ -void r600_audio_fini(struct radeon_device *rdev) -{ - if (!rdev->audio_enabled) - return; - - r600_audio_engine_enable(rdev, false); -} diff --git a/sys/dev/pci/drm/radeon/r600_blit_kms.c b/sys/dev/pci/drm/radeon/r600_blit_kms.c deleted file mode 100644 index 47157d09a4d..00000000000 --- a/sys/dev/pci/drm/radeon/r600_blit_kms.c +++ /dev/null @@ -1,786 +0,0 @@ -/* $OpenBSD: r600_blit_kms.c,v 1.5 2018/04/20 21:12:50 naddy Exp $ */ -/* - * Copyright 2009 Advanced Micro Devices, Inc. - * Copyright 2009 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - */ - -#include -#include -#include "radeon.h" - -#include "r600d.h" -#include "r600_blit_shaders.h" -#include "radeon_blit_common.h" - -/* emits 21 on rv770+, 23 on r600 */ -static void -set_render_target(struct radeon_device *rdev, int format, - int w, int h, u64 gpu_addr) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u32 cb_color_info; - int pitch, slice; - - h = roundup2(h, 8); - if (h < 8) - h = 8; - - cb_color_info = CB_FORMAT(format) | - CB_SOURCE_FORMAT(CB_SF_EXPORT_NORM) | - CB_ARRAY_MODE(ARRAY_1D_TILED_THIN1); - pitch = (w / 8) - 1; - slice = ((w * h) / 64) - 1; - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (CB_COLOR0_BASE - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, gpu_addr >> 8); - - if (rdev->family > CHIP_R600 && rdev->family < CHIP_RV770) { - radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_BASE_UPDATE, 0)); - radeon_ring_write(ring, 2 << 0); - } - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (CB_COLOR0_SIZE - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, (pitch << 0) | (slice << 10)); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (CB_COLOR0_VIEW - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, 0); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (CB_COLOR0_INFO - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, cb_color_info); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (CB_COLOR0_TILE - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, 0); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (CB_COLOR0_FRAG - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, 0); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (CB_COLOR0_MASK - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, 0); -} - -/* emits 5dw */ -static void -cp_set_surface_sync(struct radeon_device *rdev, - u32 sync_type, u32 size, - u64 mc_addr) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u32 cp_coher_size; - - if (size == 0xffffffff) - cp_coher_size = 0xffffffff; - else - cp_coher_size = ((size + 255) >> 8); - - radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); - radeon_ring_write(ring, sync_type); - radeon_ring_write(ring, cp_coher_size); - radeon_ring_write(ring, mc_addr >> 8); - radeon_ring_write(ring, 10); /* poll interval */ -} - -/* emits 21dw + 1 surface sync = 26dw */ -static void -set_shaders(struct radeon_device *rdev) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u64 gpu_addr; - u32 sq_pgm_resources; - - /* setup shader regs */ - sq_pgm_resources = (1 << 0); - - /* VS */ - gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset; - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (SQ_PGM_START_VS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, gpu_addr >> 8); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (SQ_PGM_RESOURCES_VS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, sq_pgm_resources); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (SQ_PGM_CF_OFFSET_VS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, 0); - - /* PS */ - gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.ps_offset; - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (SQ_PGM_START_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, gpu_addr >> 8); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (SQ_PGM_RESOURCES_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, sq_pgm_resources | (1 << 28)); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (SQ_PGM_EXPORTS_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, 2); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); - radeon_ring_write(ring, (SQ_PGM_CF_OFFSET_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, 0); - - gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset; - cp_set_surface_sync(rdev, PACKET3_SH_ACTION_ENA, 512, gpu_addr); -} - -/* emits 9 + 1 sync (5) = 14*/ -static void -set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u32 sq_vtx_constant_word2; - - sq_vtx_constant_word2 = SQ_VTXC_BASE_ADDR_HI(upper_32_bits(gpu_addr) & 0xff) | - SQ_VTXC_STRIDE(16); -#ifdef __BIG_ENDIAN - sq_vtx_constant_word2 |= SQ_VTXC_ENDIAN_SWAP(SQ_ENDIAN_8IN32); -#endif - - radeon_ring_write(ring, PACKET3(PACKET3_SET_RESOURCE, 7)); - radeon_ring_write(ring, 0x460); - radeon_ring_write(ring, gpu_addr & 0xffffffff); - radeon_ring_write(ring, 48 - 1); - radeon_ring_write(ring, sq_vtx_constant_word2); - radeon_ring_write(ring, 1 << 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, SQ_TEX_VTX_VALID_BUFFER << 30); - - if ((rdev->family == CHIP_RV610) || - (rdev->family == CHIP_RV620) || - (rdev->family == CHIP_RS780) || - (rdev->family == CHIP_RS880) || - (rdev->family == CHIP_RV710)) - cp_set_surface_sync(rdev, - PACKET3_TC_ACTION_ENA, 48, gpu_addr); - else - cp_set_surface_sync(rdev, - PACKET3_VC_ACTION_ENA, 48, gpu_addr); -} - -/* emits 9 */ -static void -set_tex_resource(struct radeon_device *rdev, - int format, int w, int h, int pitch, - u64 gpu_addr, u32 size) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - uint32_t sq_tex_resource_word0, sq_tex_resource_word1, sq_tex_resource_word4; - - if (h < 1) - h = 1; - - sq_tex_resource_word0 = S_038000_DIM(V_038000_SQ_TEX_DIM_2D) | - S_038000_TILE_MODE(V_038000_ARRAY_1D_TILED_THIN1); - sq_tex_resource_word0 |= S_038000_PITCH((pitch >> 3) - 1) | - S_038000_TEX_WIDTH(w - 1); - - sq_tex_resource_word1 = S_038004_DATA_FORMAT(format); - sq_tex_resource_word1 |= S_038004_TEX_HEIGHT(h - 1); - - sq_tex_resource_word4 = S_038010_REQUEST_SIZE(1) | - S_038010_DST_SEL_X(SQ_SEL_X) | - S_038010_DST_SEL_Y(SQ_SEL_Y) | - S_038010_DST_SEL_Z(SQ_SEL_Z) | - S_038010_DST_SEL_W(SQ_SEL_W); - - cp_set_surface_sync(rdev, - PACKET3_TC_ACTION_ENA, size, gpu_addr); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_RESOURCE, 7)); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, sq_tex_resource_word0); - radeon_ring_write(ring, sq_tex_resource_word1); - radeon_ring_write(ring, gpu_addr >> 8); - radeon_ring_write(ring, gpu_addr >> 8); - radeon_ring_write(ring, sq_tex_resource_word4); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, SQ_TEX_VTX_VALID_TEXTURE << 30); -} - -/* emits 12 */ -static void -set_scissors(struct radeon_device *rdev, int x1, int y1, - int x2, int y2) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); - radeon_ring_write(ring, (PA_SC_SCREEN_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, (x1 << 0) | (y1 << 16)); - radeon_ring_write(ring, (x2 << 0) | (y2 << 16)); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); - radeon_ring_write(ring, (PA_SC_GENERIC_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, (x1 << 0) | (y1 << 16) | (1 << 31)); - radeon_ring_write(ring, (x2 << 0) | (y2 << 16)); - - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); - radeon_ring_write(ring, (PA_SC_WINDOW_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); - radeon_ring_write(ring, (x1 << 0) | (y1 << 16) | (1 << 31)); - radeon_ring_write(ring, (x2 << 0) | (y2 << 16)); -} - -/* emits 10 */ -static void -draw_auto(struct radeon_device *rdev) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); - radeon_ring_write(ring, (VGT_PRIMITIVE_TYPE - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); - radeon_ring_write(ring, DI_PT_RECTLIST); - - radeon_ring_write(ring, PACKET3(PACKET3_INDEX_TYPE, 0)); - radeon_ring_write(ring, -#ifdef __BIG_ENDIAN - (2 << 2) | -#endif - DI_INDEX_SIZE_16_BIT); - - radeon_ring_write(ring, PACKET3(PACKET3_NUM_INSTANCES, 0)); - radeon_ring_write(ring, 1); - - radeon_ring_write(ring, PACKET3(PACKET3_DRAW_INDEX_AUTO, 1)); - radeon_ring_write(ring, 3); - radeon_ring_write(ring, DI_SRC_SEL_AUTO_INDEX); - -} - -/* emits 14 */ -static void -set_default_state(struct radeon_device *rdev) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - u32 sq_config, sq_gpr_resource_mgmt_1, sq_gpr_resource_mgmt_2; - u32 sq_thread_resource_mgmt, sq_stack_resource_mgmt_1, sq_stack_resource_mgmt_2; - int num_ps_gprs, num_vs_gprs, num_temp_gprs, num_gs_gprs, num_es_gprs; - int num_ps_threads, num_vs_threads, num_gs_threads, num_es_threads; - int num_ps_stack_entries, num_vs_stack_entries, num_gs_stack_entries, num_es_stack_entries; - u64 gpu_addr; - int dwords; - - switch (rdev->family) { - case CHIP_R600: - num_ps_gprs = 192; - num_vs_gprs = 56; - num_temp_gprs = 4; - num_gs_gprs = 0; - num_es_gprs = 0; - num_ps_threads = 136; - num_vs_threads = 48; - num_gs_threads = 4; - num_es_threads = 4; - num_ps_stack_entries = 128; - num_vs_stack_entries = 128; - num_gs_stack_entries = 0; - num_es_stack_entries = 0; - break; - case CHIP_RV630: - case CHIP_RV635: - num_ps_gprs = 84; - num_vs_gprs = 36; - num_temp_gprs = 4; - num_gs_gprs = 0; - num_es_gprs = 0; - num_ps_threads = 144; - num_vs_threads = 40; - num_gs_threads = 4; - num_es_threads = 4; - num_ps_stack_entries = 40; - num_vs_stack_entries = 40; - num_gs_stack_entries = 32; - num_es_stack_entries = 16; - break; - case CHIP_RV610: - case CHIP_RV620: - case CHIP_RS780: - case CHIP_RS880: - default: - num_ps_gprs = 84; - num_vs_gprs = 36; - num_temp_gprs = 4; - num_gs_gprs = 0; - num_es_gprs = 0; - num_ps_threads = 136; - num_vs_threads = 48; - num_gs_threads = 4; - num_es_threads = 4; - num_ps_stack_entries = 40; - num_vs_stack_entries = 40; - num_gs_stack_entries = 32; - num_es_stack_entries = 16; - break; - case CHIP_RV670: - num_ps_gprs = 144; - num_vs_gprs = 40; - num_temp_gprs = 4; - num_gs_gprs = 0; - num_es_gprs = 0; - num_ps_threads = 136; - num_vs_threads = 48; - num_gs_threads = 4; - num_es_threads = 4; - num_ps_stack_entries = 40; - num_vs_stack_entries = 40; - num_gs_stack_entries = 32; - num_es_stack_entries = 16; - break; - case CHIP_RV770: - num_ps_gprs = 192; - num_vs_gprs = 56; - num_temp_gprs = 4; - num_gs_gprs = 0; - num_es_gprs = 0; - num_ps_threads = 188; - num_vs_threads = 60; - num_gs_threads = 0; - num_es_threads = 0; - num_ps_stack_entries = 256; - num_vs_stack_entries = 256; - num_gs_stack_entries = 0; - num_es_stack_entries = 0; - break; - case CHIP_RV730: - case CHIP_RV740: - num_ps_gprs = 84; - num_vs_gprs = 36; - num_temp_gprs = 4; - num_gs_gprs = 0; - num_es_gprs = 0; - num_ps_threads = 188; - num_vs_threads = 60; - num_gs_threads = 0; - num_es_threads = 0; - num_ps_stack_entries = 128; - num_vs_stack_entries = 128; - num_gs_stack_entries = 0; - num_es_stack_entries = 0; - break; - case CHIP_RV710: - num_ps_gprs = 192; - num_vs_gprs = 56; - num_temp_gprs = 4; - num_gs_gprs = 0; - num_es_gprs = 0; - num_ps_threads = 144; - num_vs_threads = 48; - num_gs_threads = 0; - num_es_threads = 0; - num_ps_stack_entries = 128; - num_vs_stack_entries = 128; - num_gs_stack_entries = 0; - num_es_stack_entries = 0; - break; - } - - if ((rdev->family == CHIP_RV610) || - (rdev->family == CHIP_RV620) || - (rdev->family == CHIP_RS780) || - (rdev->family == CHIP_RS880) || - (rdev->family == CHIP_RV710)) - sq_config = 0; - else - sq_config = VC_ENABLE; - - sq_config |= (DX9_CONSTS | - ALU_INST_PREFER_VECTOR | - PS_PRIO(0) | - VS_PRIO(1) | - GS_PRIO(2) | - ES_PRIO(3)); - - sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(num_ps_gprs) | - NUM_VS_GPRS(num_vs_gprs) | - NUM_CLAUSE_TEMP_GPRS(num_temp_gprs)); - sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(num_gs_gprs) | - NUM_ES_GPRS(num_es_gprs)); - sq_thread_resource_mgmt = (NUM_PS_THREADS(num_ps_threads) | - NUM_VS_THREADS(num_vs_threads) | - NUM_GS_THREADS(num_gs_threads) | - NUM_ES_THREADS(num_es_threads)); - sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(num_ps_stack_entries) | - NUM_VS_STACK_ENTRIES(num_vs_stack_entries)); - sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(num_gs_stack_entries) | - NUM_ES_STACK_ENTRIES(num_es_stack_entries)); - - /* emit an IB pointing at default state */ - dwords = roundup2(rdev->r600_blit.state_len, 0x10); - gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.state_offset; - radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - radeon_ring_write(ring, -#ifdef __BIG_ENDIAN - (2 << 0) | -#endif - (gpu_addr & 0xFFFFFFFC)); - radeon_ring_write(ring, upper_32_bits(gpu_addr) & 0xFF); - radeon_ring_write(ring, dwords); - - /* SQ config */ - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 6)); - radeon_ring_write(ring, (SQ_CONFIG - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); - radeon_ring_write(ring, sq_config); - radeon_ring_write(ring, sq_gpr_resource_mgmt_1); - radeon_ring_write(ring, sq_gpr_resource_mgmt_2); - radeon_ring_write(ring, sq_thread_resource_mgmt); - radeon_ring_write(ring, sq_stack_resource_mgmt_1); - radeon_ring_write(ring, sq_stack_resource_mgmt_2); -} - -int r600_blit_init(struct radeon_device *rdev) -{ - u32 obj_size; - int i, r, dwords; - void *ptr; - u32 packet2s[16]; - int num_packet2s = 0; - - rdev->r600_blit.primitives.set_render_target = set_render_target; - rdev->r600_blit.primitives.cp_set_surface_sync = cp_set_surface_sync; - rdev->r600_blit.primitives.set_shaders = set_shaders; - rdev->r600_blit.primitives.set_vtx_resource = set_vtx_resource; - rdev->r600_blit.primitives.set_tex_resource = set_tex_resource; - rdev->r600_blit.primitives.set_scissors = set_scissors; - rdev->r600_blit.primitives.draw_auto = draw_auto; - rdev->r600_blit.primitives.set_default_state = set_default_state; - - rdev->r600_blit.ring_size_common = 8; /* sync semaphore */ - rdev->r600_blit.ring_size_common += 40; /* shaders + def state */ - rdev->r600_blit.ring_size_common += 5; /* done copy */ - rdev->r600_blit.ring_size_common += 16; /* fence emit for done copy */ - - rdev->r600_blit.ring_size_per_loop = 76; - /* set_render_target emits 2 extra dwords on rv6xx */ - if (rdev->family > CHIP_R600 && rdev->family < CHIP_RV770) - rdev->r600_blit.ring_size_per_loop += 2; - - rdev->r600_blit.max_dim = 8192; - - rdev->r600_blit.state_offset = 0; - - if (rdev->family >= CHIP_RV770) - rdev->r600_blit.state_len = r7xx_default_size; - else - rdev->r600_blit.state_len = r6xx_default_size; - - dwords = rdev->r600_blit.state_len; - while (dwords & 0xf) { - packet2s[num_packet2s++] = cpu_to_le32(PACKET2(0)); - dwords++; - } - - obj_size = dwords * 4; - obj_size = roundup2(obj_size, 256); - - rdev->r600_blit.vs_offset = obj_size; - obj_size += r6xx_vs_size * 4; - obj_size = roundup2(obj_size, 256); - - rdev->r600_blit.ps_offset = obj_size; - obj_size += r6xx_ps_size * 4; - obj_size = roundup2(obj_size, 256); - - /* pin copy shader into vram if not already initialized */ - if (rdev->r600_blit.shader_obj == NULL) { - r = radeon_bo_create(rdev, obj_size, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_VRAM, - NULL, &rdev->r600_blit.shader_obj); - if (r) { - DRM_ERROR("r600 failed to allocate shader\n"); - return r; - } - - r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false); - if (unlikely(r != 0)) - return r; - r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM, - &rdev->r600_blit.shader_gpu_addr); - radeon_bo_unreserve(rdev->r600_blit.shader_obj); - if (r) { - dev_err(rdev->dev, "(%d) pin blit object failed\n", r); - return r; - } - } - - DRM_DEBUG("r6xx blit allocated bo %08x vs %08x ps %08x\n", - obj_size, - rdev->r600_blit.vs_offset, rdev->r600_blit.ps_offset); - - r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false); - if (unlikely(r != 0)) - return r; - r = radeon_bo_kmap(rdev->r600_blit.shader_obj, &ptr); - if (r) { - DRM_ERROR("failed to map blit object %d\n", r); - return r; - } - if (rdev->family >= CHIP_RV770) - memcpy_toio(ptr + rdev->r600_blit.state_offset, - r7xx_default_state, rdev->r600_blit.state_len * 4); - else - memcpy_toio(ptr + rdev->r600_blit.state_offset, - r6xx_default_state, rdev->r600_blit.state_len * 4); - if (num_packet2s) - memcpy_toio(ptr + rdev->r600_blit.state_offset + (rdev->r600_blit.state_len * 4), - packet2s, num_packet2s * 4); - for (i = 0; i < r6xx_vs_size; i++) - *(u32 *)((unsigned long)ptr + rdev->r600_blit.vs_offset + i * 4) = cpu_to_le32(r6xx_vs[i]); - for (i = 0; i < r6xx_ps_size; i++) - *(u32 *)((unsigned long)ptr + rdev->r600_blit.ps_offset + i * 4) = cpu_to_le32(r6xx_ps[i]); - radeon_bo_kunmap(rdev->r600_blit.shader_obj); - radeon_bo_unreserve(rdev->r600_blit.shader_obj); - - radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); - return 0; -} - -void r600_blit_fini(struct radeon_device *rdev) -{ - int r; - - radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); - if (rdev->r600_blit.shader_obj == NULL) - return; - /* If we can't reserve the bo, unref should be enough to destroy - * it when it becomes idle. - */ - r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false); - if (!r) { - radeon_bo_unpin(rdev->r600_blit.shader_obj); - radeon_bo_unreserve(rdev->r600_blit.shader_obj); - } - radeon_bo_unref(&rdev->r600_blit.shader_obj); -} - -static unsigned r600_blit_create_rect(unsigned num_gpu_pages, - int *width, int *height, int max_dim) -{ - unsigned max_pages; - unsigned pages = num_gpu_pages; - int w, h; - - if (num_gpu_pages == 0) { - /* not supposed to be called with no pages, but just in case */ - h = 0; - w = 0; - pages = 0; - WARN_ON(1); - } else { - int rect_order = 2; - h = RECT_UNIT_H; - while (num_gpu_pages / rect_order) { - h *= 2; - rect_order *= 4; - if (h >= max_dim) { - h = max_dim; - break; - } - } - max_pages = (max_dim * h) / (RECT_UNIT_W * RECT_UNIT_H); - if (pages > max_pages) - pages = max_pages; - w = (pages * RECT_UNIT_W * RECT_UNIT_H) / h; - w = (w / RECT_UNIT_W) * RECT_UNIT_W; - pages = (w * h) / (RECT_UNIT_W * RECT_UNIT_H); - BUG_ON(pages == 0); - } - - - DRM_DEBUG("blit_rectangle: h=%d, w=%d, pages=%d\n", h, w, pages); - - /* return width and height only of the caller wants it */ - if (height) - *height = h; - if (width) - *width = w; - - return pages; -} - - -int r600_blit_prepare_copy(struct radeon_device *rdev, unsigned num_gpu_pages, - struct radeon_fence **fence, struct radeon_sa_bo **vb, - struct radeon_semaphore **sem) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - int r; - int ring_size; - int num_loops = 0; - int dwords_per_loop = rdev->r600_blit.ring_size_per_loop; - - /* num loops */ - while (num_gpu_pages) { - num_gpu_pages -= - r600_blit_create_rect(num_gpu_pages, NULL, NULL, - rdev->r600_blit.max_dim); - num_loops++; - } - - /* 48 bytes for vertex per loop */ - r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, vb, - (num_loops*48)+256, 256, true); - if (r) { - return r; - } - - r = radeon_semaphore_create(rdev, sem); - if (r) { - radeon_sa_bo_free(rdev, vb, NULL); - return r; - } - - /* calculate number of loops correctly */ - ring_size = num_loops * dwords_per_loop; - ring_size += rdev->r600_blit.ring_size_common; - r = radeon_ring_lock(rdev, ring, ring_size); - if (r) { - radeon_sa_bo_free(rdev, vb, NULL); - radeon_semaphore_free(rdev, sem, NULL); - return r; - } - - if (radeon_fence_need_sync(*fence, RADEON_RING_TYPE_GFX_INDEX)) { - radeon_semaphore_sync_rings(rdev, *sem, (*fence)->ring, - RADEON_RING_TYPE_GFX_INDEX); - radeon_fence_note_sync(*fence, RADEON_RING_TYPE_GFX_INDEX); - } else { - radeon_semaphore_free(rdev, sem, NULL); - } - - rdev->r600_blit.primitives.set_default_state(rdev); - rdev->r600_blit.primitives.set_shaders(rdev); - return 0; -} - -void r600_blit_done_copy(struct radeon_device *rdev, struct radeon_fence **fence, - struct radeon_sa_bo *vb, struct radeon_semaphore *sem) -{ - struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - int r; - - r = radeon_fence_emit(rdev, fence, RADEON_RING_TYPE_GFX_INDEX); - if (r) { - radeon_ring_unlock_undo(rdev, ring); - return; - } - - radeon_ring_unlock_commit(rdev, ring); - radeon_sa_bo_free(rdev, &vb, *fence); - radeon_semaphore_free(rdev, &sem, *fence); -} - -void r600_kms_blit_copy(struct radeon_device *rdev, - u64 src_gpu_addr, u64 dst_gpu_addr, - unsigned num_gpu_pages, - struct radeon_sa_bo *vb) -{ - u64 vb_gpu_addr; - u32 *vb_cpu_addr; - - DRM_DEBUG("emitting copy %16llx %16llx %d\n", - src_gpu_addr, dst_gpu_addr, num_gpu_pages); - vb_cpu_addr = (u32 *)radeon_sa_bo_cpu_addr(vb); - vb_gpu_addr = radeon_sa_bo_gpu_addr(vb); - - while (num_gpu_pages) { - int w, h; - unsigned size_in_bytes; - unsigned pages_per_loop = - r600_blit_create_rect(num_gpu_pages, &w, &h, - rdev->r600_blit.max_dim); - - size_in_bytes = pages_per_loop * RADEON_GPU_PAGE_SIZE; - DRM_DEBUG("rectangle w=%d h=%d\n", w, h); - - vb_cpu_addr[0] = 0; - vb_cpu_addr[1] = 0; - vb_cpu_addr[2] = 0; - vb_cpu_addr[3] = 0; - - vb_cpu_addr[4] = 0; - vb_cpu_addr[5] = int2float(h); - vb_cpu_addr[6] = 0; - vb_cpu_addr[7] = int2float(h); - - vb_cpu_addr[8] = int2float(w); - vb_cpu_addr[9] = int2float(h); - vb_cpu_addr[10] = int2float(w); - vb_cpu_addr[11] = int2float(h); - - rdev->r600_blit.primitives.set_tex_resource(rdev, FMT_8_8_8_8, - w, h, w, src_gpu_addr, size_in_bytes); - rdev->r600_blit.primitives.set_render_target(rdev, COLOR_8_8_8_8, - w, h, dst_gpu_addr); - rdev->r600_blit.primitives.set_scissors(rdev, 0, 0, w, h); - rdev->r600_blit.primitives.set_vtx_resource(rdev, vb_gpu_addr); - rdev->r600_blit.primitives.draw_auto(rdev); - rdev->r600_blit.primitives.cp_set_surface_sync(rdev, - PACKET3_CB_ACTION_ENA | PACKET3_CB0_DEST_BASE_ENA, - size_in_bytes, dst_gpu_addr); - - vb_cpu_addr += 12; - vb_gpu_addr += 4*12; - src_gpu_addr += size_in_bytes; - dst_gpu_addr += size_in_bytes; - num_gpu_pages -= pages_per_loop; - } -} - -/* 23 bits of float fractional data */ -#define I2F_FRAC_BITS 23 -#define I2F_MASK ((1 << I2F_FRAC_BITS) - 1) - -/* - * Converts unsigned integer into 32-bit IEEE floating point representation. - * Will be exact from 0 to 2^24. Above that, we round towards zero - * as the fractional bits will not fit in a float. (It would be better to - * round towards even as the fpu does, but that is slower.) - */ -__pure uint32_t int2float(uint32_t x) -{ - uint32_t msb, exponent, fraction; - - /* Zero is special */ - if (!x) return 0; - - /* Get location of the most significant bit */ - msb = fls(x); - - /* - * Use a rotate instead of a shift because that works both leftwards - * and rightwards due to the mod(32) behaviour. This means we don't - * need to check to see if we are above 2^24 or not. - */ - fraction = ror32(x, (msb - I2F_FRAC_BITS) & 0x1f) & I2F_MASK; - exponent = (127 + msb) << I2F_FRAC_BITS; - - return fraction + exponent; -} diff --git a/sys/dev/pci/drm/radeon/r600_blit_shaders.c b/sys/dev/pci/drm/radeon/r600_blit_shaders.c index 96abf766810..2497bad5fec 100644 --- a/sys/dev/pci/drm/radeon/r600_blit_shaders.c +++ b/sys/dev/pci/drm/radeon/r600_blit_shaders.c @@ -1,4 +1,3 @@ -/* $OpenBSD: r600_blit_shaders.c,v 1.4 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2009 Advanced Micro Devices, Inc. * @@ -25,14 +24,13 @@ * Alex Deucher */ -#include -#include +#include /* * R6xx+ cards need to use the 3D engine to blit data which requires * quite a bit of hw state setup. Rather than pull the whole 3D driver * (which normally generates the 3D state) into the DRM, we opt to use - * statically generated state tables. The regsiter state and shaders + * statically generated state tables. The register state and shaders * were hand generated to support blitting functionality. See the 3D * driver or documentation for descriptions of the registers and * shader instructions. diff --git a/sys/dev/pci/drm/radeon/r600_blit_shaders.h b/sys/dev/pci/drm/radeon/r600_blit_shaders.h index 7cb1ac4cf7e..f437d36dd98 100644 --- a/sys/dev/pci/drm/radeon/r600_blit_shaders.h +++ b/sys/dev/pci/drm/radeon/r600_blit_shaders.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r600_blit_shaders.h,v 1.3 2018/04/20 16:09:36 deraadt Exp $ */ /* * Copyright 2009 Advanced Micro Devices, Inc. * Copyright 2009 Red Hat Inc. @@ -36,5 +35,4 @@ extern const u32 r6xx_default_state[]; extern const u32 r6xx_ps_size, r6xx_vs_size; extern const u32 r6xx_default_size, r7xx_default_size; -__pure uint32_t int2float(uint32_t x); #endif diff --git a/sys/dev/pci/drm/radeon/r600_cs.c b/sys/dev/pci/drm/radeon/r600_cs.c index 4fb1557d05c..8004b49f542 100644 --- a/sys/dev/pci/drm/radeon/r600_cs.c +++ b/sys/dev/pci/drm/radeon/r600_cs.c @@ -1,4 +1,3 @@ -/* $OpenBSD: r600_cs.c,v 1.9 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -31,12 +30,7 @@ #include "r600d.h" #include "r600_reg_safe.h" -static int r600_cs_packet_next_reloc_mm(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc); -static int r600_cs_packet_next_reloc_nomm(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc); -typedef int (*next_reloc_t)(struct radeon_cs_parser*, struct radeon_cs_reloc**); -static next_reloc_t r600_cs_packet_next_reloc = &r600_cs_packet_next_reloc_mm; +static int r600_nomm; extern void r600_cs_legacy_get_tiling_conf(struct drm_device *dev, u32 *npipes, u32 *nbanks, u32 *group_size); @@ -787,170 +781,29 @@ static int r600_cs_track_check(struct radeon_cs_parser *p) } /** - * r600_cs_packet_parse() - parse cp packet and point ib index to next packet - * @parser: parser structure holding parsing context. - * @pkt: where to store packet informations - * - * Assume that chunk_ib_index is properly set. Will return -EINVAL - * if packet is bigger than remaining ib size. or if packets is unknown. - **/ -static int r600_cs_packet_parse(struct radeon_cs_parser *p, - struct radeon_cs_packet *pkt, - unsigned idx) -{ - struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx]; - uint32_t header; - - if (idx >= ib_chunk->length_dw) { - DRM_ERROR("Can not parse packet at %d after CS end %d !\n", - idx, ib_chunk->length_dw); - return -EINVAL; - } - header = radeon_get_ib_value(p, idx); - pkt->idx = idx; - pkt->type = CP_PACKET_GET_TYPE(header); - pkt->count = CP_PACKET_GET_COUNT(header); - pkt->one_reg_wr = 0; - switch (pkt->type) { - case PACKET_TYPE0: - pkt->reg = CP_PACKET0_GET_REG(header); - break; - case PACKET_TYPE3: - pkt->opcode = CP_PACKET3_GET_OPCODE(header); - break; - case PACKET_TYPE2: - pkt->count = -1; - break; - default: - DRM_ERROR("Unknown packet type %d at %d !\n", pkt->type, idx); - return -EINVAL; - } - if ((pkt->count + 1 + pkt->idx) >= ib_chunk->length_dw) { - DRM_ERROR("Packet (%d:%d:%d) end after CS buffer (%d) !\n", - pkt->idx, pkt->type, pkt->count, ib_chunk->length_dw); - return -EINVAL; - } - return 0; -} - -/** - * r600_cs_packet_next_reloc_mm() - parse next packet which should be reloc packet3 - * @parser: parser structure holding parsing context. - * @data: pointer to relocation data - * @offset_start: starting offset - * @offset_mask: offset mask (to align start offset on) - * @reloc: reloc informations - * - * Check next packet is relocation packet3, do bo validation and compute - * GPU offset using the provided start. - **/ -static int r600_cs_packet_next_reloc_mm(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc) -{ - struct radeon_cs_chunk *relocs_chunk; - struct radeon_cs_packet p3reloc; - unsigned idx; - int r; - - if (p->chunk_relocs_idx == -1) { - DRM_ERROR("No relocation chunk !\n"); - return -EINVAL; - } - *cs_reloc = NULL; - relocs_chunk = &p->chunks[p->chunk_relocs_idx]; - r = r600_cs_packet_parse(p, &p3reloc, p->idx); - if (r) { - return r; - } - p->idx += p3reloc.count + 2; - if (p3reloc.type != PACKET_TYPE3 || p3reloc.opcode != PACKET3_NOP) { - DRM_ERROR("No packet3 for relocation for packet at %d.\n", - p3reloc.idx); - return -EINVAL; - } - idx = radeon_get_ib_value(p, p3reloc.idx + 1); - if (idx >= relocs_chunk->length_dw) { - DRM_ERROR("Relocs at %d after relocations chunk end %d !\n", - idx, relocs_chunk->length_dw); - return -EINVAL; - } - /* FIXME: we assume reloc size is 4 dwords */ - *cs_reloc = p->relocs_ptr[(idx / 4)]; - return 0; -} - -/** - * r600_cs_packet_next_reloc_nomm() - parse next packet which should be reloc packet3 - * @parser: parser structure holding parsing context. - * @data: pointer to relocation data - * @offset_start: starting offset - * @offset_mask: offset mask (to align start offset on) - * @reloc: reloc informations - * - * Check next packet is relocation packet3, do bo validation and compute - * GPU offset using the provided start. - **/ -static int r600_cs_packet_next_reloc_nomm(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc) -{ - struct radeon_cs_chunk *relocs_chunk; - struct radeon_cs_packet p3reloc; - unsigned idx; - int r; - - if (p->chunk_relocs_idx == -1) { - DRM_ERROR("No relocation chunk !\n"); - return -EINVAL; - } - *cs_reloc = NULL; - relocs_chunk = &p->chunks[p->chunk_relocs_idx]; - r = r600_cs_packet_parse(p, &p3reloc, p->idx); - if (r) { - return r; - } - p->idx += p3reloc.count + 2; - if (p3reloc.type != PACKET_TYPE3 || p3reloc.opcode != PACKET3_NOP) { - DRM_ERROR("No packet3 for relocation for packet at %d.\n", - p3reloc.idx); - return -EINVAL; - } - idx = radeon_get_ib_value(p, p3reloc.idx + 1); - if (idx >= relocs_chunk->length_dw) { - DRM_ERROR("Relocs at %d after relocations chunk end %d !\n", - idx, relocs_chunk->length_dw); - return -EINVAL; - } - *cs_reloc = p->relocs; - (*cs_reloc)->lobj.gpu_offset = (u64)relocs_chunk->kdata[idx + 3] << 32; - (*cs_reloc)->lobj.gpu_offset |= relocs_chunk->kdata[idx + 0]; - return 0; -} - -/** - * r600_cs_packet_next_is_pkt3_nop() - test if next packet is packet3 nop for reloc + * r600_cs_packet_parse_vline() - parse userspace VLINE packet * @parser: parser structure holding parsing context. * - * Check next packet is relocation packet3, do bo validation and compute - * GPU offset using the provided start. - **/ -static int r600_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p) + * This is an R600-specific function for parsing VLINE packets. + * Real work is done by r600_cs_common_vline_parse function. + * Here we just set up ASIC-specific register table and call + * the common implementation function. + */ +static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p) { - struct radeon_cs_packet p3reloc; - int r; + static uint32_t vline_start_end[2] = {AVIVO_D1MODE_VLINE_START_END, + AVIVO_D2MODE_VLINE_START_END}; + static uint32_t vline_status[2] = {AVIVO_D1MODE_VLINE_STATUS, + AVIVO_D2MODE_VLINE_STATUS}; - r = r600_cs_packet_parse(p, &p3reloc, p->idx); - if (r) { - return 0; - } - if (p3reloc.type != PACKET_TYPE3 || p3reloc.opcode != PACKET3_NOP) { - return 0; - } - return 1; + return r600_cs_common_vline_parse(p, vline_start_end, vline_status); } /** - * r600_cs_packet_next_vline() - parse userspace VLINE packet + * r600_cs_common_vline_parse() - common vline parser * @parser: parser structure holding parsing context. + * @vline_start_end: table of vline_start_end registers + * @vline_status: table of vline_status registers * * Userspace sends a special sequence for VLINE waits. * PACKET0 - VLINE_START_END + value @@ -960,11 +813,17 @@ static int r600_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p) * This function parses this and relocates the VLINE START END * and WAIT_REG_MEM packets to the correct crtc. * It also detects a switched off crtc and nulls out the - * wait in that case. + * wait in that case. This function is common for all ASICs that + * are R600 and newer. The parsing algorithm is the same, and only + * differs in which registers are used. + * + * Caller is the ASIC-specific function which passes the parser + * context and ASIC-specific register table */ -static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p) +int r600_cs_common_vline_parse(struct radeon_cs_parser *p, + uint32_t *vline_start_end, + uint32_t *vline_status) { - struct drm_mode_object *obj; struct drm_crtc *crtc; struct radeon_crtc *radeon_crtc; struct radeon_cs_packet p3reloc, wait_reg_mem; @@ -976,12 +835,12 @@ static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p) ib = p->ib.ptr; /* parse the WAIT_REG_MEM */ - r = r600_cs_packet_parse(p, &wait_reg_mem, p->idx); + r = radeon_cs_packet_parse(p, &wait_reg_mem, p->idx); if (r) return r; /* check its a WAIT_REG_MEM */ - if (wait_reg_mem.type != PACKET_TYPE3 || + if (wait_reg_mem.type != RADEON_PACKET_TYPE3 || wait_reg_mem.opcode != PACKET3_WAIT_REG_MEM) { DRM_ERROR("vline wait missing WAIT_REG_MEM segment\n"); return -EINVAL; @@ -990,7 +849,12 @@ static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p) wait_reg_mem_info = radeon_get_ib_value(p, wait_reg_mem.idx + 1); /* bit 4 is reg (0) or mem (1) */ if (wait_reg_mem_info & 0x10) { - DRM_ERROR("vline WAIT_REG_MEM waiting on MEM rather than REG\n"); + DRM_ERROR("vline WAIT_REG_MEM waiting on MEM instead of REG\n"); + return -EINVAL; + } + /* bit 8 is me (0) or pfp (1) */ + if (wait_reg_mem_info & 0x100) { + DRM_ERROR("vline WAIT_REG_MEM waiting on PFP instead of ME\n"); return -EINVAL; } /* waiting for value to be equal */ @@ -998,18 +862,18 @@ static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p) DRM_ERROR("vline WAIT_REG_MEM function not equal\n"); return -EINVAL; } - if ((radeon_get_ib_value(p, wait_reg_mem.idx + 2) << 2) != AVIVO_D1MODE_VLINE_STATUS) { + if ((radeon_get_ib_value(p, wait_reg_mem.idx + 2) << 2) != vline_status[0]) { DRM_ERROR("vline WAIT_REG_MEM bad reg\n"); return -EINVAL; } - if (radeon_get_ib_value(p, wait_reg_mem.idx + 5) != AVIVO_D1MODE_VLINE_STAT) { + if (radeon_get_ib_value(p, wait_reg_mem.idx + 5) != RADEON_VLINE_STAT) { DRM_ERROR("vline WAIT_REG_MEM bad bit mask\n"); return -EINVAL; } /* jump over the NOP */ - r = r600_cs_packet_parse(p, &p3reloc, p->idx + wait_reg_mem.count + 2); + r = radeon_cs_packet_parse(p, &p3reloc, p->idx + wait_reg_mem.count + 2); if (r) return r; @@ -1019,19 +883,18 @@ static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p) header = radeon_get_ib_value(p, h_idx); crtc_id = radeon_get_ib_value(p, h_idx + 2 + 7 + 1); - reg = CP_PACKET0_GET_REG(header); + reg = R600_CP_PACKET0_GET_REG(header); - obj = drm_mode_object_find(p->rdev->ddev, crtc_id, DRM_MODE_OBJECT_CRTC); - if (!obj) { + crtc = drm_crtc_find(p->rdev->ddev, crtc_id); + if (!crtc) { DRM_ERROR("cannot find crtc %d\n", crtc_id); - return -EINVAL; + return -ENOENT; } - crtc = obj_to_crtc(obj); radeon_crtc = to_radeon_crtc(crtc); crtc_id = radeon_crtc->crtc_id; if (!crtc->enabled) { - /* if the CRTC isn't enabled - we need to nop out the WAIT_REG_MEM */ + /* CRTC isn't enabled - we need to nop out the WAIT_REG_MEM */ ib[h_idx + 2] = PACKET2(0); ib[h_idx + 3] = PACKET2(0); ib[h_idx + 4] = PACKET2(0); @@ -1039,20 +902,15 @@ static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p) ib[h_idx + 6] = PACKET2(0); ib[h_idx + 7] = PACKET2(0); ib[h_idx + 8] = PACKET2(0); - } else if (crtc_id == 1) { - switch (reg) { - case AVIVO_D1MODE_VLINE_START_END: - header &= ~R600_CP_PACKET0_REG_MASK; - header |= AVIVO_D2MODE_VLINE_START_END >> 2; - break; - default: - DRM_ERROR("unknown crtc reloc\n"); - return -EINVAL; - } + } else if (reg == vline_start_end[0]) { + header &= ~R600_CP_PACKET0_REG_MASK; + header |= vline_start_end[crtc_id] >> 2; ib[h_idx] = header; - ib[h_idx + 4] = AVIVO_D2MODE_VLINE_STATUS >> 2; + ib[h_idx + 4] = vline_status[crtc_id] >> 2; + } else { + DRM_ERROR("unknown crtc reloc\n"); + return -EINVAL; } - return 0; } @@ -1110,7 +968,7 @@ static int r600_cs_parse_packet0(struct radeon_cs_parser *p, static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) { struct r600_cs_track *track = (struct r600_cs_track *)p->track; - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; u32 m, i, tmp, *ib; int r; @@ -1155,13 +1013,13 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case SQ_GSTMP_RING_BASE: case SQ_PSTMP_RING_BASE: case SQ_VSTMP_RING_BASE: - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, 0); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); break; case SQ_CONFIG: track->sq_config = radeon_get_ib_value(p, idx); @@ -1172,8 +1030,8 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) break; case R_028010_DB_DEPTH_INFO: if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS) && - r600_cs_packet_next_is_pkt3_nop(p)) { - r = r600_cs_packet_next_reloc(p, &reloc); + radeon_cs_packet_next_is_pkt3_nop(p)) { + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); @@ -1182,7 +1040,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) track->db_depth_info = radeon_get_ib_value(p, idx); ib[idx] &= C_028010_ARRAY_MODE; track->db_depth_info &= C_028010_ARRAY_MODE; - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) { + if (reloc->tiling_flags & RADEON_TILING_MACRO) { ib[idx] |= S_028010_ARRAY_MODE(V_028010_ARRAY_2D_TILED_THIN1); track->db_depth_info |= S_028010_ARRAY_MODE(V_028010_ARRAY_2D_TILED_THIN1); } else { @@ -1215,7 +1073,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case VGT_STRMOUT_BUFFER_BASE_1: case VGT_STRMOUT_BUFFER_BASE_2: case VGT_STRMOUT_BUFFER_BASE_3: - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); @@ -1223,9 +1081,9 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) } tmp = (reg - VGT_STRMOUT_BUFFER_BASE_0) / 16; track->vgt_strmout_bo_offset[tmp] = radeon_get_ib_value(p, idx) << 8; - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->vgt_strmout_bo[tmp] = reloc->robj; - track->vgt_strmout_bo_mc[tmp] = reloc->lobj.gpu_offset; + track->vgt_strmout_bo_mc[tmp] = reloc->gpu_offset; track->streamout_dirty = true; break; case VGT_STRMOUT_BUFFER_SIZE_0: @@ -1238,13 +1096,13 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) track->streamout_dirty = true; break; case CP_COHER_BASE: - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_warn(p->dev, "missing reloc for CP_COHER_BASE " "0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); break; case R_028238_CB_TARGET_MASK: track->cb_target_mask = radeon_get_ib_value(p, idx); @@ -1273,18 +1131,18 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case R_0280B8_CB_COLOR6_INFO: case R_0280BC_CB_COLOR7_INFO: if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS) && - r600_cs_packet_next_is_pkt3_nop(p)) { - r = r600_cs_packet_next_reloc(p, &reloc); + radeon_cs_packet_next_is_pkt3_nop(p)) { + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg); return -EINVAL; } tmp = (reg - R_0280A0_CB_COLOR0_INFO) / 4; track->cb_color_info[tmp] = radeon_get_ib_value(p, idx); - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) { + if (reloc->tiling_flags & RADEON_TILING_MACRO) { ib[idx] |= S_0280A0_ARRAY_MODE(V_0280A0_ARRAY_2D_TILED_THIN1); track->cb_color_info[tmp] |= S_0280A0_ARRAY_MODE(V_0280A0_ARRAY_2D_TILED_THIN1); - } else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) { + } else if (reloc->tiling_flags & RADEON_TILING_MICRO) { ib[idx] |= S_0280A0_ARRAY_MODE(V_0280A0_ARRAY_1D_TILED_THIN1); track->cb_color_info[tmp] |= S_0280A0_ARRAY_MODE(V_0280A0_ARRAY_1D_TILED_THIN1); } @@ -1337,7 +1195,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case R_0280F8_CB_COLOR6_FRAG: case R_0280FC_CB_COLOR7_FRAG: tmp = (reg - R_0280E0_CB_COLOR0_FRAG) / 4; - if (!r600_cs_packet_next_is_pkt3_nop(p)) { + if (!radeon_cs_packet_next_is_pkt3_nop(p)) { if (!track->cb_color_base_last[tmp]) { dev_err(p->dev, "Broken old userspace ? no cb_color0_base supplied before trying to write 0x%08X\n", reg); return -EINVAL; @@ -1346,14 +1204,14 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) track->cb_color_frag_offset[tmp] = track->cb_color_bo_offset[tmp]; ib[idx] = track->cb_color_base_last[tmp]; } else { - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg); return -EINVAL; } track->cb_color_frag_bo[tmp] = reloc->robj; track->cb_color_frag_offset[tmp] = (u64)ib[idx] << 8; - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); } if (G_0280A0_TILE_MODE(track->cb_color_info[tmp])) { track->cb_dirty = true; @@ -1368,7 +1226,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case R_0280D8_CB_COLOR6_TILE: case R_0280DC_CB_COLOR7_TILE: tmp = (reg - R_0280C0_CB_COLOR0_TILE) / 4; - if (!r600_cs_packet_next_is_pkt3_nop(p)) { + if (!radeon_cs_packet_next_is_pkt3_nop(p)) { if (!track->cb_color_base_last[tmp]) { dev_err(p->dev, "Broken old userspace ? no cb_color0_base supplied before trying to write 0x%08X\n", reg); return -EINVAL; @@ -1377,14 +1235,14 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) track->cb_color_tile_offset[tmp] = track->cb_color_bo_offset[tmp]; ib[idx] = track->cb_color_base_last[tmp]; } else { - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg); return -EINVAL; } track->cb_color_tile_bo[tmp] = reloc->robj; track->cb_color_tile_offset[tmp] = (u64)ib[idx] << 8; - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); } if (G_0280A0_TILE_MODE(track->cb_color_info[tmp])) { track->cb_dirty = true; @@ -1412,7 +1270,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case CB_COLOR5_BASE: case CB_COLOR6_BASE: case CB_COLOR7_BASE: - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); @@ -1420,34 +1278,34 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) } tmp = (reg - CB_COLOR0_BASE) / 4; track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx) << 8; - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->cb_color_base_last[tmp] = ib[idx]; track->cb_color_bo[tmp] = reloc->robj; - track->cb_color_bo_mc[tmp] = reloc->lobj.gpu_offset; + track->cb_color_bo_mc[tmp] = reloc->gpu_offset; track->cb_dirty = true; break; case DB_DEPTH_BASE: - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } track->db_offset = radeon_get_ib_value(p, idx) << 8; - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->db_bo = reloc->robj; - track->db_bo_mc = reloc->lobj.gpu_offset; + track->db_bo_mc = reloc->gpu_offset; track->db_dirty = true; break; case DB_HTILE_DATA_BASE: - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } track->htile_offset = radeon_get_ib_value(p, idx) << 8; - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); track->htile_bo = reloc->robj; track->db_dirty = true; break; @@ -1510,22 +1368,22 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) case SQ_ALU_CONST_CACHE_VS_13: case SQ_ALU_CONST_CACHE_VS_14: case SQ_ALU_CONST_CACHE_VS_15: - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_warn(p->dev, "bad SET_CONTEXT_REG " "0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); break; case SX_MEMORY_EXPORT_BASE: - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { dev_warn(p->dev, "bad SET_CONFIG_REG " "0x%04X\n", reg); return -EINVAL; } - ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); break; case SX_MISC: track->sx_misc_kill_all_prims = (radeon_get_ib_value(p, idx) & 0x1) != 0; @@ -1767,7 +1625,7 @@ static bool r600_is_safe_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) static int r600_packet3_check(struct radeon_cs_parser *p, struct radeon_cs_packet *pkt) { - struct radeon_cs_reloc *reloc; + struct radeon_bo_list *reloc; struct r600_cs_track *track; volatile u32 *ib; unsigned idx; @@ -1805,13 +1663,13 @@ static int r600_packet3_check(struct radeon_cs_parser *p, return -EINVAL; } - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad SET PREDICATION\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + (idx_value & 0xfffffff0) + ((u64)(tmp & 0xff) << 32); @@ -1846,13 +1704,13 @@ static int r600_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad DRAW_INDEX\n"); return -EINVAL; } - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad DRAW_INDEX\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + idx_value + ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32); @@ -1898,18 +1756,21 @@ static int r600_packet3_check(struct radeon_cs_parser *p, if (idx_value & 0x10) { uint64_t offset; - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad WAIT_REG_MEM\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + (radeon_get_ib_value(p, idx+1) & 0xfffffff0) + ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); ib[idx+1] = (ib[idx+1] & 0x3) | (offset & 0xfffffff0); ib[idx+2] = upper_32_bits(offset) & 0xff; + } else if (idx_value & 0x100) { + DRM_ERROR("cannot use PFP on REG wait\n"); + return -EINVAL; } break; case PACKET3_CP_DMA: @@ -1932,7 +1793,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, return -EINVAL; } /* src address space is memory */ - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad CP DMA SRC\n"); return -EINVAL; @@ -1941,7 +1802,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, tmp = radeon_get_ib_value(p, idx) + ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32); - offset = reloc->lobj.gpu_offset + tmp; + offset = reloc->gpu_offset + tmp; if ((tmp + size) > radeon_bo_size(reloc->robj)) { dev_warn(p->dev, "CP DMA src buffer too small (%llu %lu)\n", @@ -1962,7 +1823,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("CP DMA DAIC only supported for registers\n"); return -EINVAL; } - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad CP DMA DST\n"); return -EINVAL; @@ -1971,7 +1832,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, tmp = radeon_get_ib_value(p, idx+2) + ((u64)(radeon_get_ib_value(p, idx+3) & 0xff) << 32); - offset = reloc->lobj.gpu_offset + tmp; + offset = reloc->gpu_offset + tmp; if ((tmp + size) > radeon_bo_size(reloc->robj)) { dev_warn(p->dev, "CP DMA dst buffer too small (%llu %lu)\n", @@ -1992,12 +1853,12 @@ static int r600_packet3_check(struct radeon_cs_parser *p, /* 0xffffffff/0x0 is flush all cache flag */ if (radeon_get_ib_value(p, idx + 1) != 0xffffffff || radeon_get_ib_value(p, idx + 2) != 0) { - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad SURFACE_SYNC\n"); return -EINVAL; } - ib[idx+2] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx+2] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); } break; case PACKET3_EVENT_WRITE: @@ -2008,12 +1869,12 @@ static int r600_packet3_check(struct radeon_cs_parser *p, if (pkt->count) { uint64_t offset; - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad EVENT_WRITE\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + (radeon_get_ib_value(p, idx+1) & 0xfffffff8) + ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); @@ -2029,13 +1890,13 @@ static int r600_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad EVENT_WRITE_EOP\n"); return -EINVAL; } - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad EVENT_WRITE\n"); return -EINVAL; } - offset = reloc->lobj.gpu_offset + + offset = reloc->gpu_offset + (radeon_get_ib_value(p, idx+1) & 0xfffffffc) + ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); @@ -2095,32 +1956,32 @@ static int r600_packet3_check(struct radeon_cs_parser *p, switch (G__SQ_VTX_CONSTANT_TYPE(radeon_get_ib_value(p, idx+(i*7)+6+1))) { case SQ_TEX_VTX_VALID_TEXTURE: /* tex base */ - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad SET_RESOURCE\n"); return -EINVAL; } - base_offset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + base_offset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff); if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { - if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) + if (reloc->tiling_flags & RADEON_TILING_MACRO) ib[idx+1+(i*7)+0] |= S_038000_TILE_MODE(V_038000_ARRAY_2D_TILED_THIN1); - else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) + else if (reloc->tiling_flags & RADEON_TILING_MICRO) ib[idx+1+(i*7)+0] |= S_038000_TILE_MODE(V_038000_ARRAY_1D_TILED_THIN1); } texture = reloc->robj; /* tex mip base */ - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad SET_RESOURCE\n"); return -EINVAL; } - mip_offset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + mip_offset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff); mipmap = reloc->robj; r = r600_check_texture_resource(p, idx+(i*7)+1, texture, mipmap, base_offset + radeon_get_ib_value(p, idx+1+(i*7)+2), mip_offset + radeon_get_ib_value(p, idx+1+(i*7)+3), - reloc->lobj.tiling_flags); + reloc->tiling_flags); if (r) return r; ib[idx+1+(i*7)+2] += base_offset; @@ -2130,7 +1991,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, { uint64_t offset64; /* vtx base */ - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad SET_RESOURCE\n"); return -EINVAL; @@ -2144,7 +2005,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, ib[idx+1+(i*7)+1] = radeon_bo_size(reloc->robj) - offset; } - offset64 = reloc->lobj.gpu_offset + offset; + offset64 = reloc->gpu_offset + offset; ib[idx+1+(i*8)+0] = offset64; ib[idx+1+(i*8)+2] = (ib[idx+1+(i*8)+2] & 0xffffff00) | (upper_32_bits(offset64) & 0xff); @@ -2231,7 +2092,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, { u64 offset; - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad STRMOUT_BASE_UPDATE reloc\n"); return -EINVAL; @@ -2254,7 +2115,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, offset + 4, radeon_bo_size(reloc->robj)); return -EINVAL; } - ib[idx+1] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff); + ib[idx+1] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); } break; case PACKET3_SURFACE_BASE_UPDATE: @@ -2275,7 +2136,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, /* Updating memory at DST_ADDRESS. */ if (idx_value & 0x1) { u64 offset; - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing dst reloc)\n"); return -EINVAL; @@ -2287,14 +2148,14 @@ static int r600_packet3_check(struct radeon_cs_parser *p, offset + 4, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+1] = offset; ib[idx+2] = upper_32_bits(offset) & 0xff; } /* Reading data from SRC_ADDRESS. */ if (((idx_value >> 1) & 0x3) == 2) { u64 offset; - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing src reloc)\n"); return -EINVAL; @@ -2306,7 +2167,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, offset + 4, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+3] = offset; ib[idx+4] = upper_32_bits(offset) & 0xff; } @@ -2319,7 +2180,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, DRM_ERROR("bad MEM_WRITE (invalid count)\n"); return -EINVAL; } - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad MEM_WRITE (missing reloc)\n"); return -EINVAL; @@ -2335,7 +2196,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, offset + 8, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+0] = offset; ib[idx+1] = upper_32_bits(offset) & 0xff; break; @@ -2348,7 +2209,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, if (idx_value & 0x1) { u64 offset; /* SRC is memory. */ - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad COPY_DW (missing src reloc)\n"); return -EINVAL; @@ -2360,7 +2221,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, offset + 4, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+1] = offset; ib[idx+2] = upper_32_bits(offset) & 0xff; } else { @@ -2372,7 +2233,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, if (idx_value & 0x2) { u64 offset; /* DST is memory. */ - r = r600_cs_packet_next_reloc(p, &reloc); + r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm); if (r) { DRM_ERROR("bad COPY_DW (missing dst reloc)\n"); return -EINVAL; @@ -2384,7 +2245,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, offset + 4, radeon_bo_size(reloc->robj)); return -EINVAL; } - offset += reloc->lobj.gpu_offset; + offset += reloc->gpu_offset; ib[idx+3] = offset; ib[idx+4] = upper_32_bits(offset) & 0xff; } else { @@ -2427,7 +2288,7 @@ int r600_cs_parse(struct radeon_cs_parser *p) p->track = track; } do { - r = r600_cs_packet_parse(p, &pkt, p->idx); + r = radeon_cs_packet_parse(p, &pkt, p->idx); if (r) { kfree(p->track); p->track = NULL; @@ -2435,12 +2296,12 @@ int r600_cs_parse(struct radeon_cs_parser *p) } p->idx += pkt.count + 2; switch (pkt.type) { - case PACKET_TYPE0: + case RADEON_PACKET_TYPE0: r = r600_cs_parse_packet0(p, &pkt); break; - case PACKET_TYPE2: + case RADEON_PACKET_TYPE2: break; - case PACKET_TYPE3: + case RADEON_PACKET_TYPE3: r = r600_packet3_check(p, &pkt); break; default: @@ -2454,7 +2315,7 @@ int r600_cs_parse(struct radeon_cs_parser *p) p->track = NULL; return r; } - } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw); + } while (p->idx < p->chunk_ib->length_dw); #if 0 for (r = 0; r < p->ib.length_dw; r++) { printk(KERN_INFO "%05d 0x%08X\n", r, p->ib.ptr[r]); @@ -2466,19 +2327,7 @@ int r600_cs_parse(struct radeon_cs_parser *p) return 0; } -/* don't these need UMS functions */ -#if 0 -static int r600_cs_parser_relocs_legacy(struct radeon_cs_parser *p) -{ - if (p->chunk_relocs_idx == -1) { - return 0; - } - p->relocs = kzalloc(sizeof(struct radeon_cs_reloc), GFP_KERNEL); - if (p->relocs == NULL) { - return -ENOMEM; - } - return 0; -} +#ifdef CONFIG_DRM_RADEON_UMS /** * cs_parser_fini() - clean parser states @@ -2493,17 +2342,24 @@ static void r600_cs_parser_fini(struct radeon_cs_parser *parser, int error) unsigned i; kfree(parser->relocs); - for (i = 0; i < parser->nchunks; i++) { - kfree(parser->chunks[i].kdata); - if (parser->rdev && (parser->rdev->flags & RADEON_IS_AGP)) { - kfree(parser->chunks[i].kpage[0]); - kfree(parser->chunks[i].kpage[1]); - } - } + for (i = 0; i < parser->nchunks; i++) + drm_free_large(parser->chunks[i].kdata); kfree(parser->chunks); kfree(parser->chunks_array); } +static int r600_cs_parser_relocs_legacy(struct radeon_cs_parser *p) +{ + if (p->chunk_relocs == NULL) { + return 0; + } + p->relocs = kzalloc(sizeof(struct radeon_bo_list), GFP_KERNEL); + if (p->relocs == NULL) { + return -ENOMEM; + } + return 0; +} + int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp, unsigned family, u32 *ib, int *l) { @@ -2521,9 +2377,7 @@ int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp, /* initialize parser */ memset(&parser, 0, sizeof(struct radeon_cs_parser)); parser.filp = filp; -#ifdef notyet - parser.dev = dev; -#endif + parser.dev = &dev->pdev->dev; parser.rdev = NULL; parser.family = family; parser.track = track; @@ -2543,16 +2397,15 @@ int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp, /* Copy the packet into the IB, the parser will read from the * input memory (cached) and write to the IB (which can be * uncached). */ - ib_chunk = &parser.chunks[parser.chunk_ib_idx]; + ib_chunk = parser.chunk_ib; parser.ib.length_dw = ib_chunk->length_dw; *l = parser.ib.length_dw; - r = r600_cs_parse(&parser); - if (r) { - DRM_ERROR("Invalid command stream !\n"); + if (copy_from_user(ib, ib_chunk->user_ptr, ib_chunk->length_dw * 4)) { + r = -EFAULT; r600_cs_parser_fini(&parser, r); return r; } - r = radeon_cs_finish_pages(&parser); + r = r600_cs_parse(&parser); if (r) { DRM_ERROR("Invalid command stream !\n"); r600_cs_parser_fini(&parser, r); @@ -2561,13 +2414,14 @@ int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp, r600_cs_parser_fini(&parser, r); return r; } -#endif void r600_cs_legacy_init(void) { - r600_cs_packet_next_reloc = &r600_cs_packet_next_reloc_nomm; + r600_nomm = 1; } +#endif + /* * DMA */ @@ -2580,24 +2434,24 @@ void r600_cs_legacy_init(void) * GPU offset using the provided start. **/ int r600_dma_cs_next_reloc(struct radeon_cs_parser *p, - struct radeon_cs_reloc **cs_reloc) + struct radeon_bo_list **cs_reloc) { struct radeon_cs_chunk *relocs_chunk; unsigned idx; *cs_reloc = NULL; - if (p->chunk_relocs_idx == -1) { + if (p->chunk_relocs == NULL) { DRM_ERROR("No relocation chunk !\n"); return -EINVAL; } - relocs_chunk = &p->chunks[p->chunk_relocs_idx]; + relocs_chunk = p->chunk_relocs; idx = p->dma_reloc_idx; if (idx >= p->nrelocs) { DRM_ERROR("Relocs at %d after relocations chunk end %d !\n", idx, p->nrelocs); return -EINVAL; } - *cs_reloc = p->relocs_ptr[idx]; + *cs_reloc = &p->relocs[idx]; p->dma_reloc_idx++; return 0; } @@ -2617,8 +2471,8 @@ int r600_dma_cs_next_reloc(struct radeon_cs_parser *p, **/ int r600_dma_cs_parse(struct radeon_cs_parser *p) { - struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx]; - struct radeon_cs_reloc *src_reloc, *dst_reloc; + struct radeon_cs_chunk *ib_chunk = p->chunk_ib; + struct radeon_bo_list *src_reloc, *dst_reloc; u32 header, cmd, count, tiled; volatile u32 *ib = p->ib.ptr; u32 idx, idx_value; @@ -2648,14 +2502,14 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p) dst_offset = radeon_get_ib_value(p, idx+1); dst_offset <<= 8; - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8); + ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); p->idx += count + 5; } else { dst_offset = radeon_get_ib_value(p, idx+1); dst_offset |= ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32; - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+2] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; + ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+2] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; p->idx += count + 3; } if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { @@ -2682,22 +2536,22 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p) /* tiled src, linear dst */ src_offset = radeon_get_ib_value(p, idx+1); src_offset <<= 8; - ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8); + ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8); dst_offset = radeon_get_ib_value(p, idx+5); dst_offset |= ((u64)(radeon_get_ib_value(p, idx+6) & 0xff)) << 32; - ib[idx+5] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+6] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; + ib[idx+5] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+6] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; } else { /* linear src, tiled dst */ src_offset = radeon_get_ib_value(p, idx+5); src_offset |= ((u64)(radeon_get_ib_value(p, idx+6) & 0xff)) << 32; - ib[idx+5] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+6] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; + ib[idx+5] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+6] += upper_32_bits(src_reloc->gpu_offset) & 0xff; dst_offset = radeon_get_ib_value(p, idx+1); dst_offset <<= 8; - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8); + ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); } p->idx += 7; } else { @@ -2707,10 +2561,10 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p) dst_offset = radeon_get_ib_value(p, idx+1); dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32; - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff; - ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; + ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+3] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; + ib[idx+4] += upper_32_bits(src_reloc->gpu_offset) & 0xff; p->idx += 5; } else { src_offset = radeon_get_ib_value(p, idx+2); @@ -2718,10 +2572,10 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p) dst_offset = radeon_get_ib_value(p, idx+1); dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff0000)) << 16; - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+3] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff; - ib[idx+3] += (upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff) << 16; + ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xfffffffc); + ib[idx+3] += upper_32_bits(src_reloc->gpu_offset) & 0xff; + ib[idx+3] += (upper_32_bits(dst_reloc->gpu_offset) & 0xff) << 16; p->idx += 4; } } @@ -2753,8 +2607,8 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p) dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); return -EINVAL; } - ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc); - ib[idx+3] += (upper_32_bits(dst_reloc->lobj.gpu_offset) << 16) & 0x00ff0000; + ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); + ib[idx+3] += (upper_32_bits(dst_reloc->gpu_offset) << 16) & 0x00ff0000; p->idx += 4; break; case DMA_PACKET_NOP: @@ -2764,7 +2618,7 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p) DRM_ERROR("Unknown packet type %d at %d !\n", cmd, idx); return -EINVAL; } - } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw); + } while (p->idx < p->chunk_ib->length_dw); #if 0 for (r = 0; r < p->ib->length_dw; r++) { printk(KERN_INFO "%05d 0x%08X\n", r, p->ib.ptr[r]); diff --git a/sys/dev/pci/drm/radeon/r600_dma.c b/sys/dev/pci/drm/radeon/r600_dma.c new file mode 100644 index 00000000000..ee1d48ad1e4 --- /dev/null +++ b/sys/dev/pci/drm/radeon/r600_dma.c @@ -0,0 +1,491 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "r600d.h" + +u32 r600_gpu_check_soft_reset(struct radeon_device *rdev); + +/* + * DMA + * Starting with R600, the GPU has an asynchronous + * DMA engine. The programming model is very similar + * to the 3D engine (ring buffer, IBs, etc.), but the + * DMA controller has it's own packet format that is + * different form the PM4 format used by the 3D engine. + * It supports copying data, writing embedded data, + * solid fills, and a number of other things. It also + * has support for tiling/detiling of buffers. + */ + +/** + * r600_dma_get_rptr - get the current read pointer + * + * @rdev: radeon_device pointer + * @ring: radeon ring pointer + * + * Get the current rptr from the hardware (r6xx+). + */ +uint32_t r600_dma_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + u32 rptr; + + if (rdev->wb.enabled) + rptr = rdev->wb.wb[ring->rptr_offs/4]; + else + rptr = RREG32(DMA_RB_RPTR); + + return (rptr & 0x3fffc) >> 2; +} + +/** + * r600_dma_get_wptr - get the current write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon ring pointer + * + * Get the current wptr from the hardware (r6xx+). + */ +uint32_t r600_dma_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + return (RREG32(DMA_RB_WPTR) & 0x3fffc) >> 2; +} + +/** + * r600_dma_set_wptr - commit the write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon ring pointer + * + * Write the wptr back to the hardware (r6xx+). + */ +void r600_dma_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + WREG32(DMA_RB_WPTR, (ring->wptr << 2) & 0x3fffc); +} + +/** + * r600_dma_stop - stop the async dma engine + * + * @rdev: radeon_device pointer + * + * Stop the async dma engine (r6xx-evergreen). + */ +void r600_dma_stop(struct radeon_device *rdev) +{ + u32 rb_cntl = RREG32(DMA_RB_CNTL); + + if (rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + + rb_cntl &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL, rb_cntl); + + rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false; +} + +/** + * r600_dma_resume - setup and start the async dma engine + * + * @rdev: radeon_device pointer + * + * Set up the DMA ring buffer and enable it. (r6xx-evergreen). + * Returns 0 for success, error for failure. + */ +int r600_dma_resume(struct radeon_device *rdev) +{ + struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; + u32 rb_cntl, dma_cntl, ib_cntl; + u32 rb_bufsz; + int r; + + WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0); + WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0); + + /* Set ring buffer size in dwords */ + rb_bufsz = order_base_2(ring->ring_size / 4); + rb_cntl = rb_bufsz << 1; +#ifdef __BIG_ENDIAN + rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE; +#endif + WREG32(DMA_RB_CNTL, rb_cntl); + + /* Initialize the ring buffer's read and write pointers */ + WREG32(DMA_RB_RPTR, 0); + WREG32(DMA_RB_WPTR, 0); + + /* set the wb address whether it's enabled or not */ + WREG32(DMA_RB_RPTR_ADDR_HI, + upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF); + WREG32(DMA_RB_RPTR_ADDR_LO, + ((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC)); + + if (rdev->wb.enabled) + rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE; + + WREG32(DMA_RB_BASE, ring->gpu_addr >> 8); + + /* enable DMA IBs */ + ib_cntl = DMA_IB_ENABLE; +#ifdef __BIG_ENDIAN + ib_cntl |= DMA_IB_SWAP_ENABLE; +#endif + WREG32(DMA_IB_CNTL, ib_cntl); + + dma_cntl = RREG32(DMA_CNTL); + dma_cntl &= ~CTXEMPTY_INT_ENABLE; + WREG32(DMA_CNTL, dma_cntl); + + if (rdev->family >= CHIP_RV770) + WREG32(DMA_MODE, 1); + + ring->wptr = 0; + WREG32(DMA_RB_WPTR, ring->wptr << 2); + + WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE); + + ring->ready = true; + + r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring); + if (r) { + ring->ready = false; + return r; + } + + if (rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); + + return 0; +} + +/** + * r600_dma_fini - tear down the async dma engine + * + * @rdev: radeon_device pointer + * + * Stop the async dma engine and free the ring (r6xx-evergreen). + */ +void r600_dma_fini(struct radeon_device *rdev) +{ + r600_dma_stop(rdev); + radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]); +} + +/** + * r600_dma_is_lockup - Check if the DMA engine is locked up + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Check if the async DMA engine is locked up. + * Returns true if the engine appears to be locked up, false if not. + */ +bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +{ + u32 reset_mask = r600_gpu_check_soft_reset(rdev); + + if (!(reset_mask & RADEON_RESET_DMA)) { + radeon_ring_lockup_update(rdev, ring); + return false; + } + return radeon_ring_test_lockup(rdev, ring); +} + + +/** + * r600_dma_ring_test - simple async dma engine test + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Test the DMA engine by writing using it to write an + * value to memory. (r6xx-SI). + * Returns 0 for success, error for failure. + */ +int r600_dma_ring_test(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + unsigned i; + int r; + unsigned index; + u32 tmp; + u64 gpu_addr; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + index = R600_WB_DMA_RING_TEST_OFFSET; + else + index = CAYMAN_WB_DMA1_RING_TEST_OFFSET; + + gpu_addr = rdev->wb.gpu_addr + index; + + tmp = 0xCAFEDEAD; + rdev->wb.wb[index/4] = cpu_to_le32(tmp); + + r = radeon_ring_lock(rdev, ring, 4); + if (r) { + DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r); + return r; + } + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1)); + radeon_ring_write(ring, lower_32_bits(gpu_addr)); + radeon_ring_write(ring, upper_32_bits(gpu_addr) & 0xff); + radeon_ring_write(ring, 0xDEADBEEF); + radeon_ring_unlock_commit(rdev, ring, false); + + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = le32_to_cpu(rdev->wb.wb[index/4]); + if (tmp == 0xDEADBEEF) + break; + DRM_UDELAY(1); + } + + if (i < rdev->usec_timeout) { + DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); + } else { + DRM_ERROR("radeon: ring %d test failed (0x%08X)\n", + ring->idx, tmp); + r = -EINVAL; + } + return r; +} + +/** + * r600_dma_fence_ring_emit - emit a fence on the DMA ring + * + * @rdev: radeon_device pointer + * @fence: radeon fence object + * + * Add a DMA fence packet to the ring to write + * the fence seq number and DMA trap packet to generate + * an interrupt if needed (r6xx-r7xx). + */ +void r600_dma_fence_ring_emit(struct radeon_device *rdev, + struct radeon_fence *fence) +{ + struct radeon_ring *ring = &rdev->ring[fence->ring]; + u64 addr = rdev->fence_drv[fence->ring].gpu_addr; + + /* write the fence */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0)); + radeon_ring_write(ring, addr & 0xfffffffc); + radeon_ring_write(ring, (upper_32_bits(addr) & 0xff)); + radeon_ring_write(ring, lower_32_bits(fence->seq)); + /* generate an interrupt */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0)); +} + +/** + * r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * @semaphore: radeon semaphore object + * @emit_wait: wait or signal semaphore + * + * Add a DMA semaphore packet to the ring wait on or signal + * other rings (r6xx-SI). + */ +bool r600_dma_semaphore_ring_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait) +{ + u64 addr = semaphore->gpu_addr; + u32 s = emit_wait ? 0 : 1; + + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0)); + radeon_ring_write(ring, addr & 0xfffffffc); + radeon_ring_write(ring, upper_32_bits(addr) & 0xff); + + return true; +} + +/** + * r600_dma_ib_test - test an IB on the DMA engine + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Test a simple IB in the DMA ring (r6xx-SI). + * Returns 0 on success, error on failure. + */ +int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) +{ + struct radeon_ib ib; + unsigned i; + unsigned index; + int r; + u32 tmp = 0; + u64 gpu_addr; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + index = R600_WB_DMA_RING_TEST_OFFSET; + else + index = CAYMAN_WB_DMA1_RING_TEST_OFFSET; + + gpu_addr = rdev->wb.gpu_addr + index; + + r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); + if (r) { + DRM_ERROR("radeon: failed to get ib (%d).\n", r); + return r; + } + + ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1); + ib.ptr[1] = lower_32_bits(gpu_addr); + ib.ptr[2] = upper_32_bits(gpu_addr) & 0xff; + ib.ptr[3] = 0xDEADBEEF; + ib.length_dw = 4; + + r = radeon_ib_schedule(rdev, &ib, NULL, false); + if (r) { + radeon_ib_free(rdev, &ib); + DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); + return r; + } + r = radeon_fence_wait(ib.fence, false); + if (r) { + DRM_ERROR("radeon: fence wait failed (%d).\n", r); + return r; + } + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = le32_to_cpu(rdev->wb.wb[index/4]); + if (tmp == 0xDEADBEEF) + break; + DRM_UDELAY(1); + } + if (i < rdev->usec_timeout) { + DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); + } else { + DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp); + r = -EINVAL; + } + radeon_ib_free(rdev, &ib); + return r; +} + +/** + * r600_dma_ring_ib_execute - Schedule an IB on the DMA engine + * + * @rdev: radeon_device pointer + * @ib: IB object to schedule + * + * Schedule an IB in the DMA ring (r6xx-r7xx). + */ +void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) +{ + struct radeon_ring *ring = &rdev->ring[ib->ring]; + + if (rdev->wb.enabled) { + u32 next_rptr = ring->wptr + 4; + while ((next_rptr & 7) != 5) + next_rptr++; + next_rptr += 3; + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1)); + radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); + radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff); + radeon_ring_write(ring, next_rptr); + } + + /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. + * Pad as necessary with NOPs. + */ + while ((ring->wptr & 7) != 5) + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0)); + radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); + radeon_ring_write(ring, (ib->length_dw << 16) | (upper_32_bits(ib->gpu_addr) & 0xFF)); + +} + +/** + * r600_copy_dma - copy pages using the DMA engine + * + * @rdev: radeon_device pointer + * @src_offset: src GPU address + * @dst_offset: dst GPU address + * @num_gpu_pages: number of GPU pages to xfer + * @resv: reservation object to sync to + * + * Copy GPU paging using the DMA engine (r6xx). + * Used by the radeon ttm implementation to move pages if + * registered as the asic copy callback. + */ +struct radeon_fence *r600_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv) +{ + struct radeon_fence *fence; + struct radeon_sync sync; + int ring_index = rdev->asic->copy.dma_ring_index; + struct radeon_ring *ring = &rdev->ring[ring_index]; + u32 size_in_dw, cur_size_in_dw; + int i, num_loops; + int r = 0; + + radeon_sync_create(&sync); + + size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; + num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE); + r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8); + if (r) { + DRM_ERROR("radeon: moving bo (%d).\n", r); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); + + for (i = 0; i < num_loops; i++) { + cur_size_in_dw = size_in_dw; + if (cur_size_in_dw > 0xFFFE) + cur_size_in_dw = 0xFFFE; + size_in_dw -= cur_size_in_dw; + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw)); + radeon_ring_write(ring, dst_offset & 0xfffffffc); + radeon_ring_write(ring, src_offset & 0xfffffffc); + radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) | + (upper_32_bits(src_offset) & 0xff))); + src_offset += cur_size_in_dw * 4; + dst_offset += cur_size_in_dw * 4; + } + + r = radeon_fence_emit(rdev, &fence, ring->idx); + if (r) { + radeon_ring_unlock_undo(rdev, ring); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_ring_unlock_commit(rdev, ring, false); + radeon_sync_free(rdev, &sync, fence); + + return fence; +} diff --git a/sys/dev/pci/drm/radeon/r600_dpm.c b/sys/dev/pci/drm/radeon/r600_dpm.c new file mode 100644 index 00000000000..c64fb66085c --- /dev/null +++ b/sys/dev/pci/drm/radeon/r600_dpm.c @@ -0,0 +1,1368 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "r600d.h" +#include "r600_dpm.h" +#include "atom.h" + +const u32 r600_utc[R600_PM_NUMBER_OF_TC] = +{ + R600_UTC_DFLT_00, + R600_UTC_DFLT_01, + R600_UTC_DFLT_02, + R600_UTC_DFLT_03, + R600_UTC_DFLT_04, + R600_UTC_DFLT_05, + R600_UTC_DFLT_06, + R600_UTC_DFLT_07, + R600_UTC_DFLT_08, + R600_UTC_DFLT_09, + R600_UTC_DFLT_10, + R600_UTC_DFLT_11, + R600_UTC_DFLT_12, + R600_UTC_DFLT_13, + R600_UTC_DFLT_14, +}; + +const u32 r600_dtc[R600_PM_NUMBER_OF_TC] = +{ + R600_DTC_DFLT_00, + R600_DTC_DFLT_01, + R600_DTC_DFLT_02, + R600_DTC_DFLT_03, + R600_DTC_DFLT_04, + R600_DTC_DFLT_05, + R600_DTC_DFLT_06, + R600_DTC_DFLT_07, + R600_DTC_DFLT_08, + R600_DTC_DFLT_09, + R600_DTC_DFLT_10, + R600_DTC_DFLT_11, + R600_DTC_DFLT_12, + R600_DTC_DFLT_13, + R600_DTC_DFLT_14, +}; + +void r600_dpm_print_class_info(u32 class, u32 class2) +{ + printk("\tui class: "); + switch (class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) { + case ATOM_PPLIB_CLASSIFICATION_UI_NONE: + default: + printk("none\n"); + break; + case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY: + printk("battery\n"); + break; + case ATOM_PPLIB_CLASSIFICATION_UI_BALANCED: + printk("balanced\n"); + break; + case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE: + printk("performance\n"); + break; + } + printk("\tinternal class: "); + if (((class & ~ATOM_PPLIB_CLASSIFICATION_UI_MASK) == 0) && + (class2 == 0)) + printk("none"); + else { + if (class & ATOM_PPLIB_CLASSIFICATION_BOOT) + printk("boot "); + if (class & ATOM_PPLIB_CLASSIFICATION_THERMAL) + printk("thermal "); + if (class & ATOM_PPLIB_CLASSIFICATION_LIMITEDPOWERSOURCE) + printk("limited_pwr "); + if (class & ATOM_PPLIB_CLASSIFICATION_REST) + printk("rest "); + if (class & ATOM_PPLIB_CLASSIFICATION_FORCED) + printk("forced "); + if (class & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE) + printk("3d_perf "); + if (class & ATOM_PPLIB_CLASSIFICATION_OVERDRIVETEMPLATE) + printk("ovrdrv "); + if (class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + printk("uvd "); + if (class & ATOM_PPLIB_CLASSIFICATION_3DLOW) + printk("3d_low "); + if (class & ATOM_PPLIB_CLASSIFICATION_ACPI) + printk("acpi "); + if (class & ATOM_PPLIB_CLASSIFICATION_HD2STATE) + printk("uvd_hd2 "); + if (class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) + printk("uvd_hd "); + if (class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) + printk("uvd_sd "); + if (class2 & ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2) + printk("limited_pwr2 "); + if (class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) + printk("ulv "); + if (class2 & ATOM_PPLIB_CLASSIFICATION2_MVC) + printk("uvd_mvc "); + } + printk("\n"); +} + +void r600_dpm_print_cap_info(u32 caps) +{ + printk("\tcaps: "); + if (caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) + printk("single_disp "); + if (caps & ATOM_PPLIB_SUPPORTS_VIDEO_PLAYBACK) + printk("video "); + if (caps & ATOM_PPLIB_DISALLOW_ON_DC) + printk("no_dc "); + printk("\n"); +} + +void r600_dpm_print_ps_status(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + printk("\tstatus: "); + if (rps == rdev->pm.dpm.current_ps) + printk("c "); + if (rps == rdev->pm.dpm.requested_ps) + printk("r "); + if (rps == rdev->pm.dpm.boot_ps) + printk("b "); + printk("\n"); +} + +u32 r600_dpm_get_vblank_time(struct radeon_device *rdev) +{ + struct drm_device *dev = rdev->ddev; + struct drm_crtc *crtc; + struct radeon_crtc *radeon_crtc; + u32 vblank_in_pixels; + u32 vblank_time_us = 0xffffffff; /* if the displays are off, vblank time is max */ + + if (rdev->num_crtc && rdev->mode_info.mode_config_initialized) { + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + radeon_crtc = to_radeon_crtc(crtc); + if (crtc->enabled && radeon_crtc->enabled && radeon_crtc->hw_mode.clock) { + vblank_in_pixels = + radeon_crtc->hw_mode.crtc_htotal * + (radeon_crtc->hw_mode.crtc_vblank_end - + radeon_crtc->hw_mode.crtc_vdisplay + + (radeon_crtc->v_border * 2)); + + vblank_time_us = vblank_in_pixels * 1000 / radeon_crtc->hw_mode.clock; + break; + } + } + } + + return vblank_time_us; +} + +u32 r600_dpm_get_vrefresh(struct radeon_device *rdev) +{ + struct drm_device *dev = rdev->ddev; + struct drm_crtc *crtc; + struct radeon_crtc *radeon_crtc; + u32 vrefresh = 0; + + if (rdev->num_crtc && rdev->mode_info.mode_config_initialized) { + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + radeon_crtc = to_radeon_crtc(crtc); + if (crtc->enabled && radeon_crtc->enabled && radeon_crtc->hw_mode.clock) { + vrefresh = drm_mode_vrefresh(&radeon_crtc->hw_mode); + break; + } + } + } + return vrefresh; +} + +void r600_calculate_u_and_p(u32 i, u32 r_c, u32 p_b, + u32 *p, u32 *u) +{ + u32 b_c = 0; + u32 i_c; + u32 tmp; + + i_c = (i * r_c) / 100; + tmp = i_c >> p_b; + + while (tmp) { + b_c++; + tmp >>= 1; + } + + *u = (b_c + 1) / 2; + *p = i_c / (1 << (2 * (*u))); +} + +int r600_calculate_at(u32 t, u32 h, u32 fh, u32 fl, u32 *tl, u32 *th) +{ + u32 k, a, ah, al; + u32 t1; + + if ((fl == 0) || (fh == 0) || (fl > fh)) + return -EINVAL; + + k = (100 * fh) / fl; + t1 = (t * (k - 100)); + a = (1000 * (100 * h + t1)) / (10000 + (t1 / 100)); + a = (a + 5) / 10; + ah = ((a * t) + 5000) / 10000; + al = a - ah; + + *th = t - ah; + *tl = t + al; + + return 0; +} + +void r600_gfx_clockgating_enable(struct radeon_device *rdev, bool enable) +{ + int i; + + if (enable) { + WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN); + } else { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN); + + WREG32(CG_RLC_REQ_AND_RSP, 0x2); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (((RREG32(CG_RLC_REQ_AND_RSP) & CG_RLC_RSP_TYPE_MASK) >> CG_RLC_RSP_TYPE_SHIFT) == 1) + break; + udelay(1); + } + + WREG32(CG_RLC_REQ_AND_RSP, 0x0); + + WREG32(GRBM_PWR_CNTL, 0x1); + RREG32(GRBM_PWR_CNTL); + } +} + +void r600_dynamicpm_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN); + else + WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN); +} + +void r600_enable_thermal_protection(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS); + else + WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS); +} + +void r600_enable_acpi_pm(struct radeon_device *rdev) +{ + WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN); +} + +void r600_enable_dynamic_pcie_gen2(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE); + else + WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE); +} + +bool r600_dynamicpm_enabled(struct radeon_device *rdev) +{ + if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN) + return true; + else + return false; +} + +void r600_enable_sclk_control(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF); + else + WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF); +} + +void r600_enable_mclk_control(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF); + else + WREG32_P(MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF); +} + +void r600_enable_spll_bypass(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(CG_SPLL_FUNC_CNTL, SPLL_BYPASS_EN, ~SPLL_BYPASS_EN); + else + WREG32_P(CG_SPLL_FUNC_CNTL, 0, ~SPLL_BYPASS_EN); +} + +void r600_wait_for_spll_change(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(CG_SPLL_FUNC_CNTL) & SPLL_CHG_STATUS) + break; + udelay(1); + } +} + +void r600_set_bsp(struct radeon_device *rdev, u32 u, u32 p) +{ + WREG32(CG_BSP, BSP(p) | BSU(u)); +} + +void r600_set_at(struct radeon_device *rdev, + u32 l_to_m, u32 m_to_h, + u32 h_to_m, u32 m_to_l) +{ + WREG32(CG_RT, FLS(l_to_m) | FMS(m_to_h)); + WREG32(CG_LT, FHS(h_to_m) | FMS(m_to_l)); +} + +void r600_set_tc(struct radeon_device *rdev, + u32 index, u32 u_t, u32 d_t) +{ + WREG32(CG_FFCT_0 + (index * 4), UTC_0(u_t) | DTC_0(d_t)); +} + +void r600_select_td(struct radeon_device *rdev, + enum r600_td td) +{ + if (td == R600_TD_AUTO) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL); + else + WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL); + if (td == R600_TD_UP) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE); + if (td == R600_TD_DOWN) + WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE); +} + +void r600_set_vrc(struct radeon_device *rdev, u32 vrv) +{ + WREG32(CG_FTV, vrv); +} + +void r600_set_tpu(struct radeon_device *rdev, u32 u) +{ + WREG32_P(CG_TPC, TPU(u), ~TPU_MASK); +} + +void r600_set_tpc(struct radeon_device *rdev, u32 c) +{ + WREG32_P(CG_TPC, TPCC(c), ~TPCC_MASK); +} + +void r600_set_sstu(struct radeon_device *rdev, u32 u) +{ + WREG32_P(CG_SSP, CG_SSTU(u), ~CG_SSTU_MASK); +} + +void r600_set_sst(struct radeon_device *rdev, u32 t) +{ + WREG32_P(CG_SSP, CG_SST(t), ~CG_SST_MASK); +} + +void r600_set_git(struct radeon_device *rdev, u32 t) +{ + WREG32_P(CG_GIT, CG_GICST(t), ~CG_GICST_MASK); +} + +void r600_set_fctu(struct radeon_device *rdev, u32 u) +{ + WREG32_P(CG_FC_T, FC_TU(u), ~FC_TU_MASK); +} + +void r600_set_fct(struct radeon_device *rdev, u32 t) +{ + WREG32_P(CG_FC_T, FC_T(t), ~FC_T_MASK); +} + +void r600_set_ctxcgtt3d_rphc(struct radeon_device *rdev, u32 p) +{ + WREG32_P(CG_CTX_CGTT3D_R, PHC(p), ~PHC_MASK); +} + +void r600_set_ctxcgtt3d_rsdc(struct radeon_device *rdev, u32 s) +{ + WREG32_P(CG_CTX_CGTT3D_R, SDC(s), ~SDC_MASK); +} + +void r600_set_vddc3d_oorsu(struct radeon_device *rdev, u32 u) +{ + WREG32_P(CG_VDDC3D_OOR, SU(u), ~SU_MASK); +} + +void r600_set_vddc3d_oorphc(struct radeon_device *rdev, u32 p) +{ + WREG32_P(CG_VDDC3D_OOR, PHC(p), ~PHC_MASK); +} + +void r600_set_vddc3d_oorsdc(struct radeon_device *rdev, u32 s) +{ + WREG32_P(CG_VDDC3D_OOR, SDC(s), ~SDC_MASK); +} + +void r600_set_mpll_lock_time(struct radeon_device *rdev, u32 lock_time) +{ + WREG32_P(MPLL_TIME, MPLL_LOCK_TIME(lock_time), ~MPLL_LOCK_TIME_MASK); +} + +void r600_set_mpll_reset_time(struct radeon_device *rdev, u32 reset_time) +{ + WREG32_P(MPLL_TIME, MPLL_RESET_TIME(reset_time), ~MPLL_RESET_TIME_MASK); +} + +void r600_engine_clock_entry_enable(struct radeon_device *rdev, + u32 index, bool enable) +{ + if (enable) + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2), + STEP_0_SPLL_ENTRY_VALID, ~STEP_0_SPLL_ENTRY_VALID); + else + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2), + 0, ~STEP_0_SPLL_ENTRY_VALID); +} + +void r600_engine_clock_entry_enable_pulse_skipping(struct radeon_device *rdev, + u32 index, bool enable) +{ + if (enable) + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2), + STEP_0_SPLL_STEP_ENABLE, ~STEP_0_SPLL_STEP_ENABLE); + else + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2), + 0, ~STEP_0_SPLL_STEP_ENABLE); +} + +void r600_engine_clock_entry_enable_post_divider(struct radeon_device *rdev, + u32 index, bool enable) +{ + if (enable) + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2), + STEP_0_POST_DIV_EN, ~STEP_0_POST_DIV_EN); + else + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2), + 0, ~STEP_0_POST_DIV_EN); +} + +void r600_engine_clock_entry_set_post_divider(struct radeon_device *rdev, + u32 index, u32 divider) +{ + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART1 + (index * 4 * 2), + STEP_0_SPLL_POST_DIV(divider), ~STEP_0_SPLL_POST_DIV_MASK); +} + +void r600_engine_clock_entry_set_reference_divider(struct radeon_device *rdev, + u32 index, u32 divider) +{ + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART1 + (index * 4 * 2), + STEP_0_SPLL_REF_DIV(divider), ~STEP_0_SPLL_REF_DIV_MASK); +} + +void r600_engine_clock_entry_set_feedback_divider(struct radeon_device *rdev, + u32 index, u32 divider) +{ + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART1 + (index * 4 * 2), + STEP_0_SPLL_FB_DIV(divider), ~STEP_0_SPLL_FB_DIV_MASK); +} + +void r600_engine_clock_entry_set_step_time(struct radeon_device *rdev, + u32 index, u32 step_time) +{ + WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART1 + (index * 4 * 2), + STEP_0_SPLL_STEP_TIME(step_time), ~STEP_0_SPLL_STEP_TIME_MASK); +} + +void r600_vid_rt_set_ssu(struct radeon_device *rdev, u32 u) +{ + WREG32_P(VID_RT, SSTU(u), ~SSTU_MASK); +} + +void r600_vid_rt_set_vru(struct radeon_device *rdev, u32 u) +{ + WREG32_P(VID_RT, VID_CRTU(u), ~VID_CRTU_MASK); +} + +void r600_vid_rt_set_vrt(struct radeon_device *rdev, u32 rt) +{ + WREG32_P(VID_RT, VID_CRT(rt), ~VID_CRT_MASK); +} + +void r600_voltage_control_enable_pins(struct radeon_device *rdev, + u64 mask) +{ + WREG32(LOWER_GPIO_ENABLE, mask & 0xffffffff); + WREG32(UPPER_GPIO_ENABLE, upper_32_bits(mask)); +} + + +void r600_voltage_control_program_voltages(struct radeon_device *rdev, + enum r600_power_level index, u64 pins) +{ + u32 tmp, mask; + u32 ix = 3 - (3 & index); + + WREG32(CTXSW_VID_LOWER_GPIO_CNTL + (ix * 4), pins & 0xffffffff); + + mask = 7 << (3 * ix); + tmp = RREG32(VID_UPPER_GPIO_CNTL); + tmp = (tmp & ~mask) | ((pins >> (32 - (3 * ix))) & mask); + WREG32(VID_UPPER_GPIO_CNTL, tmp); +} + +void r600_voltage_control_deactivate_static_control(struct radeon_device *rdev, + u64 mask) +{ + u32 gpio; + + gpio = RREG32(GPIOPAD_MASK); + gpio &= ~mask; + WREG32(GPIOPAD_MASK, gpio); + + gpio = RREG32(GPIOPAD_EN); + gpio &= ~mask; + WREG32(GPIOPAD_EN, gpio); + + gpio = RREG32(GPIOPAD_A); + gpio &= ~mask; + WREG32(GPIOPAD_A, gpio); +} + +void r600_power_level_enable(struct radeon_device *rdev, + enum r600_power_level index, bool enable) +{ + u32 ix = 3 - (3 & index); + + if (enable) + WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), CTXSW_FREQ_STATE_ENABLE, + ~CTXSW_FREQ_STATE_ENABLE); + else + WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), 0, + ~CTXSW_FREQ_STATE_ENABLE); +} + +void r600_power_level_set_voltage_index(struct radeon_device *rdev, + enum r600_power_level index, u32 voltage_index) +{ + u32 ix = 3 - (3 & index); + + WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), + CTXSW_FREQ_VIDS_CFG_INDEX(voltage_index), ~CTXSW_FREQ_VIDS_CFG_INDEX_MASK); +} + +void r600_power_level_set_mem_clock_index(struct radeon_device *rdev, + enum r600_power_level index, u32 mem_clock_index) +{ + u32 ix = 3 - (3 & index); + + WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), + CTXSW_FREQ_MCLK_CFG_INDEX(mem_clock_index), ~CTXSW_FREQ_MCLK_CFG_INDEX_MASK); +} + +void r600_power_level_set_eng_clock_index(struct radeon_device *rdev, + enum r600_power_level index, u32 eng_clock_index) +{ + u32 ix = 3 - (3 & index); + + WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), + CTXSW_FREQ_SCLK_CFG_INDEX(eng_clock_index), ~CTXSW_FREQ_SCLK_CFG_INDEX_MASK); +} + +void r600_power_level_set_watermark_id(struct radeon_device *rdev, + enum r600_power_level index, + enum r600_display_watermark watermark_id) +{ + u32 ix = 3 - (3 & index); + u32 tmp = 0; + + if (watermark_id == R600_DISPLAY_WATERMARK_HIGH) + tmp = CTXSW_FREQ_DISPLAY_WATERMARK; + WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), tmp, ~CTXSW_FREQ_DISPLAY_WATERMARK); +} + +void r600_power_level_set_pcie_gen2(struct radeon_device *rdev, + enum r600_power_level index, bool compatible) +{ + u32 ix = 3 - (3 & index); + u32 tmp = 0; + + if (compatible) + tmp = CTXSW_FREQ_GEN2PCIE_VOLT; + WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), tmp, ~CTXSW_FREQ_GEN2PCIE_VOLT); +} + +enum r600_power_level r600_power_level_get_current_index(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK; + tmp >>= CURRENT_PROFILE_INDEX_SHIFT; + return tmp; +} + +enum r600_power_level r600_power_level_get_target_index(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_PROFILE_INDEX_MASK; + tmp >>= TARGET_PROFILE_INDEX_SHIFT; + return tmp; +} + +void r600_power_level_set_enter_index(struct radeon_device *rdev, + enum r600_power_level index) +{ + WREG32_P(TARGET_AND_CURRENT_PROFILE_INDEX, DYN_PWR_ENTER_INDEX(index), + ~DYN_PWR_ENTER_INDEX_MASK); +} + +void r600_wait_for_power_level_unequal(struct radeon_device *rdev, + enum r600_power_level index) +{ + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + if (r600_power_level_get_target_index(rdev) != index) + break; + udelay(1); + } + + for (i = 0; i < rdev->usec_timeout; i++) { + if (r600_power_level_get_current_index(rdev) != index) + break; + udelay(1); + } +} + +void r600_wait_for_power_level(struct radeon_device *rdev, + enum r600_power_level index) +{ + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + if (r600_power_level_get_target_index(rdev) == index) + break; + udelay(1); + } + + for (i = 0; i < rdev->usec_timeout; i++) { + if (r600_power_level_get_current_index(rdev) == index) + break; + udelay(1); + } +} + +void r600_start_dpm(struct radeon_device *rdev) +{ + r600_enable_sclk_control(rdev, false); + r600_enable_mclk_control(rdev, false); + + r600_dynamicpm_enable(rdev, true); + + radeon_wait_for_vblank(rdev, 0); + radeon_wait_for_vblank(rdev, 1); + + r600_enable_spll_bypass(rdev, true); + r600_wait_for_spll_change(rdev); + r600_enable_spll_bypass(rdev, false); + r600_wait_for_spll_change(rdev); + + r600_enable_spll_bypass(rdev, true); + r600_wait_for_spll_change(rdev); + r600_enable_spll_bypass(rdev, false); + r600_wait_for_spll_change(rdev); + + r600_enable_sclk_control(rdev, true); + r600_enable_mclk_control(rdev, true); +} + +void r600_stop_dpm(struct radeon_device *rdev) +{ + r600_dynamicpm_enable(rdev, false); +} + +int r600_dpm_pre_set_power_state(struct radeon_device *rdev) +{ + return 0; +} + +void r600_dpm_post_set_power_state(struct radeon_device *rdev) +{ + +} + +bool r600_is_uvd_state(u32 class, u32 class2) +{ + if (class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + return true; + if (class & ATOM_PPLIB_CLASSIFICATION_HD2STATE) + return true; + if (class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) + return true; + if (class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) + return true; + if (class2 & ATOM_PPLIB_CLASSIFICATION2_MVC) + return true; + return false; +} + +static int r600_set_thermal_temperature_range(struct radeon_device *rdev, + int min_temp, int max_temp) +{ + int low_temp = 0 * 1000; + int high_temp = 255 * 1000; + + if (low_temp < min_temp) + low_temp = min_temp; + if (high_temp > max_temp) + high_temp = max_temp; + if (high_temp < low_temp) { + DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp); + return -EINVAL; + } + + WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK); + WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK); + WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK); + + rdev->pm.dpm.thermal.min_temp = low_temp; + rdev->pm.dpm.thermal.max_temp = high_temp; + + return 0; +} + +bool r600_is_internal_thermal_sensor(enum radeon_int_thermal_type sensor) +{ + switch (sensor) { + case THERMAL_TYPE_RV6XX: + case THERMAL_TYPE_RV770: + case THERMAL_TYPE_EVERGREEN: + case THERMAL_TYPE_SUMO: + case THERMAL_TYPE_NI: + case THERMAL_TYPE_SI: + case THERMAL_TYPE_CI: + case THERMAL_TYPE_KV: + return true; + case THERMAL_TYPE_ADT7473_WITH_INTERNAL: + case THERMAL_TYPE_EMC2103_WITH_INTERNAL: + return false; /* need special handling */ + case THERMAL_TYPE_NONE: + case THERMAL_TYPE_EXTERNAL: + case THERMAL_TYPE_EXTERNAL_GPIO: + default: + return false; + } +} + +int r600_dpm_late_enable(struct radeon_device *rdev) +{ + int ret; + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + ret = r600_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) + return ret; + rdev->irq.dpm_thermal = true; + radeon_irq_set(rdev); + } + + return 0; +} + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; + struct _ATOM_PPLIB_POWERPLAYTABLE4 pplib4; + struct _ATOM_PPLIB_POWERPLAYTABLE5 pplib5; +}; + +union fan_info { + struct _ATOM_PPLIB_FANTABLE fan; + struct _ATOM_PPLIB_FANTABLE2 fan2; + struct _ATOM_PPLIB_FANTABLE3 fan3; +}; + +static int r600_parse_clk_voltage_dep_table(struct radeon_clock_voltage_dependency_table *radeon_table, + ATOM_PPLIB_Clock_Voltage_Dependency_Table *atom_table) +{ + u32 size = atom_table->ucNumEntries * + sizeof(struct radeon_clock_voltage_dependency_entry); + int i; + ATOM_PPLIB_Clock_Voltage_Dependency_Record *entry; + + radeon_table->entries = kzalloc(size, GFP_KERNEL); + if (!radeon_table->entries) + return -ENOMEM; + + entry = &atom_table->entries[0]; + for (i = 0; i < atom_table->ucNumEntries; i++) { + radeon_table->entries[i].clk = le16_to_cpu(entry->usClockLow) | + (entry->ucClockHigh << 16); + radeon_table->entries[i].v = le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_Clock_Voltage_Dependency_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_Clock_Voltage_Dependency_Record)); + } + radeon_table->count = atom_table->ucNumEntries; + + return 0; +} + +int r600_get_platform_caps(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps); + rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime); + rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime); + + return 0; +} + +/* sizeof(ATOM_PPLIB_EXTENDEDHEADER) */ +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2 12 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3 14 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4 16 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V5 18 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6 20 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7 22 + +int r600_parse_extended_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + union power_info *power_info; + union fan_info *fan_info; + ATOM_PPLIB_Clock_Voltage_Dependency_Table *dep_table; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + int ret, i; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + /* fan table */ + if (le16_to_cpu(power_info->pplib.usTableSize) >= + sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE3)) { + if (power_info->pplib3.usFanTableOffset) { + fan_info = (union fan_info *)(mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib3.usFanTableOffset)); + rdev->pm.dpm.fan.t_hyst = fan_info->fan.ucTHyst; + rdev->pm.dpm.fan.t_min = le16_to_cpu(fan_info->fan.usTMin); + rdev->pm.dpm.fan.t_med = le16_to_cpu(fan_info->fan.usTMed); + rdev->pm.dpm.fan.t_high = le16_to_cpu(fan_info->fan.usTHigh); + rdev->pm.dpm.fan.pwm_min = le16_to_cpu(fan_info->fan.usPWMMin); + rdev->pm.dpm.fan.pwm_med = le16_to_cpu(fan_info->fan.usPWMMed); + rdev->pm.dpm.fan.pwm_high = le16_to_cpu(fan_info->fan.usPWMHigh); + if (fan_info->fan.ucFanTableFormat >= 2) + rdev->pm.dpm.fan.t_max = le16_to_cpu(fan_info->fan2.usTMax); + else + rdev->pm.dpm.fan.t_max = 10900; + rdev->pm.dpm.fan.cycle_delay = 100000; + if (fan_info->fan.ucFanTableFormat >= 3) { + rdev->pm.dpm.fan.control_mode = fan_info->fan3.ucFanControlMode; + rdev->pm.dpm.fan.default_max_fan_pwm = + le16_to_cpu(fan_info->fan3.usFanPWMMax); + rdev->pm.dpm.fan.default_fan_output_sensitivity = 4836; + rdev->pm.dpm.fan.fan_output_sensitivity = + le16_to_cpu(fan_info->fan3.usFanOutputSensitivity); + } + rdev->pm.dpm.fan.ucode_fan_control = true; + } + } + + /* clock dependancy tables, shedding tables */ + if (le16_to_cpu(power_info->pplib.usTableSize) >= + sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE4)) { + if (power_info->pplib4.usVddcDependencyOnSCLKOffset) { + dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usVddcDependencyOnSCLKOffset)); + ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk, + dep_table); + if (ret) + return ret; + } + if (power_info->pplib4.usVddciDependencyOnMCLKOffset) { + dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usVddciDependencyOnMCLKOffset)); + ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + dep_table); + if (ret) { + kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries); + return ret; + } + } + if (power_info->pplib4.usVddcDependencyOnMCLKOffset) { + dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usVddcDependencyOnMCLKOffset)); + ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + dep_table); + if (ret) { + kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries); + kfree(rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries); + return ret; + } + } + if (power_info->pplib4.usMvddDependencyOnMCLKOffset) { + dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usMvddDependencyOnMCLKOffset)); + ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk, + dep_table); + if (ret) { + kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries); + kfree(rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries); + kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries); + return ret; + } + } + if (power_info->pplib4.usMaxClockVoltageOnDCOffset) { + ATOM_PPLIB_Clock_Voltage_Limit_Table *clk_v = + (ATOM_PPLIB_Clock_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usMaxClockVoltageOnDCOffset)); + if (clk_v->ucNumEntries) { + rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk = + le16_to_cpu(clk_v->entries[0].usSclkLow) | + (clk_v->entries[0].ucSclkHigh << 16); + rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk = + le16_to_cpu(clk_v->entries[0].usMclkLow) | + (clk_v->entries[0].ucMclkHigh << 16); + rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc = + le16_to_cpu(clk_v->entries[0].usVddc); + rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddci = + le16_to_cpu(clk_v->entries[0].usVddci); + } + } + if (power_info->pplib4.usVddcPhaseShedLimitsTableOffset) { + ATOM_PPLIB_PhaseSheddingLimits_Table *psl = + (ATOM_PPLIB_PhaseSheddingLimits_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usVddcPhaseShedLimitsTableOffset)); + ATOM_PPLIB_PhaseSheddingLimits_Record *entry; + + rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries = + kzalloc(psl->ucNumEntries * + sizeof(struct radeon_phase_shedding_limits_entry), + GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + + entry = &psl->entries[0]; + for (i = 0; i < psl->ucNumEntries; i++) { + rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries[i].sclk = + le16_to_cpu(entry->usSclkLow) | (entry->ucSclkHigh << 16); + rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries[i].mclk = + le16_to_cpu(entry->usMclkLow) | (entry->ucMclkHigh << 16); + rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries[i].voltage = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_PhaseSheddingLimits_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_PhaseSheddingLimits_Record)); + } + rdev->pm.dpm.dyn_state.phase_shedding_limits_table.count = + psl->ucNumEntries; + } + } + + /* cac data */ + if (le16_to_cpu(power_info->pplib.usTableSize) >= + sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE5)) { + rdev->pm.dpm.tdp_limit = le32_to_cpu(power_info->pplib5.ulTDPLimit); + rdev->pm.dpm.near_tdp_limit = le32_to_cpu(power_info->pplib5.ulNearTDPLimit); + rdev->pm.dpm.near_tdp_limit_adjusted = rdev->pm.dpm.near_tdp_limit; + rdev->pm.dpm.tdp_od_limit = le16_to_cpu(power_info->pplib5.usTDPODLimit); + if (rdev->pm.dpm.tdp_od_limit) + rdev->pm.dpm.power_control = true; + else + rdev->pm.dpm.power_control = false; + rdev->pm.dpm.tdp_adjustment = 0; + rdev->pm.dpm.sq_ramping_threshold = le32_to_cpu(power_info->pplib5.ulSQRampingThreshold); + rdev->pm.dpm.cac_leakage = le32_to_cpu(power_info->pplib5.ulCACLeakage); + rdev->pm.dpm.load_line_slope = le16_to_cpu(power_info->pplib5.usLoadLineSlope); + if (power_info->pplib5.usCACLeakageTableOffset) { + ATOM_PPLIB_CAC_Leakage_Table *cac_table = + (ATOM_PPLIB_CAC_Leakage_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib5.usCACLeakageTableOffset)); + ATOM_PPLIB_CAC_Leakage_Record *entry; + u32 size = cac_table->ucNumEntries * sizeof(struct radeon_cac_leakage_table); + rdev->pm.dpm.dyn_state.cac_leakage_table.entries = kzalloc(size, GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.cac_leakage_table.entries) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + entry = &cac_table->entries[0]; + for (i = 0; i < cac_table->ucNumEntries; i++) { + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) { + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc1 = + le16_to_cpu(entry->usVddc1); + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc2 = + le16_to_cpu(entry->usVddc2); + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc3 = + le16_to_cpu(entry->usVddc3); + } else { + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc = + le16_to_cpu(entry->usVddc); + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].leakage = + le32_to_cpu(entry->ulLeakageValue); + } + entry = (ATOM_PPLIB_CAC_Leakage_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_CAC_Leakage_Record)); + } + rdev->pm.dpm.dyn_state.cac_leakage_table.count = cac_table->ucNumEntries; + } + } + + /* ext tables */ + if (le16_to_cpu(power_info->pplib.usTableSize) >= + sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE3)) { + ATOM_PPLIB_EXTENDEDHEADER *ext_hdr = (ATOM_PPLIB_EXTENDEDHEADER *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib3.usExtendendedHeaderOffset)); + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2) && + ext_hdr->usVCETableOffset) { + VCEClockInfoArray *array = (VCEClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usVCETableOffset) + 1); + ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *limits = + (ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usVCETableOffset) + 1 + + 1 + array->ucNumEntries * sizeof(VCEClockInfo)); + ATOM_PPLIB_VCE_State_Table *states = + (ATOM_PPLIB_VCE_State_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usVCETableOffset) + 1 + + 1 + (array->ucNumEntries * sizeof (VCEClockInfo)) + + 1 + (limits->numEntries * sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record))); + ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record *entry; + ATOM_PPLIB_VCE_State_Record *state_entry; + VCEClockInfo *vce_clk; + u32 size = limits->numEntries * + sizeof(struct radeon_vce_clock_voltage_dependency_entry); + rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries = + kzalloc(size, GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count = + limits->numEntries; + entry = &limits->entries[0]; + state_entry = &states->entries[0]; + for (i = 0; i < limits->numEntries; i++) { + vce_clk = (VCEClockInfo *) + ((u8 *)&array->entries[0] + + (entry->ucVCEClockInfoIndex * sizeof(VCEClockInfo))); + rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].evclk = + le16_to_cpu(vce_clk->usEVClkLow) | (vce_clk->ucEVClkHigh << 16); + rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].ecclk = + le16_to_cpu(vce_clk->usECClkLow) | (vce_clk->ucECClkHigh << 16); + rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].v = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record)); + } + for (i = 0; i < states->numEntries; i++) { + if (i >= RADEON_MAX_VCE_LEVELS) + break; + vce_clk = (VCEClockInfo *) + ((u8 *)&array->entries[0] + + (state_entry->ucVCEClockInfoIndex * sizeof(VCEClockInfo))); + rdev->pm.dpm.vce_states[i].evclk = + le16_to_cpu(vce_clk->usEVClkLow) | (vce_clk->ucEVClkHigh << 16); + rdev->pm.dpm.vce_states[i].ecclk = + le16_to_cpu(vce_clk->usECClkLow) | (vce_clk->ucECClkHigh << 16); + rdev->pm.dpm.vce_states[i].clk_idx = + state_entry->ucClockInfoIndex & 0x3f; + rdev->pm.dpm.vce_states[i].pstate = + (state_entry->ucClockInfoIndex & 0xc0) >> 6; + state_entry = (ATOM_PPLIB_VCE_State_Record *) + ((u8 *)state_entry + sizeof(ATOM_PPLIB_VCE_State_Record)); + } + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3) && + ext_hdr->usUVDTableOffset) { + UVDClockInfoArray *array = (UVDClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usUVDTableOffset) + 1); + ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table *limits = + (ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usUVDTableOffset) + 1 + + 1 + (array->ucNumEntries * sizeof (UVDClockInfo))); + ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record *entry; + u32 size = limits->numEntries * + sizeof(struct radeon_uvd_clock_voltage_dependency_entry); + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries = + kzalloc(size, GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count = + limits->numEntries; + entry = &limits->entries[0]; + for (i = 0; i < limits->numEntries; i++) { + UVDClockInfo *uvd_clk = (UVDClockInfo *) + ((u8 *)&array->entries[0] + + (entry->ucUVDClockInfoIndex * sizeof(UVDClockInfo))); + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].vclk = + le16_to_cpu(uvd_clk->usVClkLow) | (uvd_clk->ucVClkHigh << 16); + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].dclk = + le16_to_cpu(uvd_clk->usDClkLow) | (uvd_clk->ucDClkHigh << 16); + rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].v = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record)); + } + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4) && + ext_hdr->usSAMUTableOffset) { + ATOM_PPLIB_SAMClk_Voltage_Limit_Table *limits = + (ATOM_PPLIB_SAMClk_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usSAMUTableOffset) + 1); + ATOM_PPLIB_SAMClk_Voltage_Limit_Record *entry; + u32 size = limits->numEntries * + sizeof(struct radeon_clock_voltage_dependency_entry); + rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries = + kzalloc(size, GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count = + limits->numEntries; + entry = &limits->entries[0]; + for (i = 0; i < limits->numEntries; i++) { + rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[i].clk = + le16_to_cpu(entry->usSAMClockLow) | (entry->ucSAMClockHigh << 16); + rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[i].v = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_SAMClk_Voltage_Limit_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_SAMClk_Voltage_Limit_Record)); + } + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V5) && + ext_hdr->usPPMTableOffset) { + ATOM_PPLIB_PPM_Table *ppm = (ATOM_PPLIB_PPM_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usPPMTableOffset)); + rdev->pm.dpm.dyn_state.ppm_table = + kzalloc(sizeof(struct radeon_ppm_table), GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.ppm_table) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + rdev->pm.dpm.dyn_state.ppm_table->ppm_design = ppm->ucPpmDesign; + rdev->pm.dpm.dyn_state.ppm_table->cpu_core_number = + le16_to_cpu(ppm->usCpuCoreNumber); + rdev->pm.dpm.dyn_state.ppm_table->platform_tdp = + le32_to_cpu(ppm->ulPlatformTDP); + rdev->pm.dpm.dyn_state.ppm_table->small_ac_platform_tdp = + le32_to_cpu(ppm->ulSmallACPlatformTDP); + rdev->pm.dpm.dyn_state.ppm_table->platform_tdc = + le32_to_cpu(ppm->ulPlatformTDC); + rdev->pm.dpm.dyn_state.ppm_table->small_ac_platform_tdc = + le32_to_cpu(ppm->ulSmallACPlatformTDC); + rdev->pm.dpm.dyn_state.ppm_table->apu_tdp = + le32_to_cpu(ppm->ulApuTDP); + rdev->pm.dpm.dyn_state.ppm_table->dgpu_tdp = + le32_to_cpu(ppm->ulDGpuTDP); + rdev->pm.dpm.dyn_state.ppm_table->dgpu_ulv_power = + le32_to_cpu(ppm->ulDGpuUlvPower); + rdev->pm.dpm.dyn_state.ppm_table->tj_max = + le32_to_cpu(ppm->ulTjmax); + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6) && + ext_hdr->usACPTableOffset) { + ATOM_PPLIB_ACPClk_Voltage_Limit_Table *limits = + (ATOM_PPLIB_ACPClk_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usACPTableOffset) + 1); + ATOM_PPLIB_ACPClk_Voltage_Limit_Record *entry; + u32 size = limits->numEntries * + sizeof(struct radeon_clock_voltage_dependency_entry); + rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries = + kzalloc(size, GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count = + limits->numEntries; + entry = &limits->entries[0]; + for (i = 0; i < limits->numEntries; i++) { + rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[i].clk = + le16_to_cpu(entry->usACPClockLow) | (entry->ucACPClockHigh << 16); + rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[i].v = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_ACPClk_Voltage_Limit_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_ACPClk_Voltage_Limit_Record)); + } + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7) && + ext_hdr->usPowerTuneTableOffset) { + u8 rev = *(u8 *)(mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usPowerTuneTableOffset)); + ATOM_PowerTune_Table *pt; + rdev->pm.dpm.dyn_state.cac_tdp_table = + kzalloc(sizeof(struct radeon_cac_tdp_table), GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.cac_tdp_table) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + if (rev > 0) { + ATOM_PPLIB_POWERTUNE_Table_V1 *ppt = (ATOM_PPLIB_POWERTUNE_Table_V1 *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usPowerTuneTableOffset)); + rdev->pm.dpm.dyn_state.cac_tdp_table->maximum_power_delivery_limit = + le16_to_cpu(ppt->usMaximumPowerDeliveryLimit); + pt = &ppt->power_tune_table; + } else { + ATOM_PPLIB_POWERTUNE_Table *ppt = (ATOM_PPLIB_POWERTUNE_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usPowerTuneTableOffset)); + rdev->pm.dpm.dyn_state.cac_tdp_table->maximum_power_delivery_limit = 255; + pt = &ppt->power_tune_table; + } + rdev->pm.dpm.dyn_state.cac_tdp_table->tdp = le16_to_cpu(pt->usTDP); + rdev->pm.dpm.dyn_state.cac_tdp_table->configurable_tdp = + le16_to_cpu(pt->usConfigurableTDP); + rdev->pm.dpm.dyn_state.cac_tdp_table->tdc = le16_to_cpu(pt->usTDC); + rdev->pm.dpm.dyn_state.cac_tdp_table->battery_power_limit = + le16_to_cpu(pt->usBatteryPowerLimit); + rdev->pm.dpm.dyn_state.cac_tdp_table->small_power_limit = + le16_to_cpu(pt->usSmallPowerLimit); + rdev->pm.dpm.dyn_state.cac_tdp_table->low_cac_leakage = + le16_to_cpu(pt->usLowCACLeakage); + rdev->pm.dpm.dyn_state.cac_tdp_table->high_cac_leakage = + le16_to_cpu(pt->usHighCACLeakage); + } + } + + return 0; +} + +void r600_free_extended_power_table(struct radeon_device *rdev) +{ + struct radeon_dpm_dynamic_state *dyn_state = &rdev->pm.dpm.dyn_state; + + kfree(dyn_state->vddc_dependency_on_sclk.entries); + kfree(dyn_state->vddci_dependency_on_mclk.entries); + kfree(dyn_state->vddc_dependency_on_mclk.entries); + kfree(dyn_state->mvdd_dependency_on_mclk.entries); + kfree(dyn_state->cac_leakage_table.entries); + kfree(dyn_state->phase_shedding_limits_table.entries); + kfree(dyn_state->ppm_table); + kfree(dyn_state->cac_tdp_table); + kfree(dyn_state->vce_clock_voltage_dependency_table.entries); + kfree(dyn_state->uvd_clock_voltage_dependency_table.entries); + kfree(dyn_state->samu_clock_voltage_dependency_table.entries); + kfree(dyn_state->acp_clock_voltage_dependency_table.entries); +} + +enum radeon_pcie_gen r600_get_pcie_gen_support(struct radeon_device *rdev, + u32 sys_mask, + enum radeon_pcie_gen asic_gen, + enum radeon_pcie_gen default_gen) +{ + switch (asic_gen) { + case RADEON_PCIE_GEN1: + return RADEON_PCIE_GEN1; + case RADEON_PCIE_GEN2: + return RADEON_PCIE_GEN2; + case RADEON_PCIE_GEN3: + return RADEON_PCIE_GEN3; + default: + if ((sys_mask & DRM_PCIE_SPEED_80) && (default_gen == RADEON_PCIE_GEN3)) + return RADEON_PCIE_GEN3; + else if ((sys_mask & DRM_PCIE_SPEED_50) && (default_gen == RADEON_PCIE_GEN2)) + return RADEON_PCIE_GEN2; + else + return RADEON_PCIE_GEN1; + } + return RADEON_PCIE_GEN1; +} + +u16 r600_get_pcie_lane_support(struct radeon_device *rdev, + u16 asic_lanes, + u16 default_lanes) +{ + switch (asic_lanes) { + case 0: + default: + return default_lanes; + case 1: + return 1; + case 2: + return 2; + case 4: + return 4; + case 8: + return 8; + case 12: + return 12; + case 16: + return 16; + } +} + +u8 r600_encode_pci_lane_width(u32 lanes) +{ + u8 encoded_lanes[] = { 0, 1, 2, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6 }; + + if (lanes > 16) + return 0; + + return encoded_lanes[lanes]; +} diff --git a/sys/dev/pci/drm/radeon/r600_dpm.h b/sys/dev/pci/drm/radeon/r600_dpm.h new file mode 100644 index 00000000000..bd499d749bc --- /dev/null +++ b/sys/dev/pci/drm/radeon/r600_dpm.h @@ -0,0 +1,236 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __R600_DPM_H__ +#define __R600_DPM_H__ + +#define R600_ASI_DFLT 10000 +#define R600_BSP_DFLT 0x41EB +#define R600_BSU_DFLT 0x2 +#define R600_AH_DFLT 5 +#define R600_RLP_DFLT 25 +#define R600_RMP_DFLT 65 +#define R600_LHP_DFLT 40 +#define R600_LMP_DFLT 15 +#define R600_TD_DFLT 0 +#define R600_UTC_DFLT_00 0x24 +#define R600_UTC_DFLT_01 0x22 +#define R600_UTC_DFLT_02 0x22 +#define R600_UTC_DFLT_03 0x22 +#define R600_UTC_DFLT_04 0x22 +#define R600_UTC_DFLT_05 0x22 +#define R600_UTC_DFLT_06 0x22 +#define R600_UTC_DFLT_07 0x22 +#define R600_UTC_DFLT_08 0x22 +#define R600_UTC_DFLT_09 0x22 +#define R600_UTC_DFLT_10 0x22 +#define R600_UTC_DFLT_11 0x22 +#define R600_UTC_DFLT_12 0x22 +#define R600_UTC_DFLT_13 0x22 +#define R600_UTC_DFLT_14 0x22 +#define R600_DTC_DFLT_00 0x24 +#define R600_DTC_DFLT_01 0x22 +#define R600_DTC_DFLT_02 0x22 +#define R600_DTC_DFLT_03 0x22 +#define R600_DTC_DFLT_04 0x22 +#define R600_DTC_DFLT_05 0x22 +#define R600_DTC_DFLT_06 0x22 +#define R600_DTC_DFLT_07 0x22 +#define R600_DTC_DFLT_08 0x22 +#define R600_DTC_DFLT_09 0x22 +#define R600_DTC_DFLT_10 0x22 +#define R600_DTC_DFLT_11 0x22 +#define R600_DTC_DFLT_12 0x22 +#define R600_DTC_DFLT_13 0x22 +#define R600_DTC_DFLT_14 0x22 +#define R600_VRC_DFLT 0x0000C003 +#define R600_VOLTAGERESPONSETIME_DFLT 1000 +#define R600_BACKBIASRESPONSETIME_DFLT 1000 +#define R600_VRU_DFLT 0x3 +#define R600_SPLLSTEPTIME_DFLT 0x1000 +#define R600_SPLLSTEPUNIT_DFLT 0x3 +#define R600_TPU_DFLT 0 +#define R600_TPC_DFLT 0x200 +#define R600_SSTU_DFLT 0 +#define R600_SST_DFLT 0x00C8 +#define R600_GICST_DFLT 0x200 +#define R600_FCT_DFLT 0x0400 +#define R600_FCTU_DFLT 0 +#define R600_CTXCGTT3DRPHC_DFLT 0x20 +#define R600_CTXCGTT3DRSDC_DFLT 0x40 +#define R600_VDDC3DOORPHC_DFLT 0x100 +#define R600_VDDC3DOORSDC_DFLT 0x7 +#define R600_VDDC3DOORSU_DFLT 0 +#define R600_MPLLLOCKTIME_DFLT 100 +#define R600_MPLLRESETTIME_DFLT 150 +#define R600_VCOSTEPPCT_DFLT 20 +#define R600_ENDINGVCOSTEPPCT_DFLT 5 +#define R600_REFERENCEDIVIDER_DFLT 4 + +#define R600_PM_NUMBER_OF_TC 15 +#define R600_PM_NUMBER_OF_SCLKS 20 +#define R600_PM_NUMBER_OF_MCLKS 4 +#define R600_PM_NUMBER_OF_VOLTAGE_LEVELS 4 +#define R600_PM_NUMBER_OF_ACTIVITY_LEVELS 3 + +/* XXX are these ok? */ +#define R600_TEMP_RANGE_MIN (90 * 1000) +#define R600_TEMP_RANGE_MAX (120 * 1000) + +#define FDO_PWM_MODE_STATIC 1 +#define FDO_PWM_MODE_STATIC_RPM 5 + +enum r600_power_level { + R600_POWER_LEVEL_LOW = 0, + R600_POWER_LEVEL_MEDIUM = 1, + R600_POWER_LEVEL_HIGH = 2, + R600_POWER_LEVEL_CTXSW = 3, +}; + +enum r600_td { + R600_TD_AUTO, + R600_TD_UP, + R600_TD_DOWN, +}; + +enum r600_display_watermark { + R600_DISPLAY_WATERMARK_LOW = 0, + R600_DISPLAY_WATERMARK_HIGH = 1, +}; + +enum r600_display_gap +{ + R600_PM_DISPLAY_GAP_VBLANK_OR_WM = 0, + R600_PM_DISPLAY_GAP_VBLANK = 1, + R600_PM_DISPLAY_GAP_WATERMARK = 2, + R600_PM_DISPLAY_GAP_IGNORE = 3, +}; + +extern const u32 r600_utc[R600_PM_NUMBER_OF_TC]; +extern const u32 r600_dtc[R600_PM_NUMBER_OF_TC]; + +void r600_dpm_print_class_info(u32 class, u32 class2); +void r600_dpm_print_cap_info(u32 caps); +void r600_dpm_print_ps_status(struct radeon_device *rdev, + struct radeon_ps *rps); +u32 r600_dpm_get_vblank_time(struct radeon_device *rdev); +u32 r600_dpm_get_vrefresh(struct radeon_device *rdev); +bool r600_is_uvd_state(u32 class, u32 class2); +void r600_calculate_u_and_p(u32 i, u32 r_c, u32 p_b, + u32 *p, u32 *u); +int r600_calculate_at(u32 t, u32 h, u32 fh, u32 fl, u32 *tl, u32 *th); +void r600_gfx_clockgating_enable(struct radeon_device *rdev, bool enable); +void r600_dynamicpm_enable(struct radeon_device *rdev, bool enable); +void r600_enable_thermal_protection(struct radeon_device *rdev, bool enable); +void r600_enable_acpi_pm(struct radeon_device *rdev); +void r600_enable_dynamic_pcie_gen2(struct radeon_device *rdev, bool enable); +bool r600_dynamicpm_enabled(struct radeon_device *rdev); +void r600_enable_sclk_control(struct radeon_device *rdev, bool enable); +void r600_enable_mclk_control(struct radeon_device *rdev, bool enable); +void r600_enable_spll_bypass(struct radeon_device *rdev, bool enable); +void r600_wait_for_spll_change(struct radeon_device *rdev); +void r600_set_bsp(struct radeon_device *rdev, u32 u, u32 p); +void r600_set_at(struct radeon_device *rdev, + u32 l_to_m, u32 m_to_h, + u32 h_to_m, u32 m_to_l); +void r600_set_tc(struct radeon_device *rdev, u32 index, u32 u_t, u32 d_t); +void r600_select_td(struct radeon_device *rdev, enum r600_td td); +void r600_set_vrc(struct radeon_device *rdev, u32 vrv); +void r600_set_tpu(struct radeon_device *rdev, u32 u); +void r600_set_tpc(struct radeon_device *rdev, u32 c); +void r600_set_sstu(struct radeon_device *rdev, u32 u); +void r600_set_sst(struct radeon_device *rdev, u32 t); +void r600_set_git(struct radeon_device *rdev, u32 t); +void r600_set_fctu(struct radeon_device *rdev, u32 u); +void r600_set_fct(struct radeon_device *rdev, u32 t); +void r600_set_ctxcgtt3d_rphc(struct radeon_device *rdev, u32 p); +void r600_set_ctxcgtt3d_rsdc(struct radeon_device *rdev, u32 s); +void r600_set_vddc3d_oorsu(struct radeon_device *rdev, u32 u); +void r600_set_vddc3d_oorphc(struct radeon_device *rdev, u32 p); +void r600_set_vddc3d_oorsdc(struct radeon_device *rdev, u32 s); +void r600_set_mpll_lock_time(struct radeon_device *rdev, u32 lock_time); +void r600_set_mpll_reset_time(struct radeon_device *rdev, u32 reset_time); +void r600_engine_clock_entry_enable(struct radeon_device *rdev, + u32 index, bool enable); +void r600_engine_clock_entry_enable_pulse_skipping(struct radeon_device *rdev, + u32 index, bool enable); +void r600_engine_clock_entry_enable_post_divider(struct radeon_device *rdev, + u32 index, bool enable); +void r600_engine_clock_entry_set_post_divider(struct radeon_device *rdev, + u32 index, u32 divider); +void r600_engine_clock_entry_set_reference_divider(struct radeon_device *rdev, + u32 index, u32 divider); +void r600_engine_clock_entry_set_feedback_divider(struct radeon_device *rdev, + u32 index, u32 divider); +void r600_engine_clock_entry_set_step_time(struct radeon_device *rdev, + u32 index, u32 step_time); +void r600_vid_rt_set_ssu(struct radeon_device *rdev, u32 u); +void r600_vid_rt_set_vru(struct radeon_device *rdev, u32 u); +void r600_vid_rt_set_vrt(struct radeon_device *rdev, u32 rt); +void r600_voltage_control_enable_pins(struct radeon_device *rdev, + u64 mask); +void r600_voltage_control_program_voltages(struct radeon_device *rdev, + enum r600_power_level index, u64 pins); +void r600_voltage_control_deactivate_static_control(struct radeon_device *rdev, + u64 mask); +void r600_power_level_enable(struct radeon_device *rdev, + enum r600_power_level index, bool enable); +void r600_power_level_set_voltage_index(struct radeon_device *rdev, + enum r600_power_level index, u32 voltage_index); +void r600_power_level_set_mem_clock_index(struct radeon_device *rdev, + enum r600_power_level index, u32 mem_clock_index); +void r600_power_level_set_eng_clock_index(struct radeon_device *rdev, + enum r600_power_level index, u32 eng_clock_index); +void r600_power_level_set_watermark_id(struct radeon_device *rdev, + enum r600_power_level index, + enum r600_display_watermark watermark_id); +void r600_power_level_set_pcie_gen2(struct radeon_device *rdev, + enum r600_power_level index, bool compatible); +enum r600_power_level r600_power_level_get_current_index(struct radeon_device *rdev); +enum r600_power_level r600_power_level_get_target_index(struct radeon_device *rdev); +void r600_power_level_set_enter_index(struct radeon_device *rdev, + enum r600_power_level index); +void r600_wait_for_power_level_unequal(struct radeon_device *rdev, + enum r600_power_level index); +void r600_wait_for_power_level(struct radeon_device *rdev, + enum r600_power_level index); +void r600_start_dpm(struct radeon_device *rdev); +void r600_stop_dpm(struct radeon_device *rdev); + +bool r600_is_internal_thermal_sensor(enum radeon_int_thermal_type sensor); + +int r600_get_platform_caps(struct radeon_device *rdev); + +int r600_parse_extended_power_table(struct radeon_device *rdev); +void r600_free_extended_power_table(struct radeon_device *rdev); + +enum radeon_pcie_gen r600_get_pcie_gen_support(struct radeon_device *rdev, + u32 sys_mask, + enum radeon_pcie_gen asic_gen, + enum radeon_pcie_gen default_gen); + +u16 r600_get_pcie_lane_support(struct radeon_device *rdev, + u16 asic_lanes, + u16 default_lanes); +u8 r600_encode_pci_lane_width(u32 lanes); + +#endif diff --git a/sys/dev/pci/drm/radeon/r600_hdmi.c b/sys/dev/pci/drm/radeon/r600_hdmi.c index 9871a2e00f6..07df68f80b8 100644 --- a/sys/dev/pci/drm/radeon/r600_hdmi.c +++ b/sys/dev/pci/drm/radeon/r600_hdmi.c @@ -1,4 +1,3 @@ -/* $OpenBSD: r600_hdmi.c,v 1.4 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -24,10 +23,12 @@ * * Authors: Christian König */ +#include #include #include #include "radeon.h" #include "radeon_asic.h" +#include "radeon_audio.h" #include "r600d.h" #include "atom.h" @@ -54,154 +55,168 @@ enum r600_hdmi_iec_status_bits { AUDIO_STATUS_LEVEL = 0x80 }; -static const struct radeon_hdmi_acr r600_hdmi_predefined_acr[] = { - /* 32kHz 44.1kHz 48kHz */ - /* Clock N CTS N CTS N CTS */ - { 25174, 4576, 28125, 7007, 31250, 6864, 28125 }, /* 25,20/1.001 MHz */ - { 25200, 4096, 25200, 6272, 28000, 6144, 25200 }, /* 25.20 MHz */ - { 27000, 4096, 27000, 6272, 30000, 6144, 27000 }, /* 27.00 MHz */ - { 27027, 4096, 27027, 6272, 30030, 6144, 27027 }, /* 27.00*1.001 MHz */ - { 54000, 4096, 54000, 6272, 60000, 6144, 54000 }, /* 54.00 MHz */ - { 54054, 4096, 54054, 6272, 60060, 6144, 54054 }, /* 54.00*1.001 MHz */ - { 74175, 11648, 210937, 17836, 234375, 11648, 140625 }, /* 74.25/1.001 MHz */ - { 74250, 4096, 74250, 6272, 82500, 6144, 74250 }, /* 74.25 MHz */ - { 148351, 11648, 421875, 8918, 234375, 5824, 140625 }, /* 148.50/1.001 MHz */ - { 148500, 4096, 148500, 6272, 165000, 6144, 148500 }, /* 148.50 MHz */ - { 0, 4096, 0, 6272, 0, 6144, 0 } /* Other */ -}; - -/* - * calculate CTS value if it's not found in the table - */ -static void r600_hdmi_calc_cts(uint32_t clock, int *CTS, int N, int freq) +static struct r600_audio_pin r600_audio_status(struct radeon_device *rdev) { - if (*CTS == 0) - *CTS = clock * N / (128 * freq) * 1000; - DRM_DEBUG("Using ACR timing N=%d CTS=%d for frequency %d\n", - N, *CTS, freq); -} + struct r600_audio_pin status; + uint32_t value; -struct radeon_hdmi_acr r600_hdmi_acr(uint32_t clock) -{ - struct radeon_hdmi_acr res; - u8 i; + value = RREG32(R600_AUDIO_RATE_BPS_CHANNEL); + + /* number of channels */ + status.channels = (value & 0x7) + 1; + + /* bits per sample */ + switch ((value & 0xF0) >> 4) { + case 0x0: + status.bits_per_sample = 8; + break; + case 0x1: + status.bits_per_sample = 16; + break; + case 0x2: + status.bits_per_sample = 20; + break; + case 0x3: + status.bits_per_sample = 24; + break; + case 0x4: + status.bits_per_sample = 32; + break; + default: + dev_err(rdev->dev, "Unknown bits per sample 0x%x, using 16\n", + (int)value); + status.bits_per_sample = 16; + } - for (i = 0; r600_hdmi_predefined_acr[i].clock != clock && - r600_hdmi_predefined_acr[i].clock != 0; i++) - ; - res = r600_hdmi_predefined_acr[i]; + /* current sampling rate in HZ */ + if (value & 0x4000) + status.rate = 44100; + else + status.rate = 48000; + status.rate *= ((value >> 11) & 0x7) + 1; + status.rate /= ((value >> 8) & 0x7) + 1; + + value = RREG32(R600_AUDIO_STATUS_BITS); - /* In case some CTS are missing */ - r600_hdmi_calc_cts(clock, &res.cts_32khz, res.n_32khz, 32000); - r600_hdmi_calc_cts(clock, &res.cts_44_1khz, res.n_44_1khz, 44100); - r600_hdmi_calc_cts(clock, &res.cts_48khz, res.n_48khz, 48000); + /* iec 60958 status bits */ + status.status_bits = value & 0xff; - return res; + /* iec 60958 category code */ + status.category_code = (value >> 8) & 0xff; + + return status; } /* - * update the N and CTS parameters for a given pixel clock rate + * update all hdmi interfaces with current audio parameters */ -static void r600_hdmi_update_ACR(struct drm_encoder *encoder, uint32_t clock) +void r600_audio_update_hdmi(struct work_struct *work) { - struct drm_device *dev = encoder->dev; - struct radeon_device *rdev = dev->dev_private; - struct radeon_hdmi_acr acr = r600_hdmi_acr(clock); - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - uint32_t offset = dig->afmt->offset; + struct radeon_device *rdev = container_of(work, struct radeon_device, + audio_work); + struct drm_device *dev = rdev->ddev; + struct r600_audio_pin audio_status = r600_audio_status(rdev); + struct drm_encoder *encoder; + bool changed = false; + + if (rdev->audio.pin[0].channels != audio_status.channels || + rdev->audio.pin[0].rate != audio_status.rate || + rdev->audio.pin[0].bits_per_sample != audio_status.bits_per_sample || + rdev->audio.pin[0].status_bits != audio_status.status_bits || + rdev->audio.pin[0].category_code != audio_status.category_code) { + rdev->audio.pin[0] = audio_status; + changed = true; + } + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + if (!radeon_encoder_is_digital(encoder)) + continue; + if (changed || r600_hdmi_buffer_status_changed(encoder)) + r600_hdmi_update_audio_settings(encoder); + } +} + +/* enable the audio stream */ +void r600_audio_enable(struct radeon_device *rdev, + struct r600_audio_pin *pin, + u8 enable_mask) +{ + u32 tmp = RREG32(AZ_HOT_PLUG_CONTROL); - WREG32(HDMI0_ACR_32_0 + offset, HDMI0_ACR_CTS_32(acr.cts_32khz)); - WREG32(HDMI0_ACR_32_1 + offset, acr.n_32khz); + if (!pin) + return; - WREG32(HDMI0_ACR_44_0 + offset, HDMI0_ACR_CTS_44(acr.cts_44_1khz)); - WREG32(HDMI0_ACR_44_1 + offset, acr.n_44_1khz); + if (enable_mask) { + tmp |= AUDIO_ENABLED; + if (enable_mask & 1) + tmp |= PIN0_AUDIO_ENABLED; + if (enable_mask & 2) + tmp |= PIN1_AUDIO_ENABLED; + if (enable_mask & 4) + tmp |= PIN2_AUDIO_ENABLED; + if (enable_mask & 8) + tmp |= PIN3_AUDIO_ENABLED; + } else { + tmp &= ~(AUDIO_ENABLED | + PIN0_AUDIO_ENABLED | + PIN1_AUDIO_ENABLED | + PIN2_AUDIO_ENABLED | + PIN3_AUDIO_ENABLED); + } - WREG32(HDMI0_ACR_48_0 + offset, HDMI0_ACR_CTS_48(acr.cts_48khz)); - WREG32(HDMI0_ACR_48_1 + offset, acr.n_48khz); + WREG32(AZ_HOT_PLUG_CONTROL, tmp); } -/* - * calculate the crc for a given info frame - */ -static void r600_hdmi_infoframe_checksum(uint8_t packetType, - uint8_t versionNumber, - uint8_t length, - uint8_t *frame) +struct r600_audio_pin *r600_audio_get_pin(struct radeon_device *rdev) { - int i; - frame[0] = packetType + versionNumber + length; - for (i = 1; i <= length; i++) - frame[0] += frame[i]; - frame[0] = 0x100 - frame[0]; + /* only one pin on 6xx-NI */ + return &rdev->audio.pin[0]; } -/* - * build a HDMI Video Info Frame - */ -static void r600_hdmi_videoinfoframe( - struct drm_encoder *encoder, - enum r600_hdmi_color_format color_format, - int active_information_present, - uint8_t active_format_aspect_ratio, - uint8_t scan_information, - uint8_t colorimetry, - uint8_t ex_colorimetry, - uint8_t quantization, - int ITC, - uint8_t picture_aspect_ratio, - uint8_t video_format_identification, - uint8_t pixel_repetition, - uint8_t non_uniform_picture_scaling, - uint8_t bar_info_data_valid, - uint16_t top_bar, - uint16_t bottom_bar, - uint16_t left_bar, - uint16_t right_bar -) +void r600_hdmi_update_acr(struct drm_encoder *encoder, long offset, + const struct radeon_hdmi_acr *acr) { struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - uint32_t offset = dig->afmt->offset; - uint8_t frame[14]; - - frame[0x0] = 0; - frame[0x1] = - (scan_information & 0x3) | - ((bar_info_data_valid & 0x3) << 2) | - ((active_information_present & 0x1) << 4) | - ((color_format & 0x3) << 5); - frame[0x2] = - (active_format_aspect_ratio & 0xF) | - ((picture_aspect_ratio & 0x3) << 4) | - ((colorimetry & 0x3) << 6); - frame[0x3] = - (non_uniform_picture_scaling & 0x3) | - ((quantization & 0x3) << 2) | - ((ex_colorimetry & 0x7) << 4) | - ((ITC & 0x1) << 7); - frame[0x4] = (video_format_identification & 0x7F); - frame[0x5] = (pixel_repetition & 0xF); - frame[0x6] = (top_bar & 0xFF); - frame[0x7] = (top_bar >> 8); - frame[0x8] = (bottom_bar & 0xFF); - frame[0x9] = (bottom_bar >> 8); - frame[0xA] = (left_bar & 0xFF); - frame[0xB] = (left_bar >> 8); - frame[0xC] = (right_bar & 0xFF); - frame[0xD] = (right_bar >> 8); - - r600_hdmi_infoframe_checksum(0x82, 0x02, 0x0D, frame); - /* Our header values (type, version, length) should be alright, Intel - * is using the same. Checksum function also seems to be OK, it works - * fine for audio infoframe. However calculated value is always lower - * by 2 in comparison to fglrx. It breaks displaying anything in case - * of TVs that strictly check the checksum. Hack it manually here to - * workaround this issue. */ - frame[0x0] += 2; + /* DCE 3.0 uses register that's normally for CRC_CONTROL */ + uint32_t acr_ctl = ASIC_IS_DCE3(rdev) ? DCE3_HDMI0_ACR_PACKET_CONTROL : + HDMI0_ACR_PACKET_CONTROL; + WREG32_P(acr_ctl + offset, + HDMI0_ACR_SOURCE | /* select SW CTS value */ + HDMI0_ACR_AUTO_SEND, /* allow hw to sent ACR packets when required */ + ~(HDMI0_ACR_SOURCE | + HDMI0_ACR_AUTO_SEND)); + + WREG32_P(HDMI0_ACR_32_0 + offset, + HDMI0_ACR_CTS_32(acr->cts_32khz), + ~HDMI0_ACR_CTS_32_MASK); + WREG32_P(HDMI0_ACR_32_1 + offset, + HDMI0_ACR_N_32(acr->n_32khz), + ~HDMI0_ACR_N_32_MASK); + + WREG32_P(HDMI0_ACR_44_0 + offset, + HDMI0_ACR_CTS_44(acr->cts_44_1khz), + ~HDMI0_ACR_CTS_44_MASK); + WREG32_P(HDMI0_ACR_44_1 + offset, + HDMI0_ACR_N_44(acr->n_44_1khz), + ~HDMI0_ACR_N_44_MASK); + + WREG32_P(HDMI0_ACR_48_0 + offset, + HDMI0_ACR_CTS_48(acr->cts_48khz), + ~HDMI0_ACR_CTS_48_MASK); + WREG32_P(HDMI0_ACR_48_1 + offset, + HDMI0_ACR_N_48(acr->n_48khz), + ~HDMI0_ACR_N_48_MASK); +} + +/* + * build a HDMI Video Info Frame + */ +void r600_set_avi_packet(struct radeon_device *rdev, u32 offset, + unsigned char *buffer, size_t size) +{ + uint8_t *frame = buffer + 3; WREG32(HDMI0_AVI_INFO0 + offset, frame[0x0] | (frame[0x1] << 8) | (frame[0x2] << 16) | (frame[0x3] << 24)); @@ -210,45 +225,29 @@ static void r600_hdmi_videoinfoframe( WREG32(HDMI0_AVI_INFO2 + offset, frame[0x8] | (frame[0x9] << 8) | (frame[0xA] << 16) | (frame[0xB] << 24)); WREG32(HDMI0_AVI_INFO3 + offset, - frame[0xC] | (frame[0xD] << 8)); + frame[0xC] | (frame[0xD] << 8) | (buffer[1] << 24)); + + WREG32_OR(HDMI0_INFOFRAME_CONTROL1 + offset, + HDMI0_AVI_INFO_LINE(2)); /* anything other than 0 */ + + WREG32_OR(HDMI0_INFOFRAME_CONTROL0 + offset, + HDMI0_AVI_INFO_SEND | /* enable AVI info frames */ + HDMI0_AVI_INFO_CONT); /* send AVI info frames every frame/field */ + } /* * build a Audio Info Frame */ -static void r600_hdmi_audioinfoframe( - struct drm_encoder *encoder, - uint8_t channel_count, - uint8_t coding_type, - uint8_t sample_size, - uint8_t sample_frequency, - uint8_t format, - uint8_t channel_allocation, - uint8_t level_shift, - int downmix_inhibit -) +static void r600_hdmi_update_audio_infoframe(struct drm_encoder *encoder, + const void *buffer, size_t size) { struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; uint32_t offset = dig->afmt->offset; - - uint8_t frame[11]; - - frame[0x0] = 0; - frame[0x1] = (channel_count & 0x7) | ((coding_type & 0xF) << 4); - frame[0x2] = (sample_size & 0x3) | ((sample_frequency & 0x7) << 2); - frame[0x3] = format; - frame[0x4] = channel_allocation; - frame[0x5] = ((level_shift & 0xF) << 3) | ((downmix_inhibit & 0x1) << 7); - frame[0x6] = 0; - frame[0x7] = 0; - frame[0x8] = 0; - frame[0x9] = 0; - frame[0xA] = 0; - - r600_hdmi_infoframe_checksum(0x84, 0x01, 0x0A, frame); + const u8 *frame = buffer + 3; WREG32(HDMI0_AUDIO_INFO0 + offset, frame[0x0] | (frame[0x1] << 8) | (frame[0x2] << 16) | (frame[0x3] << 24)); @@ -292,7 +291,7 @@ int r600_hdmi_buffer_status_changed(struct drm_encoder *encoder) /* * write the audio workaround status to the hardware */ -static void r600_hdmi_audio_workaround(struct drm_encoder *encoder) +void r600_hdmi_audio_workaround(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; @@ -311,86 +310,102 @@ static void r600_hdmi_audio_workaround(struct drm_encoder *encoder) value, ~HDMI0_AUDIO_TEST_EN); } - -/* - * update the info frames with the data from the current display mode - */ -void r600_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode) +void r600_hdmi_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock) { - struct drm_device *dev = encoder->dev; - struct radeon_device *rdev = dev->dev_private; - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - uint32_t offset; + struct radeon_encoder *radeon_encoder; + struct radeon_encoder_atom_dig *dig; - if (!dig || !dig->afmt) - return; - - /* Silent, r600_hdmi_enable will raise WARN for us */ - if (!dig->afmt->enabled) + if (!crtc) return; - offset = dig->afmt->offset; - r600_audio_set_clock(encoder, mode->clock); + radeon_encoder = to_radeon_encoder(crtc->encoder); + dig = radeon_encoder->enc_priv; - WREG32(HDMI0_VBI_PACKET_CONTROL + offset, - HDMI0_NULL_SEND); /* send null packets when required */ - - WREG32(HDMI0_AUDIO_CRC_CONTROL + offset, 0x1000); + if (!dig) + return; - if (ASIC_IS_DCE32(rdev)) { - WREG32(HDMI0_AUDIO_PACKET_CONTROL + offset, - HDMI0_AUDIO_DELAY_EN(1) | /* default audio delay */ - HDMI0_AUDIO_PACKETS_PER_LINE(3)); /* should be suffient for all audio modes and small enough for all hblanks */ - WREG32(AFMT_AUDIO_PACKET_CONTROL + offset, - AFMT_AUDIO_SAMPLE_SEND | /* send audio packets */ - AFMT_60958_CS_UPDATE); /* allow 60958 channel status fields to be updated */ + if (dig->dig_encoder == 0) { + WREG32(DCCG_AUDIO_DTO0_PHASE, 24000 * 100); + WREG32(DCCG_AUDIO_DTO0_MODULE, clock * 100); + WREG32(DCCG_AUDIO_DTO_SELECT, 0); /* select DTO0 */ } else { - WREG32(HDMI0_AUDIO_PACKET_CONTROL + offset, - HDMI0_AUDIO_SAMPLE_SEND | /* send audio packets */ - HDMI0_AUDIO_DELAY_EN(1) | /* default audio delay */ - HDMI0_AUDIO_PACKETS_PER_LINE(3) | /* should be suffient for all audio modes and small enough for all hblanks */ - HDMI0_60958_CS_UPDATE); /* allow 60958 channel status fields to be updated */ + WREG32(DCCG_AUDIO_DTO1_PHASE, 24000 * 100); + WREG32(DCCG_AUDIO_DTO1_MODULE, clock * 100); + WREG32(DCCG_AUDIO_DTO_SELECT, 1); /* select DTO1 */ } +} - WREG32(HDMI0_ACR_PACKET_CONTROL + offset, - HDMI0_ACR_AUTO_SEND | /* allow hw to sent ACR packets when required */ - HDMI0_ACR_SOURCE); /* select SW CTS value */ - - WREG32(HDMI0_VBI_PACKET_CONTROL + offset, - HDMI0_NULL_SEND | /* send null packets when required */ - HDMI0_GC_SEND | /* send general control packets */ - HDMI0_GC_CONT); /* send general control packets every frame */ - - /* TODO: HDMI0_AUDIO_INFO_UPDATE */ - WREG32(HDMI0_INFOFRAME_CONTROL0 + offset, - HDMI0_AVI_INFO_SEND | /* enable AVI info frames */ - HDMI0_AVI_INFO_CONT | /* send AVI info frames every frame/field */ - HDMI0_AUDIO_INFO_SEND | /* enable audio info frames (frames won't be set until audio is enabled) */ - HDMI0_AUDIO_INFO_CONT); /* send audio info frames every frame/field */ - - WREG32(HDMI0_INFOFRAME_CONTROL1 + offset, - HDMI0_AVI_INFO_LINE(2) | /* anything other than 0 */ - HDMI0_AUDIO_INFO_LINE(2)); /* anything other than 0 */ +void r600_set_vbi_packet(struct drm_encoder *encoder, u32 offset) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; - WREG32(HDMI0_GC + offset, 0); /* unset HDMI0_GC_AVMUTE */ + WREG32_OR(HDMI0_VBI_PACKET_CONTROL + offset, + HDMI0_NULL_SEND | /* send null packets when required */ + HDMI0_GC_SEND | /* send general control packets */ + HDMI0_GC_CONT); /* send general control packets every frame */ +} - r600_hdmi_videoinfoframe(encoder, RGB, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); +void r600_set_audio_packet(struct drm_encoder *encoder, u32 offset) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; - r600_hdmi_update_ACR(encoder, mode->clock); + WREG32_P(HDMI0_AUDIO_PACKET_CONTROL + offset, + HDMI0_AUDIO_SAMPLE_SEND | /* send audio packets */ + HDMI0_AUDIO_DELAY_EN(1) | /* default audio delay */ + HDMI0_AUDIO_PACKETS_PER_LINE(3) | /* should be suffient for all audio modes and small enough for all hblanks */ + HDMI0_60958_CS_UPDATE, /* allow 60958 channel status fields to be updated */ + ~(HDMI0_AUDIO_SAMPLE_SEND | + HDMI0_AUDIO_DELAY_EN_MASK | + HDMI0_AUDIO_PACKETS_PER_LINE_MASK | + HDMI0_60958_CS_UPDATE)); + + WREG32_OR(HDMI0_INFOFRAME_CONTROL0 + offset, + HDMI0_AUDIO_INFO_SEND | /* enable audio info frames (frames won't be set until audio is enabled) */ + HDMI0_AUDIO_INFO_UPDATE); /* required for audio info values to be updated */ + + WREG32_P(HDMI0_INFOFRAME_CONTROL1 + offset, + HDMI0_AUDIO_INFO_LINE(2), /* anything other than 0 */ + ~HDMI0_AUDIO_INFO_LINE_MASK); + + WREG32_AND(HDMI0_GENERIC_PACKET_CONTROL + offset, + ~(HDMI0_GENERIC0_SEND | + HDMI0_GENERIC0_CONT | + HDMI0_GENERIC0_UPDATE | + HDMI0_GENERIC1_SEND | + HDMI0_GENERIC1_CONT | + HDMI0_GENERIC0_LINE_MASK | + HDMI0_GENERIC1_LINE_MASK)); + + WREG32_P(HDMI0_60958_0 + offset, + HDMI0_60958_CS_CHANNEL_NUMBER_L(1), + ~(HDMI0_60958_CS_CHANNEL_NUMBER_L_MASK | + HDMI0_60958_CS_CLOCK_ACCURACY_MASK)); + + WREG32_P(HDMI0_60958_1 + offset, + HDMI0_60958_CS_CHANNEL_NUMBER_R(2), + ~HDMI0_60958_CS_CHANNEL_NUMBER_R_MASK); +} - /* it's unknown what these bits do excatly, but it's indeed quite useful for debugging */ - WREG32(HDMI0_RAMP_CONTROL0 + offset, 0x00FFFFFF); - WREG32(HDMI0_RAMP_CONTROL1 + offset, 0x007FFFFF); - WREG32(HDMI0_RAMP_CONTROL2 + offset, 0x00000001); - WREG32(HDMI0_RAMP_CONTROL3 + offset, 0x00000001); +void r600_set_mute(struct drm_encoder *encoder, u32 offset, bool mute) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; - r600_hdmi_audio_workaround(encoder); + if (mute) + WREG32_OR(HDMI0_GC + offset, HDMI0_GC_AVMUTE); + else + WREG32_AND(HDMI0_GC + offset, ~HDMI0_GC_AVMUTE); } -/* - * update settings with current parameters from audio engine +/** + * r600_hdmi_update_audio_settings - Update audio infoframe + * + * @encoder: drm encoder + * + * Gets info about current audio stream and updates audio infoframe. */ void r600_hdmi_update_audio_settings(struct drm_encoder *encoder) { @@ -398,9 +413,12 @@ void r600_hdmi_update_audio_settings(struct drm_encoder *encoder) struct radeon_device *rdev = dev->dev_private; struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - struct r600_audio audio = r600_audio_status(rdev); + struct r600_audio_pin audio = r600_audio_status(rdev); + uint8_t buffer[HDMI_INFOFRAME_HEADER_SIZE + HDMI_AUDIO_INFOFRAME_SIZE]; + struct hdmi_audio_infoframe frame; uint32_t offset; - uint32_t iec; + uint32_t value; + ssize_t err; if (!dig->afmt || !dig->afmt->enabled) return; @@ -412,180 +430,105 @@ void r600_hdmi_update_audio_settings(struct drm_encoder *encoder) DRM_DEBUG("0x%02X IEC60958 status bits and 0x%02X category code\n", (int)audio.status_bits, (int)audio.category_code); - iec = 0; - if (audio.status_bits & AUDIO_STATUS_PROFESSIONAL) - iec |= 1 << 0; - if (audio.status_bits & AUDIO_STATUS_NONAUDIO) - iec |= 1 << 1; - if (audio.status_bits & AUDIO_STATUS_COPYRIGHT) - iec |= 1 << 2; - if (audio.status_bits & AUDIO_STATUS_EMPHASIS) - iec |= 1 << 3; - - iec |= HDMI0_60958_CS_CATEGORY_CODE(audio.category_code); - - switch (audio.rate) { - case 32000: - iec |= HDMI0_60958_CS_SAMPLING_FREQUENCY(0x3); - break; - case 44100: - iec |= HDMI0_60958_CS_SAMPLING_FREQUENCY(0x0); - break; - case 48000: - iec |= HDMI0_60958_CS_SAMPLING_FREQUENCY(0x2); - break; - case 88200: - iec |= HDMI0_60958_CS_SAMPLING_FREQUENCY(0x8); - break; - case 96000: - iec |= HDMI0_60958_CS_SAMPLING_FREQUENCY(0xa); - break; - case 176400: - iec |= HDMI0_60958_CS_SAMPLING_FREQUENCY(0xc); - break; - case 192000: - iec |= HDMI0_60958_CS_SAMPLING_FREQUENCY(0xe); - break; + err = hdmi_audio_infoframe_init(&frame); + if (err < 0) { + DRM_ERROR("failed to setup audio infoframe\n"); + return; } - WREG32(HDMI0_60958_0 + offset, iec); + frame.channels = audio.channels; - iec = 0; - switch (audio.bits_per_sample) { - case 16: - iec |= HDMI0_60958_CS_WORD_LENGTH(0x2); - break; - case 20: - iec |= HDMI0_60958_CS_WORD_LENGTH(0x3); - break; - case 24: - iec |= HDMI0_60958_CS_WORD_LENGTH(0xb); - break; + err = hdmi_audio_infoframe_pack(&frame, buffer, sizeof(buffer)); + if (err < 0) { + DRM_ERROR("failed to pack audio infoframe\n"); + return; } - if (audio.status_bits & AUDIO_STATUS_V) - iec |= 0x5 << 16; - WREG32_P(HDMI0_60958_1 + offset, iec, ~0x5000f); - r600_hdmi_audioinfoframe(encoder, audio.channels - 1, 0, 0, 0, 0, 0, 0, - 0); + value = RREG32(HDMI0_AUDIO_PACKET_CONTROL + offset); + if (value & HDMI0_AUDIO_TEST_EN) + WREG32(HDMI0_AUDIO_PACKET_CONTROL + offset, + value & ~HDMI0_AUDIO_TEST_EN); + + WREG32_OR(HDMI0_CONTROL + offset, + HDMI0_ERROR_ACK); + + WREG32_AND(HDMI0_INFOFRAME_CONTROL0 + offset, + ~HDMI0_AUDIO_INFO_SOURCE); - r600_hdmi_audio_workaround(encoder); + r600_hdmi_update_audio_infoframe(encoder, buffer, sizeof(buffer)); + + WREG32_OR(HDMI0_INFOFRAME_CONTROL0 + offset, + HDMI0_AUDIO_INFO_CONT | + HDMI0_AUDIO_INFO_UPDATE); } /* * enable the HDMI engine */ -void r600_hdmi_enable(struct drm_encoder *encoder) +void r600_hdmi_enable(struct drm_encoder *encoder, bool enable) { struct drm_device *dev = encoder->dev; struct radeon_device *rdev = dev->dev_private; struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - uint32_t offset; - u32 hdmi; + u32 hdmi = HDMI0_ERROR_ACK; if (!dig || !dig->afmt) return; - if (ASIC_IS_DCE6(rdev)) - return; - - /* Silent, r600_hdmi_enable will raise WARN for us */ - if (dig->afmt->enabled) - return; - offset = dig->afmt->offset; - /* Older chipsets require setting HDMI and routing manually */ - if (ASIC_IS_DCE2(rdev) && !ASIC_IS_DCE3(rdev)) { - hdmi = HDMI0_ERROR_ACK | HDMI0_ENABLE; + if (!ASIC_IS_DCE3(rdev)) { + if (enable) + hdmi |= HDMI0_ENABLE; switch (radeon_encoder->encoder_id) { case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1: - WREG32_P(AVIVO_TMDSA_CNTL, AVIVO_TMDSA_CNTL_HDMI_EN, - ~AVIVO_TMDSA_CNTL_HDMI_EN); - hdmi |= HDMI0_STREAM(HDMI0_STREAM_TMDSA); + if (enable) { + WREG32_OR(AVIVO_TMDSA_CNTL, AVIVO_TMDSA_CNTL_HDMI_EN); + hdmi |= HDMI0_STREAM(HDMI0_STREAM_TMDSA); + } else { + WREG32_AND(AVIVO_TMDSA_CNTL, ~AVIVO_TMDSA_CNTL_HDMI_EN); + } break; case ENCODER_OBJECT_ID_INTERNAL_LVTM1: - WREG32_P(AVIVO_LVTMA_CNTL, AVIVO_LVTMA_CNTL_HDMI_EN, - ~AVIVO_LVTMA_CNTL_HDMI_EN); - hdmi |= HDMI0_STREAM(HDMI0_STREAM_LVTMA); + if (enable) { + WREG32_OR(AVIVO_LVTMA_CNTL, AVIVO_LVTMA_CNTL_HDMI_EN); + hdmi |= HDMI0_STREAM(HDMI0_STREAM_LVTMA); + } else { + WREG32_AND(AVIVO_LVTMA_CNTL, ~AVIVO_LVTMA_CNTL_HDMI_EN); + } break; case ENCODER_OBJECT_ID_INTERNAL_DDI: - WREG32_P(DDIA_CNTL, DDIA_HDMI_EN, ~DDIA_HDMI_EN); - hdmi |= HDMI0_STREAM(HDMI0_STREAM_DDIA); + if (enable) { + WREG32_OR(DDIA_CNTL, DDIA_HDMI_EN); + hdmi |= HDMI0_STREAM(HDMI0_STREAM_DDIA); + } else { + WREG32_AND(DDIA_CNTL, ~DDIA_HDMI_EN); + } break; case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: - hdmi |= HDMI0_STREAM(HDMI0_STREAM_DVOA); + if (enable) + hdmi |= HDMI0_STREAM(HDMI0_STREAM_DVOA); break; default: dev_err(rdev->dev, "Invalid encoder for HDMI: 0x%X\n", radeon_encoder->encoder_id); break; } - WREG32(HDMI0_CONTROL + offset, hdmi); + WREG32(HDMI0_CONTROL + dig->afmt->offset, hdmi); } if (rdev->irq.installed) { /* if irq is available use it */ - radeon_irq_kms_enable_afmt(rdev, dig->afmt->id); + /* XXX: shouldn't need this on any asics. Double check DCE2/3 */ + if (enable) + radeon_irq_kms_enable_afmt(rdev, dig->afmt->id); + else + radeon_irq_kms_disable_afmt(rdev, dig->afmt->id); } - dig->afmt->enabled = true; + dig->afmt->enabled = enable; - DRM_DEBUG("Enabling HDMI interface @ 0x%04X for encoder 0x%x\n", - offset, radeon_encoder->encoder_id); + DRM_DEBUG("%sabling HDMI interface @ 0x%04X for encoder 0x%x\n", + enable ? "En" : "Dis", dig->afmt->offset, radeon_encoder->encoder_id); } -/* - * disable the HDMI engine - */ -void r600_hdmi_disable(struct drm_encoder *encoder) -{ - struct drm_device *dev = encoder->dev; - struct radeon_device *rdev = dev->dev_private; - struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); - struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; - uint32_t offset; - - if (ASIC_IS_DCE6(rdev)) - return; - - /* Called for ATOM_ENCODER_MODE_HDMI only */ - if (!dig || !dig->afmt) { - return; - } - if (!dig->afmt->enabled) - return; - offset = dig->afmt->offset; - - DRM_DEBUG("Disabling HDMI interface @ 0x%04X for encoder 0x%x\n", - offset, radeon_encoder->encoder_id); - - /* disable irq */ - radeon_irq_kms_disable_afmt(rdev, dig->afmt->id); - - /* Older chipsets not handled by AtomBIOS */ - if (ASIC_IS_DCE2(rdev) && !ASIC_IS_DCE3(rdev)) { - switch (radeon_encoder->encoder_id) { - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1: - WREG32_P(AVIVO_TMDSA_CNTL, 0, - ~AVIVO_TMDSA_CNTL_HDMI_EN); - break; - case ENCODER_OBJECT_ID_INTERNAL_LVTM1: - WREG32_P(AVIVO_LVTMA_CNTL, 0, - ~AVIVO_LVTMA_CNTL_HDMI_EN); - break; - case ENCODER_OBJECT_ID_INTERNAL_DDI: - WREG32_P(DDIA_CNTL, 0, ~DDIA_HDMI_EN); - break; - case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: - break; - default: - dev_err(rdev->dev, "Invalid encoder for HDMI: 0x%X\n", - radeon_encoder->encoder_id); - break; - } - WREG32(HDMI0_CONTROL + offset, HDMI0_ERROR_ACK); - } - - dig->afmt->enabled = false; -} diff --git a/sys/dev/pci/drm/radeon/r600_reg.h b/sys/dev/pci/drm/radeon/r600_reg.h index afce6e25e43..3ef202629e7 100644 --- a/sys/dev/pci/drm/radeon/r600_reg.h +++ b/sys/dev/pci/drm/radeon/r600_reg.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r600_reg.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -32,6 +31,12 @@ #define R600_PCIE_PORT_INDEX 0x0038 #define R600_PCIE_PORT_DATA 0x003c +#define R600_RCU_INDEX 0x0100 +#define R600_RCU_DATA 0x0104 + +#define R600_UVD_CTX_INDEX 0xf4a0 +#define R600_UVD_CTX_DATA 0xf4a4 + #define R600_MC_VM_FB_LOCATION 0x2180 #define R600_MC_FB_BASE_MASK 0x0000FFFF #define R600_MC_FB_BASE_SHIFT 0 diff --git a/sys/dev/pci/drm/radeon/r600d.h b/sys/dev/pci/drm/radeon/r600d.h index c35244263cf..1e8495cca41 100644 --- a/sys/dev/pci/drm/radeon/r600d.h +++ b/sys/dev/pci/drm/radeon/r600d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: r600d.h,v 1.5 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2009 Advanced Micro Devices, Inc. * Copyright 2009 Red Hat Inc. @@ -45,13 +44,6 @@ #define R6XX_MAX_PIPES 8 #define R6XX_MAX_PIPES_MASK 0xff -/* PTE flags */ -#define PTE_VALID (1 << 0) -#define PTE_SYSTEM (1 << 1) -#define PTE_SNOOPED (1 << 2) -#define PTE_READABLE (1 << 5) -#define PTE_WRITEABLE (1 << 6) - /* tiling bits */ #define ARRAY_LINEAR_GENERAL 0x00000000 #define ARRAY_LINEAR_ALIGNED 0x00000001 @@ -183,6 +175,8 @@ #define CP_COHER_BASE 0x85F8 #define CP_DEBUG 0xC1FC #define R_0086D8_CP_ME_CNTL 0x86D8 +#define S_0086D8_CP_PFP_HALT(x) (((x) & 1)<<26) +#define C_0086D8_CP_PFP_HALT(x) ((x) & 0xFBFFFFFF) #define S_0086D8_CP_ME_HALT(x) (((x) & 1)<<28) #define C_0086D8_CP_ME_HALT(x) ((x) & 0xEFFFFFFF) #define CP_ME_RAM_DATA 0xC160 @@ -301,10 +295,25 @@ #define GRBM_SOFT_RESET 0x8020 #define SOFT_RESET_CP (1<<0) +#define CG_THERMAL_CTRL 0x7F0 +#define DIG_THERM_DPM(x) ((x) << 12) +#define DIG_THERM_DPM_MASK 0x000FF000 +#define DIG_THERM_DPM_SHIFT 12 #define CG_THERMAL_STATUS 0x7F4 #define ASIC_T(x) ((x) << 0) #define ASIC_T_MASK 0x1FF #define ASIC_T_SHIFT 0 +#define CG_THERMAL_INT 0x7F8 +#define DIG_THERM_INTH(x) ((x) << 8) +#define DIG_THERM_INTH_MASK 0x0000FF00 +#define DIG_THERM_INTH_SHIFT 8 +#define DIG_THERM_INTL(x) ((x) << 16) +#define DIG_THERM_INTL_MASK 0x00FF0000 +#define DIG_THERM_INTL_SHIFT 16 +#define THERM_INT_MASK_HIGH (1 << 24) +#define THERM_INT_MASK_LOW (1 << 25) + +#define RV770_CG_THERMAL_INT 0x734 #define HDP_HOST_PATH_CNTL 0x2C00 #define HDP_NONSURFACE_BASE 0x2C04 @@ -314,11 +323,12 @@ #define HDP_TILING_CONFIG 0x2F3C #define HDP_DEBUG1 0x2F34 +#define MC_CONFIG 0x2000 #define MC_VM_AGP_TOP 0x2184 #define MC_VM_AGP_BOT 0x2188 #define MC_VM_AGP_BASE 0x218C #define MC_VM_FB_LOCATION 0x2180 -#define MC_VM_L1_TLB_MCD_RD_A_CNTL 0x219C +#define MC_VM_L1_TLB_MCB_RD_UVD_CNTL 0x2124 #define ENABLE_L1_TLB (1 << 0) #define ENABLE_L1_FRAGMENT_PROCESSING (1 << 1) #define ENABLE_L1_STRICT_ORDERING (1 << 2) @@ -338,12 +348,14 @@ #define EFFECTIVE_L1_QUEUE_SIZE(x) (((x) & 7) << 15) #define EFFECTIVE_L1_QUEUE_SIZE_MASK 0x00038000 #define EFFECTIVE_L1_QUEUE_SIZE_SHIFT 15 +#define MC_VM_L1_TLB_MCD_RD_A_CNTL 0x219C #define MC_VM_L1_TLB_MCD_RD_B_CNTL 0x21A0 #define MC_VM_L1_TLB_MCB_RD_GFX_CNTL 0x21FC #define MC_VM_L1_TLB_MCB_RD_HDP_CNTL 0x2204 #define MC_VM_L1_TLB_MCB_RD_PDMA_CNTL 0x2208 #define MC_VM_L1_TLB_MCB_RD_SEM_CNTL 0x220C #define MC_VM_L1_TLB_MCB_RD_SYS_CNTL 0x2200 +#define MC_VM_L1_TLB_MCB_WR_UVD_CNTL 0x212c #define MC_VM_L1_TLB_MCD_WR_A_CNTL 0x21A4 #define MC_VM_L1_TLB_MCD_WR_B_CNTL 0x21A8 #define MC_VM_L1_TLB_MCB_WR_GFX_CNTL 0x2210 @@ -357,6 +369,8 @@ #define MC_VM_SYSTEM_APERTURE_HIGH_ADDR 0x2194 #define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR 0x2198 +#define RS_DQ_RD_RET_CONF 0x2348 + #define PA_CL_ENHANCE 0x8A14 #define CLIP_VTX_REORDER_ENA (1 << 0) #define NUM_CLIP_SEQ(x) ((x) << 1) @@ -586,6 +600,7 @@ #define L2_BUSY (1 << 0) #define WAIT_UNTIL 0x8040 +#define WAIT_CP_DMA_IDLE_bit (1 << 8) #define WAIT_2D_IDLE_bit (1 << 14) #define WAIT_3D_IDLE_bit (1 << 15) #define WAIT_2D_IDLECLEAN_bit (1 << 16) @@ -683,15 +698,19 @@ #define RLC_UCODE_ADDR 0x3f2c #define RLC_UCODE_DATA 0x3f30 -/* new for TN */ -#define TN_RLC_SAVE_AND_RESTORE_BASE 0x3f10 -#define TN_RLC_CLEAR_STATE_RESTORE_BASE 0x3f20 - #define SRBM_SOFT_RESET 0xe60 +# define SOFT_RESET_BIF (1 << 1) # define SOFT_RESET_DMA (1 << 12) # define SOFT_RESET_RLC (1 << 13) +# define SOFT_RESET_UVD (1 << 18) # define RV770_SOFT_RESET_DMA (1 << 20) +#define BIF_SCRATCH0 0x5438 + +#define BUS_CNTL 0x5420 +# define BIOS_ROM_DIS (1 << 1) +# define VGA_COHE_SPEC_TIMER_DIS (1 << 9) + #define CP_INT_CNTL 0xc124 # define CNTX_BUSY_INT_ENABLE (1 << 19) # define CNTX_EMPTY_INT_ENABLE (1 << 20) @@ -908,12 +927,37 @@ # define TARGET_LINK_SPEED_MASK (0xf << 0) # define SELECTABLE_DEEMPHASIS (1 << 6) -/* Audio clocks */ +/* Audio */ +#define AZ_HOT_PLUG_CONTROL 0x7300 +# define AZ_FORCE_CODEC_WAKE (1 << 0) +# define JACK_DETECTION_ENABLE (1 << 4) +# define UNSOLICITED_RESPONSE_ENABLE (1 << 8) +# define CODEC_HOT_PLUG_ENABLE (1 << 12) +# define AUDIO_ENABLED (1 << 31) +/* DCE3 adds */ +# define PIN0_JACK_DETECTION_ENABLE (1 << 4) +# define PIN1_JACK_DETECTION_ENABLE (1 << 5) +# define PIN2_JACK_DETECTION_ENABLE (1 << 6) +# define PIN3_JACK_DETECTION_ENABLE (1 << 7) +# define PIN0_AUDIO_ENABLED (1 << 24) +# define PIN1_AUDIO_ENABLED (1 << 25) +# define PIN2_AUDIO_ENABLED (1 << 26) +# define PIN3_AUDIO_ENABLED (1 << 27) + +/* Audio clocks DCE 2.0/3.0 */ +#define AUDIO_DTO 0x7340 +# define AUDIO_DTO_PHASE(x) (((x) & 0xffff) << 0) +# define AUDIO_DTO_MODULE(x) (((x) & 0xffff) << 16) + +/* Audio clocks DCE 3.2 */ #define DCCG_AUDIO_DTO0_PHASE 0x0514 #define DCCG_AUDIO_DTO0_MODULE 0x0518 #define DCCG_AUDIO_DTO0_LOAD 0x051c # define DTO_LOAD (1 << 31) #define DCCG_AUDIO_DTO0_CNTL 0x0520 +# define DCCG_AUDIO_DTO_WALLCLOCK_RATIO(x) (((x) & 7) << 0) +# define DCCG_AUDIO_DTO_WALLCLOCK_RATIO_MASK 7 +# define DCCG_AUDIO_DTO_WALLCLOCK_RATIO_SHIFT 0 #define DCCG_AUDIO_DTO1_PHASE 0x0524 #define DCCG_AUDIO_DTO1_MODULE 0x0528 @@ -938,6 +982,42 @@ # define DIG_MODE_SDVO 4 #define DIG1_CNTL 0x79a0 +#define AZ_F0_CODEC_PIN0_CONTROL_CHANNEL_SPEAKER 0x71bc +#define SPEAKER_ALLOCATION(x) (((x) & 0x7f) << 0) +#define SPEAKER_ALLOCATION_MASK (0x7f << 0) +#define SPEAKER_ALLOCATION_SHIFT 0 +#define HDMI_CONNECTION (1 << 16) +#define DP_CONNECTION (1 << 17) + +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR0 0x71c8 /* LPCM */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR1 0x71cc /* AC3 */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR2 0x71d0 /* MPEG1 */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR3 0x71d4 /* MP3 */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR4 0x71d8 /* MPEG2 */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR5 0x71dc /* AAC */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR6 0x71e0 /* DTS */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR7 0x71e4 /* ATRAC */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR8 0x71e8 /* one bit audio - leave at 0 (default) */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR9 0x71ec /* Dolby Digital */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR10 0x71f0 /* DTS-HD */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR11 0x71f4 /* MAT-MLP */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR12 0x71f8 /* DTS */ +#define AZ_F0_CODEC_PIN0_CONTROL_AUDIO_DESCRIPTOR13 0x71fc /* WMA Pro */ +# define MAX_CHANNELS(x) (((x) & 0x7) << 0) +/* max channels minus one. 7 = 8 channels */ +# define SUPPORTED_FREQUENCIES(x) (((x) & 0xff) << 8) +# define DESCRIPTOR_BYTE_2(x) (((x) & 0xff) << 16) +# define SUPPORTED_FREQUENCIES_STEREO(x) (((x) & 0xff) << 24) /* LPCM only */ +/* SUPPORTED_FREQUENCIES, SUPPORTED_FREQUENCIES_STEREO + * bit0 = 32 kHz + * bit1 = 44.1 kHz + * bit2 = 48 kHz + * bit3 = 88.2 kHz + * bit4 = 96 kHz + * bit5 = 176.4 kHz + * bit6 = 192 kHz + */ + /* rs6xx/rs740 and r6xx share the same HDMI blocks, however, rs6xx has only one * instance of the blocks while r6xx has 2. DCE 3.0 cards are slightly * different due to the new DIG blocks, but also have 2 instances. @@ -964,15 +1044,18 @@ #define HDMI0_AUDIO_PACKET_CONTROL 0x7408 # define HDMI0_AUDIO_SAMPLE_SEND (1 << 0) # define HDMI0_AUDIO_DELAY_EN(x) (((x) & 3) << 4) +# define HDMI0_AUDIO_DELAY_EN_MASK (3 << 4) # define HDMI0_AUDIO_SEND_MAX_PACKETS (1 << 8) # define HDMI0_AUDIO_TEST_EN (1 << 12) # define HDMI0_AUDIO_PACKETS_PER_LINE(x) (((x) & 0x1f) << 16) +# define HDMI0_AUDIO_PACKETS_PER_LINE_MASK (0x1f << 16) # define HDMI0_AUDIO_CHANNEL_SWAP (1 << 24) # define HDMI0_60958_CS_UPDATE (1 << 26) # define HDMI0_AZ_FORMAT_WTRIG_MASK (1 << 28) # define HDMI0_AZ_FORMAT_WTRIG_ACK (1 << 29) #define HDMI0_AUDIO_CRC_CONTROL 0x740c # define HDMI0_AUDIO_CRC_EN (1 << 0) +#define DCE3_HDMI0_ACR_PACKET_CONTROL 0x740c #define HDMI0_VBI_PACKET_CONTROL 0x7410 # define HDMI0_NULL_SEND (1 << 0) # define HDMI0_GC_SEND (1 << 4) @@ -982,14 +1065,16 @@ # define HDMI0_AVI_INFO_CONT (1 << 1) # define HDMI0_AUDIO_INFO_SEND (1 << 4) # define HDMI0_AUDIO_INFO_CONT (1 << 5) -# define HDMI0_AUDIO_INFO_SOURCE (1 << 6) /* 0 - sound block; 1 - hmdi regs */ +# define HDMI0_AUDIO_INFO_SOURCE (1 << 6) /* 0 - sound block; 1 - hdmi regs */ # define HDMI0_AUDIO_INFO_UPDATE (1 << 7) # define HDMI0_MPEG_INFO_SEND (1 << 8) # define HDMI0_MPEG_INFO_CONT (1 << 9) # define HDMI0_MPEG_INFO_UPDATE (1 << 10) #define HDMI0_INFOFRAME_CONTROL1 0x7418 # define HDMI0_AVI_INFO_LINE(x) (((x) & 0x3f) << 0) +# define HDMI0_AVI_INFO_LINE_MASK (0x3f << 0) # define HDMI0_AUDIO_INFO_LINE(x) (((x) & 0x3f) << 8) +# define HDMI0_AUDIO_INFO_LINE_MASK (0x3f << 8) # define HDMI0_MPEG_INFO_LINE(x) (((x) & 0x3f) << 16) #define HDMI0_GENERIC_PACKET_CONTROL 0x741c # define HDMI0_GENERIC0_SEND (1 << 0) @@ -998,7 +1083,9 @@ # define HDMI0_GENERIC1_SEND (1 << 4) # define HDMI0_GENERIC1_CONT (1 << 5) # define HDMI0_GENERIC0_LINE(x) (((x) & 0x3f) << 16) +# define HDMI0_GENERIC0_LINE_MASK (0x3f << 16) # define HDMI0_GENERIC1_LINE(x) (((x) & 0x3f) << 24) +# define HDMI0_GENERIC1_LINE_MASK (0x3f << 24) #define HDMI0_GC 0x7428 # define HDMI0_GC_AVMUTE (1 << 0) #define HDMI0_AVI_INFO0 0x7454 @@ -1054,16 +1141,22 @@ #define HDMI0_GENERIC1_6 0x74a8 #define HDMI0_ACR_32_0 0x74ac # define HDMI0_ACR_CTS_32(x) (((x) & 0xfffff) << 12) +# define HDMI0_ACR_CTS_32_MASK (0xfffff << 12) #define HDMI0_ACR_32_1 0x74b0 # define HDMI0_ACR_N_32(x) (((x) & 0xfffff) << 0) +# define HDMI0_ACR_N_32_MASK (0xfffff << 0) #define HDMI0_ACR_44_0 0x74b4 # define HDMI0_ACR_CTS_44(x) (((x) & 0xfffff) << 12) +# define HDMI0_ACR_CTS_44_MASK (0xfffff << 12) #define HDMI0_ACR_44_1 0x74b8 # define HDMI0_ACR_N_44(x) (((x) & 0xfffff) << 0) +# define HDMI0_ACR_N_44_MASK (0xfffff << 0) #define HDMI0_ACR_48_0 0x74bc # define HDMI0_ACR_CTS_48(x) (((x) & 0xfffff) << 12) +# define HDMI0_ACR_CTS_48_MASK (0xfffff << 12) #define HDMI0_ACR_48_1 0x74c0 # define HDMI0_ACR_N_48(x) (((x) & 0xfffff) << 0) +# define HDMI0_ACR_N_48_MASK (0xfffff << 0) #define HDMI0_ACR_STATUS_0 0x74c4 #define HDMI0_ACR_STATUS_1 0x74c8 #define HDMI0_AUDIO_INFO0 0x74cc @@ -1083,14 +1176,17 @@ # define HDMI0_60958_CS_CATEGORY_CODE(x) (((x) & 0xff) << 8) # define HDMI0_60958_CS_SOURCE_NUMBER(x) (((x) & 0xf) << 16) # define HDMI0_60958_CS_CHANNEL_NUMBER_L(x) (((x) & 0xf) << 20) +# define HDMI0_60958_CS_CHANNEL_NUMBER_L_MASK (0xf << 20) # define HDMI0_60958_CS_SAMPLING_FREQUENCY(x) (((x) & 0xf) << 24) # define HDMI0_60958_CS_CLOCK_ACCURACY(x) (((x) & 3) << 28) +# define HDMI0_60958_CS_CLOCK_ACCURACY_MASK (3 << 28) #define HDMI0_60958_1 0x74d8 # define HDMI0_60958_CS_WORD_LENGTH(x) (((x) & 0xf) << 0) # define HDMI0_60958_CS_ORIGINAL_SAMPLING_FREQUENCY(x) (((x) & 0xf) << 4) # define HDMI0_60958_CS_VALID_L(x) (((x) & 1) << 16) # define HDMI0_60958_CS_VALID_R(x) (((x) & 1) << 18) # define HDMI0_60958_CS_CHANNEL_NUMBER_R(x) (((x) & 0xf) << 20) +# define HDMI0_60958_CS_CHANNEL_NUMBER_R_MASK (0xf << 20) #define HDMI0_ACR_PACKET_CONTROL 0x74dc # define HDMI0_ACR_SEND (1 << 0) # define HDMI0_ACR_CONT (1 << 1) @@ -1101,6 +1197,7 @@ # define HDMI0_ACR_48 3 # define HDMI0_ACR_SOURCE (1 << 8) /* 0 - hw; 1 - cts value */ # define HDMI0_ACR_AUTO_SEND (1 << 12) +#define DCE3_HDMI0_AUDIO_CRC_CONTROL 0x74dc #define HDMI0_RAMP_CONTROL0 0x74e0 # define HDMI0_RAMP_MAX_COUNT(x) (((x) & 0xffffff) << 0) #define HDMI0_RAMP_CONTROL1 0x74e4 @@ -1141,22 +1238,352 @@ # define AFMT_AZ_FORMAT_WTRIG_ACK (1 << 29) # define AFMT_AZ_AUDIO_ENABLE_CHG_ACK (1 << 30) +/* DCE3 FMT blocks */ +#define FMT_CONTROL 0x6700 +# define FMT_PIXEL_ENCODING (1 << 16) + /* 0 = RGB 4:4:4 or YCbCr 4:4:4, 1 = YCbCr 4:2:2 */ +#define FMT_BIT_DEPTH_CONTROL 0x6710 +# define FMT_TRUNCATE_EN (1 << 0) +# define FMT_TRUNCATE_DEPTH (1 << 4) +# define FMT_SPATIAL_DITHER_EN (1 << 8) +# define FMT_SPATIAL_DITHER_MODE(x) ((x) << 9) +# define FMT_SPATIAL_DITHER_DEPTH (1 << 12) +# define FMT_FRAME_RANDOM_ENABLE (1 << 13) +# define FMT_RGB_RANDOM_ENABLE (1 << 14) +# define FMT_HIGHPASS_RANDOM_ENABLE (1 << 15) +# define FMT_TEMPORAL_DITHER_EN (1 << 16) +# define FMT_TEMPORAL_DITHER_DEPTH (1 << 20) +# define FMT_TEMPORAL_DITHER_OFFSET(x) ((x) << 21) +# define FMT_TEMPORAL_LEVEL (1 << 24) +# define FMT_TEMPORAL_DITHER_RESET (1 << 25) +# define FMT_25FRC_SEL(x) ((x) << 26) +# define FMT_50FRC_SEL(x) ((x) << 28) +# define FMT_75FRC_SEL(x) ((x) << 30) +#define FMT_CLAMP_CONTROL 0x672c +# define FMT_CLAMP_DATA_EN (1 << 0) +# define FMT_CLAMP_COLOR_FORMAT(x) ((x) << 16) +# define FMT_CLAMP_6BPC 0 +# define FMT_CLAMP_8BPC 1 +# define FMT_CLAMP_10BPC 2 + +/* Power management */ +#define CG_SPLL_FUNC_CNTL 0x600 +# define SPLL_RESET (1 << 0) +# define SPLL_SLEEP (1 << 1) +# define SPLL_REF_DIV(x) ((x) << 2) +# define SPLL_REF_DIV_MASK (7 << 2) +# define SPLL_FB_DIV(x) ((x) << 5) +# define SPLL_FB_DIV_MASK (0xff << 5) +# define SPLL_PULSEEN (1 << 13) +# define SPLL_PULSENUM(x) ((x) << 14) +# define SPLL_PULSENUM_MASK (3 << 14) +# define SPLL_SW_HILEN(x) ((x) << 16) +# define SPLL_SW_HILEN_MASK (0xf << 16) +# define SPLL_SW_LOLEN(x) ((x) << 20) +# define SPLL_SW_LOLEN_MASK (0xf << 20) +# define SPLL_DIVEN (1 << 24) +# define SPLL_BYPASS_EN (1 << 25) +# define SPLL_CHG_STATUS (1 << 29) +# define SPLL_CTLREQ (1 << 30) +# define SPLL_CTLACK (1 << 31) + +#define GENERAL_PWRMGT 0x618 +# define GLOBAL_PWRMGT_EN (1 << 0) +# define STATIC_PM_EN (1 << 1) +# define MOBILE_SU (1 << 2) +# define THERMAL_PROTECTION_DIS (1 << 3) +# define THERMAL_PROTECTION_TYPE (1 << 4) +# define ENABLE_GEN2PCIE (1 << 5) +# define SW_GPIO_INDEX(x) ((x) << 6) +# define SW_GPIO_INDEX_MASK (3 << 6) +# define LOW_VOLT_D2_ACPI (1 << 8) +# define LOW_VOLT_D3_ACPI (1 << 9) +# define VOLT_PWRMGT_EN (1 << 10) +#define CG_TPC 0x61c +# define TPCC(x) ((x) << 0) +# define TPCC_MASK (0x7fffff << 0) +# define TPU(x) ((x) << 23) +# define TPU_MASK (0x1f << 23) +#define SCLK_PWRMGT_CNTL 0x620 +# define SCLK_PWRMGT_OFF (1 << 0) +# define SCLK_TURNOFF (1 << 1) +# define SPLL_TURNOFF (1 << 2) +# define SU_SCLK_USE_BCLK (1 << 3) +# define DYNAMIC_GFX_ISLAND_PWR_DOWN (1 << 4) +# define DYNAMIC_GFX_ISLAND_PWR_LP (1 << 5) +# define CLK_TURN_ON_STAGGER (1 << 6) +# define CLK_TURN_OFF_STAGGER (1 << 7) +# define FIR_FORCE_TREND_SEL (1 << 8) +# define FIR_TREND_MODE (1 << 9) +# define DYN_GFX_CLK_OFF_EN (1 << 10) +# define VDDC3D_TURNOFF_D1 (1 << 11) +# define VDDC3D_TURNOFF_D2 (1 << 12) +# define VDDC3D_TURNOFF_D3 (1 << 13) +# define SPLL_TURNOFF_D2 (1 << 14) +# define SCLK_LOW_D1 (1 << 15) +# define DYN_GFX_CLK_OFF_MC_EN (1 << 16) +#define MCLK_PWRMGT_CNTL 0x624 +# define MPLL_PWRMGT_OFF (1 << 0) +# define YCLK_TURNOFF (1 << 1) +# define MPLL_TURNOFF (1 << 2) +# define SU_MCLK_USE_BCLK (1 << 3) +# define DLL_READY (1 << 4) +# define MC_BUSY (1 << 5) +# define MC_INT_CNTL (1 << 7) +# define MRDCKA_SLEEP (1 << 8) +# define MRDCKB_SLEEP (1 << 9) +# define MRDCKC_SLEEP (1 << 10) +# define MRDCKD_SLEEP (1 << 11) +# define MRDCKE_SLEEP (1 << 12) +# define MRDCKF_SLEEP (1 << 13) +# define MRDCKG_SLEEP (1 << 14) +# define MRDCKH_SLEEP (1 << 15) +# define MRDCKA_RESET (1 << 16) +# define MRDCKB_RESET (1 << 17) +# define MRDCKC_RESET (1 << 18) +# define MRDCKD_RESET (1 << 19) +# define MRDCKE_RESET (1 << 20) +# define MRDCKF_RESET (1 << 21) +# define MRDCKG_RESET (1 << 22) +# define MRDCKH_RESET (1 << 23) +# define DLL_READY_READ (1 << 24) +# define USE_DISPLAY_GAP (1 << 25) +# define USE_DISPLAY_URGENT_NORMAL (1 << 26) +# define USE_DISPLAY_GAP_CTXSW (1 << 27) +# define MPLL_TURNOFF_D2 (1 << 28) +# define USE_DISPLAY_URGENT_CTXSW (1 << 29) + +#define MPLL_TIME 0x634 +# define MPLL_LOCK_TIME(x) ((x) << 0) +# define MPLL_LOCK_TIME_MASK (0xffff << 0) +# define MPLL_RESET_TIME(x) ((x) << 16) +# define MPLL_RESET_TIME_MASK (0xffff << 16) + +#define SCLK_FREQ_SETTING_STEP_0_PART1 0x648 +# define STEP_0_SPLL_POST_DIV(x) ((x) << 0) +# define STEP_0_SPLL_POST_DIV_MASK (0xff << 0) +# define STEP_0_SPLL_FB_DIV(x) ((x) << 8) +# define STEP_0_SPLL_FB_DIV_MASK (0xff << 8) +# define STEP_0_SPLL_REF_DIV(x) ((x) << 16) +# define STEP_0_SPLL_REF_DIV_MASK (7 << 16) +# define STEP_0_SPLL_STEP_TIME(x) ((x) << 19) +# define STEP_0_SPLL_STEP_TIME_MASK (0x1fff << 19) +#define SCLK_FREQ_SETTING_STEP_0_PART2 0x64c +# define STEP_0_PULSE_HIGH_CNT(x) ((x) << 0) +# define STEP_0_PULSE_HIGH_CNT_MASK (0x1ff << 0) +# define STEP_0_POST_DIV_EN (1 << 9) +# define STEP_0_SPLL_STEP_ENABLE (1 << 30) +# define STEP_0_SPLL_ENTRY_VALID (1 << 31) + +#define VID_RT 0x6f8 +# define VID_CRT(x) ((x) << 0) +# define VID_CRT_MASK (0x1fff << 0) +# define VID_CRTU(x) ((x) << 13) +# define VID_CRTU_MASK (7 << 13) +# define SSTU(x) ((x) << 16) +# define SSTU_MASK (7 << 16) +#define CTXSW_PROFILE_INDEX 0x6fc +# define CTXSW_FREQ_VIDS_CFG_INDEX(x) ((x) << 0) +# define CTXSW_FREQ_VIDS_CFG_INDEX_MASK (3 << 0) +# define CTXSW_FREQ_VIDS_CFG_INDEX_SHIFT 0 +# define CTXSW_FREQ_MCLK_CFG_INDEX(x) ((x) << 2) +# define CTXSW_FREQ_MCLK_CFG_INDEX_MASK (3 << 2) +# define CTXSW_FREQ_MCLK_CFG_INDEX_SHIFT 2 +# define CTXSW_FREQ_SCLK_CFG_INDEX(x) ((x) << 4) +# define CTXSW_FREQ_SCLK_CFG_INDEX_MASK (0x1f << 4) +# define CTXSW_FREQ_SCLK_CFG_INDEX_SHIFT 4 +# define CTXSW_FREQ_STATE_SPLL_RESET_EN (1 << 9) +# define CTXSW_FREQ_STATE_ENABLE (1 << 10) +# define CTXSW_FREQ_DISPLAY_WATERMARK (1 << 11) +# define CTXSW_FREQ_GEN2PCIE_VOLT (1 << 12) + +#define TARGET_AND_CURRENT_PROFILE_INDEX 0x70c +# define TARGET_PROFILE_INDEX_MASK (3 << 0) +# define TARGET_PROFILE_INDEX_SHIFT 0 +# define CURRENT_PROFILE_INDEX_MASK (3 << 2) +# define CURRENT_PROFILE_INDEX_SHIFT 2 +# define DYN_PWR_ENTER_INDEX(x) ((x) << 4) +# define DYN_PWR_ENTER_INDEX_MASK (3 << 4) +# define DYN_PWR_ENTER_INDEX_SHIFT 4 +# define CURR_MCLK_INDEX_MASK (3 << 6) +# define CURR_MCLK_INDEX_SHIFT 6 +# define CURR_SCLK_INDEX_MASK (0x1f << 8) +# define CURR_SCLK_INDEX_SHIFT 8 +# define CURR_VID_INDEX_MASK (3 << 13) +# define CURR_VID_INDEX_SHIFT 13 + +#define LOWER_GPIO_ENABLE 0x710 +#define UPPER_GPIO_ENABLE 0x714 +#define CTXSW_VID_LOWER_GPIO_CNTL 0x718 + +#define VID_UPPER_GPIO_CNTL 0x740 +#define CG_CTX_CGTT3D_R 0x744 +# define PHC(x) ((x) << 0) +# define PHC_MASK (0x1ff << 0) +# define SDC(x) ((x) << 9) +# define SDC_MASK (0x3fff << 9) +#define CG_VDDC3D_OOR 0x748 +# define SU(x) ((x) << 23) +# define SU_MASK (0xf << 23) +#define CG_FTV 0x74c +#define CG_FFCT_0 0x750 +# define UTC_0(x) ((x) << 0) +# define UTC_0_MASK (0x3ff << 0) +# define DTC_0(x) ((x) << 10) +# define DTC_0_MASK (0x3ff << 10) + +#define CG_BSP 0x78c +# define BSP(x) ((x) << 0) +# define BSP_MASK (0xffff << 0) +# define BSU(x) ((x) << 16) +# define BSU_MASK (0xf << 16) +#define CG_RT 0x790 +# define FLS(x) ((x) << 0) +# define FLS_MASK (0xffff << 0) +# define FMS(x) ((x) << 16) +# define FMS_MASK (0xffff << 16) +#define CG_LT 0x794 +# define FHS(x) ((x) << 0) +# define FHS_MASK (0xffff << 0) +#define CG_GIT 0x798 +# define CG_GICST(x) ((x) << 0) +# define CG_GICST_MASK (0xffff << 0) +# define CG_GIPOT(x) ((x) << 16) +# define CG_GIPOT_MASK (0xffff << 16) + +#define CG_SSP 0x7a8 +# define CG_SST(x) ((x) << 0) +# define CG_SST_MASK (0xffff << 0) +# define CG_SSTU(x) ((x) << 16) +# define CG_SSTU_MASK (0xf << 16) + +#define CG_RLC_REQ_AND_RSP 0x7c4 +# define RLC_CG_REQ_TYPE_MASK 0xf +# define RLC_CG_REQ_TYPE_SHIFT 0 +# define CG_RLC_RSP_TYPE_MASK 0xf0 +# define CG_RLC_RSP_TYPE_SHIFT 4 + +#define CG_FC_T 0x7cc +# define FC_T(x) ((x) << 0) +# define FC_T_MASK (0xffff << 0) +# define FC_TU(x) ((x) << 16) +# define FC_TU_MASK (0x1f << 16) + +#define GPIOPAD_MASK 0x1798 +#define GPIOPAD_A 0x179c +#define GPIOPAD_EN 0x17a0 + +#define GRBM_PWR_CNTL 0x800c +# define REQ_TYPE_MASK 0xf +# define REQ_TYPE_SHIFT 0 +# define RSP_TYPE_MASK 0xf0 +# define RSP_TYPE_SHIFT 4 + +/* + * UVD + */ +#define UVD_SEMA_ADDR_LOW 0xef00 +#define UVD_SEMA_ADDR_HIGH 0xef04 +#define UVD_SEMA_CMD 0xef08 + +#define UVD_GPCOM_VCPU_CMD 0xef0c +#define UVD_GPCOM_VCPU_DATA0 0xef10 +#define UVD_GPCOM_VCPU_DATA1 0xef14 +#define UVD_ENGINE_CNTL 0xef18 + +#define UVD_SEMA_CNTL 0xf400 +#define UVD_RB_ARB_CTRL 0xf480 + +#define UVD_LMI_EXT40_ADDR 0xf498 +#define UVD_CGC_GATE 0xf4a8 +#define UVD_LMI_CTRL2 0xf4f4 +#define UVD_MASTINT_EN 0xf500 +#define UVD_FW_START 0xf51C +#define UVD_LMI_ADDR_EXT 0xf594 +#define UVD_LMI_CTRL 0xf598 +#define UVD_LMI_SWAP_CNTL 0xf5b4 +#define UVD_MP_SWAP_CNTL 0xf5bC +#define UVD_MPC_CNTL 0xf5dC +#define UVD_MPC_SET_MUXA0 0xf5e4 +#define UVD_MPC_SET_MUXA1 0xf5e8 +#define UVD_MPC_SET_MUXB0 0xf5eC +#define UVD_MPC_SET_MUXB1 0xf5f0 +#define UVD_MPC_SET_MUX 0xf5f4 +#define UVD_MPC_SET_ALU 0xf5f8 + +#define UVD_VCPU_CACHE_OFFSET0 0xf608 +#define UVD_VCPU_CACHE_SIZE0 0xf60c +#define UVD_VCPU_CACHE_OFFSET1 0xf610 +#define UVD_VCPU_CACHE_SIZE1 0xf614 +#define UVD_VCPU_CACHE_OFFSET2 0xf618 +#define UVD_VCPU_CACHE_SIZE2 0xf61c + +#define UVD_VCPU_CNTL 0xf660 +#define UVD_SOFT_RESET 0xf680 +#define RBC_SOFT_RESET (1<<0) +#define LBSI_SOFT_RESET (1<<1) +#define LMI_SOFT_RESET (1<<2) +#define VCPU_SOFT_RESET (1<<3) +#define CSM_SOFT_RESET (1<<5) +#define CXW_SOFT_RESET (1<<6) +#define TAP_SOFT_RESET (1<<7) +#define LMI_UMC_SOFT_RESET (1<<13) +#define UVD_RBC_IB_BASE 0xf684 +#define UVD_RBC_IB_SIZE 0xf688 +#define UVD_RBC_RB_BASE 0xf68c +#define UVD_RBC_RB_RPTR 0xf690 +#define UVD_RBC_RB_WPTR 0xf694 +#define UVD_RBC_RB_WPTR_CNTL 0xf698 + +#define UVD_STATUS 0xf6bc + +#define UVD_SEMA_TIMEOUT_STATUS 0xf6c0 +#define UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL 0xf6c4 +#define UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL 0xf6c8 +#define UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL 0xf6cc + +#define UVD_RBC_RB_CNTL 0xf6a4 +#define UVD_RBC_RB_RPTR_ADDR 0xf6a8 + +#define UVD_CONTEXT_ID 0xf6f4 + +/* rs780 only */ +#define GFX_MACRO_BYPASS_CNTL 0x30c0 +#define SPLL_BYPASS_CNTL (1 << 0) +#define UPLL_BYPASS_CNTL (1 << 1) + +#define CG_UPLL_FUNC_CNTL 0x7e0 +# define UPLL_RESET_MASK 0x00000001 +# define UPLL_SLEEP_MASK 0x00000002 +# define UPLL_BYPASS_EN_MASK 0x00000004 +# define UPLL_CTLREQ_MASK 0x00000008 +# define UPLL_FB_DIV(x) ((x) << 4) +# define UPLL_FB_DIV_MASK 0x0000FFF0 +# define UPLL_REF_DIV(x) ((x) << 16) +# define UPLL_REF_DIV_MASK 0x003F0000 +# define UPLL_REFCLK_SRC_SEL_MASK 0x20000000 +# define UPLL_CTLACK_MASK 0x40000000 +# define UPLL_CTLACK2_MASK 0x80000000 +#define CG_UPLL_FUNC_CNTL_2 0x7e4 +# define UPLL_SW_HILEN(x) ((x) << 0) +# define UPLL_SW_LOLEN(x) ((x) << 4) +# define UPLL_SW_HILEN2(x) ((x) << 8) +# define UPLL_SW_LOLEN2(x) ((x) << 12) +# define UPLL_DIVEN_MASK 0x00010000 +# define UPLL_DIVEN2_MASK 0x00020000 +# define UPLL_SW_MASK 0x0003FFFF +# define VCLK_SRC_SEL(x) ((x) << 20) +# define VCLK_SRC_SEL_MASK 0x01F00000 +# define DCLK_SRC_SEL(x) ((x) << 25) +# define DCLK_SRC_SEL_MASK 0x3E000000 + /* * PM4 */ -#define PACKET_TYPE0 0 -#define PACKET_TYPE1 1 -#define PACKET_TYPE2 2 -#define PACKET_TYPE3 3 - -#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) -#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) -#define CP_PACKET0_GET_REG(h) (((h) & 0xFFFF) << 2) -#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) -#define PACKET0(reg, n) ((PACKET_TYPE0 << 30) | \ +#define PACKET0(reg, n) ((RADEON_PACKET_TYPE0 << 30) | \ (((reg) >> 2) & 0xFFFF) | \ ((n) & 0x3FFF) << 16) -#define PACKET3(op, n) ((PACKET_TYPE3 << 30) | \ +#define PACKET3(op, n) ((RADEON_PACKET_TYPE3 << 30) | \ (((op) & 0xFF) << 8) | \ ((n) & 0x3FFF) << 16) @@ -1219,6 +1646,7 @@ */ # define PACKET3_CP_DMA_CMD_SAIC (1 << 28) # define PACKET3_CP_DMA_CMD_DAIC (1 << 29) +#define PACKET3_PFP_SYNC_ME 0x42 /* r7xx+ only */ #define PACKET3_SURFACE_SYNC 0x43 # define PACKET3_CB0_DEST_BASE_ENA (1 << 6) # define PACKET3_FULL_CACHE_ENA (1 << 20) /* r7xx+ only */ @@ -1281,6 +1709,14 @@ #define PACKET3_STRMOUT_BASE_UPDATE 0x72 /* r7xx */ #define PACKET3_SURFACE_BASE_UPDATE 0x73 +#define R_000011_K8_FB_LOCATION 0x11 +#define R_000012_MC_MISC_UMA_CNTL 0x12 +#define G_000012_K8_ADDR_EXT(x) (((x) >> 0) & 0xFF) +#define R_0028F8_MC_INDEX 0x28F8 +#define S_0028F8_MC_IND_ADDR(x) (((x) & 0x1FF) << 0) +#define C_0028F8_MC_IND_ADDR 0xFFFFFE00 +#define S_0028F8_MC_IND_WR_EN(x) (((x) & 0x1) << 9) +#define R_0028FC_MC_DATA 0x28FC #define R_008020_GRBM_SOFT_RESET 0x8020 #define S_008020_SOFT_RESET_CP(x) (((x) & 1) << 0) @@ -1330,6 +1766,7 @@ #define G_008010_VC_BUSY(x) (((x) >> 11) & 1) #define G_008010_DB03_CLEAN(x) (((x) >> 12) & 1) #define G_008010_CB03_CLEAN(x) (((x) >> 13) & 1) +#define G_008010_TA_BUSY(x) (((x) >> 14) & 1) #define G_008010_VGT_BUSY_NO_DMA(x) (((x) >> 16) & 1) #define G_008010_VGT_BUSY(x) (((x) >> 17) & 1) #define G_008010_TA03_BUSY(x) (((x) >> 18) & 1) @@ -1397,6 +1834,7 @@ #define G_000E50_MCDW_BUSY(x) (((x) >> 13) & 1) #define G_000E50_SEM_BUSY(x) (((x) >> 14) & 1) #define G_000E50_RLC_BUSY(x) (((x) >> 15) & 1) +#define G_000E50_IH_BUSY(x) (((x) >> 17) & 1) #define G_000E50_BIF_BUSY(x) (((x) >> 29) & 1) #define R_000E60_SRBM_SOFT_RESET 0x0E60 #define S_000E60_SOFT_RESET_BIF(x) (((x) & 1) << 1) diff --git a/sys/dev/pci/drm/radeon/radeon.h b/sys/dev/pci/drm/radeon/radeon.h index 748cf91ce2c..2acad16f0e9 100644 --- a/sys/dev/pci/drm/radeon/radeon.h +++ b/sys/dev/pci/drm/radeon/radeon.h @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon.h,v 1.20 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -61,12 +60,16 @@ * are considered as fatal) */ +#include + #include #include #include #include #include +#include + #include #include #include @@ -99,26 +102,32 @@ extern int radeon_hw_i2c; extern int radeon_pcie_gen2; extern int radeon_msi; extern int radeon_lockup_timeout; +extern int radeon_fastfb; +extern int radeon_dpm; +extern int radeon_aspm; +extern int radeon_runtime_pm; +extern int radeon_hard_reset; +extern int radeon_vm_size; +extern int radeon_vm_block_size; +extern int radeon_deep_color; +extern int radeon_use_pflipirq; +extern int radeon_bapm; +extern int radeon_backlight; extern int radeon_auxch; +extern int radeon_mst; /* * Copy from radeon_drv.h so we don't have to include both and have conflicting * symbol; */ #define RADEON_MAX_USEC_TIMEOUT 100000 /* 100 ms */ -#define RADEON_FENCE_JIFFIES_TIMEOUT (hz / 2) +#define RADEON_FENCE_JIFFIES_TIMEOUT (HZ / 2) /* RADEON_IB_POOL_SIZE must be a power of 2 */ #define RADEON_IB_POOL_SIZE 16 #define RADEON_DEBUGFS_MAX_COMPONENTS 32 #define RADEONFB_CONN_LIMIT 4 #define RADEON_BIOS_NUM_SCRATCH 8 -/* max number of rings */ -#define RADEON_NUM_RINGS 5 - -/* fence seq are set to this number when signaled */ -#define RADEON_FENCE_SIGNALED_SEQ 0LL - /* internal ring indices */ /* r1xx+ has gfx CP ring */ #define RADEON_RING_TYPE_GFX_INDEX 0 @@ -132,15 +141,88 @@ extern int radeon_auxch; /* cayman add a second async dma ring */ #define CAYMAN_RING_TYPE_DMA1_INDEX 4 +/* R600+ */ +#define R600_RING_TYPE_UVD_INDEX 5 + +/* TN+ */ +#define TN_RING_TYPE_VCE1_INDEX 6 +#define TN_RING_TYPE_VCE2_INDEX 7 + +/* max number of rings */ +#define RADEON_NUM_RINGS 8 + +/* number of hw syncs before falling back on blocking */ +#define RADEON_NUM_SYNCS 4 + /* hardcode those limit for now */ #define RADEON_VA_IB_OFFSET (1 << 20) #define RADEON_VA_RESERVED_SIZE (8 << 20) #define RADEON_IB_VM_MAX_SIZE (64 << 10) +/* hard reset data */ +#define RADEON_ASIC_RESET_DATA 0x39d5e86b + /* reset flags */ #define RADEON_RESET_GFX (1 << 0) #define RADEON_RESET_COMPUTE (1 << 1) #define RADEON_RESET_DMA (1 << 2) +#define RADEON_RESET_CP (1 << 3) +#define RADEON_RESET_GRBM (1 << 4) +#define RADEON_RESET_DMA1 (1 << 5) +#define RADEON_RESET_RLC (1 << 6) +#define RADEON_RESET_SEM (1 << 7) +#define RADEON_RESET_IH (1 << 8) +#define RADEON_RESET_VMC (1 << 9) +#define RADEON_RESET_MC (1 << 10) +#define RADEON_RESET_DISPLAY (1 << 11) + +/* CG block flags */ +#define RADEON_CG_BLOCK_GFX (1 << 0) +#define RADEON_CG_BLOCK_MC (1 << 1) +#define RADEON_CG_BLOCK_SDMA (1 << 2) +#define RADEON_CG_BLOCK_UVD (1 << 3) +#define RADEON_CG_BLOCK_VCE (1 << 4) +#define RADEON_CG_BLOCK_HDP (1 << 5) +#define RADEON_CG_BLOCK_BIF (1 << 6) + +/* CG flags */ +#define RADEON_CG_SUPPORT_GFX_MGCG (1 << 0) +#define RADEON_CG_SUPPORT_GFX_MGLS (1 << 1) +#define RADEON_CG_SUPPORT_GFX_CGCG (1 << 2) +#define RADEON_CG_SUPPORT_GFX_CGLS (1 << 3) +#define RADEON_CG_SUPPORT_GFX_CGTS (1 << 4) +#define RADEON_CG_SUPPORT_GFX_CGTS_LS (1 << 5) +#define RADEON_CG_SUPPORT_GFX_CP_LS (1 << 6) +#define RADEON_CG_SUPPORT_GFX_RLC_LS (1 << 7) +#define RADEON_CG_SUPPORT_MC_LS (1 << 8) +#define RADEON_CG_SUPPORT_MC_MGCG (1 << 9) +#define RADEON_CG_SUPPORT_SDMA_LS (1 << 10) +#define RADEON_CG_SUPPORT_SDMA_MGCG (1 << 11) +#define RADEON_CG_SUPPORT_BIF_LS (1 << 12) +#define RADEON_CG_SUPPORT_UVD_MGCG (1 << 13) +#define RADEON_CG_SUPPORT_VCE_MGCG (1 << 14) +#define RADEON_CG_SUPPORT_HDP_LS (1 << 15) +#define RADEON_CG_SUPPORT_HDP_MGCG (1 << 16) + +/* PG flags */ +#define RADEON_PG_SUPPORT_GFX_PG (1 << 0) +#define RADEON_PG_SUPPORT_GFX_SMG (1 << 1) +#define RADEON_PG_SUPPORT_GFX_DMG (1 << 2) +#define RADEON_PG_SUPPORT_UVD (1 << 3) +#define RADEON_PG_SUPPORT_VCE (1 << 4) +#define RADEON_PG_SUPPORT_CP (1 << 5) +#define RADEON_PG_SUPPORT_GDS (1 << 6) +#define RADEON_PG_SUPPORT_RLC_SMU_HS (1 << 7) +#define RADEON_PG_SUPPORT_SDMA (1 << 8) +#define RADEON_PG_SUPPORT_ACP (1 << 9) +#define RADEON_PG_SUPPORT_SAMU (1 << 10) + +/* max cursor sizes (in pixels) */ +#define CURSOR_WIDTH 64 +#define CURSOR_HEIGHT 64 + +#define CIK_CURSOR_WIDTH 128 +#define CIK_CURSOR_HEIGHT 128 /* * Errata workarounds. @@ -164,8 +246,9 @@ bool radeon_get_bios(struct radeon_device *rdev); * Dummy page */ struct radeon_dummy_page { + uint64_t entry; struct drm_dmamem *dmah; - bus_addr_t addr; + dma_addr_t addr; }; int radeon_dummy_page_init(struct radeon_device *rdev); void radeon_dummy_page_fini(struct radeon_device *rdev); @@ -184,27 +267,85 @@ struct radeon_clock { uint32_t default_mclk; uint32_t default_sclk; uint32_t default_dispclk; + uint32_t current_dispclk; uint32_t dp_extclk; uint32_t max_pixel_clock; + uint32_t vco_freq; }; /* * Power management */ int radeon_pm_init(struct radeon_device *rdev); +int radeon_pm_late_init(struct radeon_device *rdev); void radeon_pm_fini(struct radeon_device *rdev); void radeon_pm_compute_clocks(struct radeon_device *rdev); void radeon_pm_suspend(struct radeon_device *rdev); void radeon_pm_resume(struct radeon_device *rdev); void radeon_combios_get_power_modes(struct radeon_device *rdev); void radeon_atombios_get_power_modes(struct radeon_device *rdev); +int radeon_atom_get_clock_dividers(struct radeon_device *rdev, + u8 clock_type, + u32 clock, + bool strobe_mode, + struct atom_clock_dividers *dividers); +int radeon_atom_get_memory_pll_dividers(struct radeon_device *rdev, + u32 clock, + bool strobe_mode, + struct atom_mpll_param *mpll_param); void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 voltage_type); +int radeon_atom_get_voltage_gpio_settings(struct radeon_device *rdev, + u16 voltage_level, u8 voltage_type, + u32 *gpio_value, u32 *gpio_mask); +void radeon_atom_set_engine_dram_timings(struct radeon_device *rdev, + u32 eng_clock, u32 mem_clock); +int radeon_atom_get_voltage_step(struct radeon_device *rdev, + u8 voltage_type, u16 *voltage_step); +int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type, + u16 voltage_id, u16 *voltage); +int radeon_atom_get_leakage_vddc_based_on_leakage_idx(struct radeon_device *rdev, + u16 *voltage, + u16 leakage_idx); +int radeon_atom_get_leakage_id_from_vbios(struct radeon_device *rdev, + u16 *leakage_id); +int radeon_atom_get_leakage_vddc_based_on_leakage_params(struct radeon_device *rdev, + u16 *vddc, u16 *vddci, + u16 virtual_voltage_id, + u16 vbios_voltage_id); +int radeon_atom_get_voltage_evv(struct radeon_device *rdev, + u16 virtual_voltage_id, + u16 *voltage); +int radeon_atom_round_to_true_voltage(struct radeon_device *rdev, + u8 voltage_type, + u16 nominal_voltage, + u16 *true_voltage); +int radeon_atom_get_min_voltage(struct radeon_device *rdev, + u8 voltage_type, u16 *min_voltage); +int radeon_atom_get_max_voltage(struct radeon_device *rdev, + u8 voltage_type, u16 *max_voltage); +int radeon_atom_get_voltage_table(struct radeon_device *rdev, + u8 voltage_type, u8 voltage_mode, + struct atom_voltage_table *voltage_table); +bool radeon_atom_is_voltage_gpio(struct radeon_device *rdev, + u8 voltage_type, u8 voltage_mode); +int radeon_atom_get_svi2_info(struct radeon_device *rdev, + u8 voltage_type, + u8 *svd_gpio_id, u8 *svc_gpio_id); +void radeon_atom_update_memory_dll(struct radeon_device *rdev, + u32 mem_clock); +void radeon_atom_set_ac_timing(struct radeon_device *rdev, + u32 mem_clock); +int radeon_atom_init_mc_reg_table(struct radeon_device *rdev, + u8 module_index, + struct atom_mc_reg_table *reg_table); +int radeon_atom_get_memory_info(struct radeon_device *rdev, + u8 module_index, struct atom_memory_info *mem_info); +int radeon_atom_get_mclk_range_table(struct radeon_device *rdev, + bool gddr5, u8 module_index, + struct atom_memory_clock_range_table *mclk_range_table); +int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type, + u16 voltage_id, u16 *voltage); void rs690_pm_info(struct radeon_device *rdev); -extern int rv6xx_get_temp(struct radeon_device *rdev); -extern int rv770_get_temp(struct radeon_device *rdev); -extern int evergreen_get_temp(struct radeon_device *rdev); -extern int sumo_get_temp(struct radeon_device *rdev); -extern int si_get_temp(struct radeon_device *rdev); extern void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, unsigned *bankh, unsigned *mtaspect, unsigned *tile_split); @@ -213,35 +354,39 @@ extern void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, * Fences. */ struct radeon_fence_driver { + struct radeon_device *rdev; uint32_t scratch_reg; uint64_t gpu_addr; volatile uint32_t *cpu_addr; /* sync_seq is protected by ring emission lock */ uint64_t sync_seq[RADEON_NUM_RINGS]; atomic64_t last_seq; - unsigned long last_activity; - bool initialized; + bool initialized, delayed_irq; + struct delayed_work lockup_work; }; struct radeon_fence { - struct radeon_device *rdev; - struct kref kref; - /* protected by radeon_fence.lock */ - uint64_t seq; + struct fence base; + + struct radeon_device *rdev; + uint64_t seq; /* RB, DMA, etc. */ - unsigned ring; + unsigned ring; + bool is_vm_update; + + wait_queue_t fence_wake; }; int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring); int radeon_fence_driver_init(struct radeon_device *rdev); void radeon_fence_driver_fini(struct radeon_device *rdev); -void radeon_fence_driver_force_completion(struct radeon_device *rdev); +void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring); int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence **fence, int ring); void radeon_fence_process(struct radeon_device *rdev, int ring); bool radeon_fence_signaled(struct radeon_fence *fence); int radeon_fence_wait(struct radeon_fence *fence, bool interruptible); -int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring); -int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring); +int radeon_fence_wait_next(struct radeon_device *rdev, int ring); +int radeon_fence_wait_empty(struct radeon_device *rdev, int ring); int radeon_fence_wait_any(struct radeon_device *rdev, struct radeon_fence **fences, bool intr); @@ -304,20 +449,33 @@ struct radeon_mman { struct ttm_bo_device bdev; bool mem_global_referenced; bool initialized; + +#if defined(CONFIG_DEBUG_FS) + struct dentry *vram; + struct dentry *gtt; +#endif +}; + +struct radeon_bo_list { + struct radeon_bo *robj; + struct ttm_validate_buffer tv; + uint64_t gpu_offset; + unsigned prefered_domains; + unsigned allowed_domains; + uint32_t tiling_flags; }; /* bo virtual address in a specific vm */ struct radeon_bo_va { /* protected by bo being reserved */ struct list_head bo_list; - uint64_t soffset; - uint64_t eoffset; uint32_t flags; - bool valid; + struct radeon_fence *last_pt_update; unsigned ref_count; - /* protected by vm rwlock */ - struct list_head vm_list; + /* protected by vm mutex */ + struct interval_tree_node it; + struct list_head vm_status; /* constant after initialization */ struct radeon_vm *vm; @@ -325,14 +483,15 @@ struct radeon_bo_va { }; struct radeon_bo { - struct drm_gem_object gem_base; - /* Protected by gem.rwlock */ + /* Protected by gem.mutex */ struct list_head list; /* Protected by tbo.reserved */ - u32 placements[3]; + u32 initial_domain; + struct ttm_place placements[4]; struct ttm_placement placement; struct ttm_buffer_object tbo; struct ttm_bo_kmap_obj kmap; + u32 flags; unsigned pin_count; void *kptr; u32 tiling_flags; @@ -344,20 +503,17 @@ struct radeon_bo { struct list_head va; /* Constant after initialization */ struct radeon_device *rdev; + struct drm_gem_object gem_base; + + struct ttm_bo_kmap_obj dma_buf_vmap; + pid_t pid; - struct ttm_bo_kmap_obj dma_buf_vmap; - int vmapping_count; + struct radeon_mn *mn; + struct list_head mn_list; }; #define gem_to_radeon_bo(gobj) container_of((gobj), struct radeon_bo, gem_base) -struct radeon_bo_list { - struct ttm_validate_buffer tv; - struct radeon_bo *bo; - uint64_t gpu_offset; - unsigned rdomain; - unsigned wdomain; - u32 tiling_flags; -}; +int radeon_gem_debugfs_init(struct radeon_device *rdev); /* sub-allocation manager, it has to be protected by another lock. * By conception this is an helper for other part of the driver @@ -417,9 +573,9 @@ struct radeon_gem { int radeon_gem_init(struct radeon_device *rdev); void radeon_gem_fini(struct radeon_device *rdev); -int radeon_gem_object_create(struct radeon_device *rdev, int size, +int radeon_gem_object_create(struct radeon_device *rdev, unsigned long size, int alignment, int initial_domain, - bool discardable, bool kernel, + u32 flags, bool kernel, struct drm_gem_object **obj); int radeon_mode_dumb_create(struct drm_file *file_priv, @@ -428,33 +584,48 @@ int radeon_mode_dumb_create(struct drm_file *file_priv, int radeon_mode_dumb_mmap(struct drm_file *filp, struct drm_device *dev, uint32_t handle, uint64_t *offset_p); -int radeon_mode_dumb_destroy(struct drm_file *file_priv, - struct drm_device *dev, - uint32_t handle); /* * Semaphores. */ -/* everything here is constant */ struct radeon_semaphore { - struct radeon_sa_bo *sa_bo; - signed waiters; - uint64_t gpu_addr; + struct radeon_sa_bo *sa_bo; + signed waiters; + uint64_t gpu_addr; }; int radeon_semaphore_create(struct radeon_device *rdev, struct radeon_semaphore **semaphore); -void radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring, +bool radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring, struct radeon_semaphore *semaphore); -void radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring, +bool radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring, struct radeon_semaphore *semaphore); -int radeon_semaphore_sync_rings(struct radeon_device *rdev, - struct radeon_semaphore *semaphore, - int signaler, int waiter); void radeon_semaphore_free(struct radeon_device *rdev, struct radeon_semaphore **semaphore, struct radeon_fence *fence); +/* + * Synchronization + */ +struct radeon_sync { + struct radeon_semaphore *semaphores[RADEON_NUM_SYNCS]; + struct radeon_fence *sync_to[RADEON_NUM_RINGS]; + struct radeon_fence *last_vm_update; +}; + +void radeon_sync_create(struct radeon_sync *sync); +void radeon_sync_fence(struct radeon_sync *sync, + struct radeon_fence *fence); +int radeon_sync_resv(struct radeon_device *rdev, + struct radeon_sync *sync, + struct reservation_object *resv, + bool shared); +int radeon_sync_rings(struct radeon_device *rdev, + struct radeon_sync *sync, + int waiting_ring); +void radeon_sync_free(struct radeon_device *rdev, struct radeon_sync *sync, + struct radeon_fence *fence); + /* * GART structures, functions & helpers */ @@ -465,8 +636,14 @@ struct radeon_mc; #define RADEON_GPU_PAGE_SHIFT 12 #define RADEON_GPU_PAGE_ALIGN(a) (((a) + RADEON_GPU_PAGE_MASK) & ~RADEON_GPU_PAGE_MASK) +#define RADEON_GART_PAGE_DUMMY 0 +#define RADEON_GART_PAGE_VALID (1 << 0) +#define RADEON_GART_PAGE_READ (1 << 1) +#define RADEON_GART_PAGE_WRITE (1 << 2) +#define RADEON_GART_PAGE_SNOOP (1 << 3) + struct radeon_gart { - bus_addr_t table_addr; + dma_addr_t table_addr; struct drm_dmamem *dmah; struct radeon_bo *robj; void *ptr; @@ -474,7 +651,7 @@ struct radeon_gart { unsigned num_cpu_pages; unsigned table_size; struct vm_page **pages; - bus_addr_t *pages_addr; + uint64_t *pages_entry; bool ready; }; @@ -490,17 +667,16 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, int pages); int radeon_gart_bind(struct radeon_device *rdev, unsigned offset, int pages, struct vm_page **pagelist, - bus_addr_t *dma_addr); -void radeon_gart_restore(struct radeon_device *rdev); + dma_addr_t *dma_addr, uint32_t flags); /* * GPU MC structures, functions & helpers */ struct radeon_mc { - bus_size_t aper_size; - bus_addr_t aper_base; - bus_addr_t agp_base; + resource_size_t aper_size; + resource_size_t aper_base; + resource_size_t agp_base; /* for some chips with <= 32MB we need to lie * about vram size near mc fb location */ u64 mc_vram_size; @@ -516,6 +692,7 @@ struct radeon_mc { bool vram_is_ddr; bool igp_sideport_enabled; u64 gtt_base_align; + u64 mc_mask; }; bool radeon_combios_sideport_present(struct radeon_device *rdev); @@ -534,19 +711,40 @@ struct radeon_scratch { int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg); void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg); +/* + * GPU doorbell structures, functions & helpers + */ +#define RADEON_MAX_DOORBELLS 1024 /* Reserve at most 1024 doorbell slots for radeon-owned rings. */ + +struct radeon_doorbell { + /* doorbell mmio */ + resource_size_t base; + resource_size_t size; + bus_space_handle_t bsh; + u32 num_doorbells; /* Number of doorbells actually reserved for radeon. */ + DECLARE_BITMAP(used, RADEON_MAX_DOORBELLS); +}; + +int radeon_doorbell_get(struct radeon_device *rdev, u32 *page); +void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell); +void radeon_doorbell_get_kfd_info(struct radeon_device *rdev, + phys_addr_t *aperture_base, + size_t *aperture_size, + size_t *start_offset); /* * IRQS. */ -struct radeon_unpin_work { - struct task task; - struct radeon_device *rdev; - int crtc_id; - struct radeon_fence *fence; +struct radeon_flip_work { + struct work_struct flip_work; + struct work_struct unpin_work; + struct radeon_device *rdev; + int crtc_id; + uint64_t base; struct drm_pending_vblank_event *event; - struct radeon_bo *old_rbo; - u64 new_crtc_base; + struct radeon_bo *old_rbo; + struct fence *fence; }; struct r500_irq_stat_regs { @@ -585,16 +783,29 @@ struct evergreen_irq_stat_regs { u32 afmt_status6; }; +struct cik_irq_stat_regs { + u32 disp_int; + u32 disp_int_cont; + u32 disp_int_cont2; + u32 disp_int_cont3; + u32 disp_int_cont4; + u32 disp_int_cont5; + u32 disp_int_cont6; + u32 d1grph_int; + u32 d2grph_int; + u32 d3grph_int; + u32 d4grph_int; + u32 d5grph_int; + u32 d6grph_int; +}; + union radeon_irq_stat_regs { struct r500_irq_stat_regs r500; struct r600_irq_stat_regs r600; struct evergreen_irq_stat_regs evergreen; + struct cik_irq_stat_regs cik; }; -#define RADEON_MAX_HPD_PINS 6 -#define RADEON_MAX_CRTCS 6 -#define RADEON_MAX_AFMT_BLOCKS 6 - struct radeon_irq { bool installed; spinlock_t lock; @@ -605,11 +816,13 @@ struct radeon_irq { bool hpd[RADEON_MAX_HPD_PINS]; bool afmt[RADEON_MAX_AFMT_BLOCKS]; union radeon_irq_stat_regs stat_regs; + bool dpm_thermal; }; int radeon_irq_kms_init(struct radeon_device *rdev); void radeon_irq_kms_fini(struct radeon_device *rdev); void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring); +bool radeon_irq_kms_sw_irq_get_delayed(struct radeon_device *rdev, int ring); void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev, int ring); void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc); void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc); @@ -617,7 +830,6 @@ void radeon_irq_kms_enable_afmt(struct radeon_device *rdev, int block); void radeon_irq_kms_disable_afmt(struct radeon_device *rdev, int block); void radeon_irq_kms_enable_hpd(struct radeon_device *rdev, unsigned hpd_mask); void radeon_irq_kms_disable_hpd(struct radeon_device *rdev, unsigned hpd_mask); -bool radeon_msi_ok(struct radeon_device *rdev); /* * CP & rings. @@ -632,37 +844,46 @@ struct radeon_ib { struct radeon_fence *fence; struct radeon_vm *vm; bool is_const_ib; - struct radeon_fence *sync_to[RADEON_NUM_RINGS]; - struct radeon_semaphore *semaphore; + struct radeon_sync sync; }; struct radeon_ring { struct radeon_bo *ring_obj; volatile uint32_t *ring; - unsigned rptr; unsigned rptr_offs; - unsigned rptr_reg; unsigned rptr_save_reg; u64 next_rptr_gpu_addr; volatile u32 *next_rptr_cpu_addr; unsigned wptr; unsigned wptr_old; - unsigned wptr_reg; unsigned ring_size; unsigned ring_free_dw; int count_dw; - unsigned long last_activity; - unsigned last_rptr; + atomic_t last_rptr; + atomic64_t last_activity; uint64_t gpu_addr; uint32_t align_mask; uint32_t ptr_mask; bool ready; - u32 ptr_reg_shift; - u32 ptr_reg_mask; u32 nop; u32 idx; u64 last_semaphore_signal_addr; u64 last_semaphore_wait_addr; + /* for CIK queues */ + u32 me; + u32 pipe; + u32 queue; + struct radeon_bo *mqd_obj; + u32 doorbell_index; + unsigned wptr_offs; +}; + +struct radeon_mec { + struct radeon_bo *hpd_eop_obj; + u64 hpd_eop_gpu_addr; + u32 num_pipe; + u32 num_mec; + u32 num_queue; }; /* @@ -672,38 +893,75 @@ struct radeon_ring { /* maximum number of VMIDs */ #define RADEON_NUM_VM 16 -/* defines number of bits in page table versus page directory, - * a page is 4KB so we have 12 bits offset, 9 bits in the page - * table and the remaining 19 bits are in the page directory */ -#define RADEON_VM_BLOCK_SIZE 9 - /* number of entries in page table */ -#define RADEON_VM_PTE_COUNT (1 << RADEON_VM_BLOCK_SIZE) +#define RADEON_VM_PTE_COUNT (1 << radeon_vm_block_size) + +/* PTBs (Page Table Blocks) need to be aligned to 32K */ +#define RADEON_VM_PTB_ALIGN_SIZE 32768 +#define RADEON_VM_PTB_ALIGN_MASK (RADEON_VM_PTB_ALIGN_SIZE - 1) +#define RADEON_VM_PTB_ALIGN(a) (((a) + RADEON_VM_PTB_ALIGN_MASK) & ~RADEON_VM_PTB_ALIGN_MASK) + +#define R600_PTE_VALID (1 << 0) +#define R600_PTE_SYSTEM (1 << 1) +#define R600_PTE_SNOOPED (1 << 2) +#define R600_PTE_READABLE (1 << 5) +#define R600_PTE_WRITEABLE (1 << 6) + +/* PTE (Page Table Entry) fragment field for different page sizes */ +#define R600_PTE_FRAG_4KB (0 << 7) +#define R600_PTE_FRAG_64KB (4 << 7) +#define R600_PTE_FRAG_256KB (6 << 7) + +/* flags needed to be set so we can copy directly from the GART table */ +#define R600_PTE_GART_MASK ( R600_PTE_READABLE | R600_PTE_WRITEABLE | \ + R600_PTE_SYSTEM | R600_PTE_VALID ) + +struct radeon_vm_pt { + struct radeon_bo *bo; + uint64_t addr; +}; + +struct radeon_vm_id { + unsigned id; + uint64_t pd_gpu_addr; + /* last flushed PD/PT update */ + struct radeon_fence *flushed_updates; + /* last use of vmid */ + struct radeon_fence *last_id_use; +}; struct radeon_vm { - struct list_head list; - struct list_head va; - unsigned id; + struct rwlock mutex; + + struct rb_root va; + + /* protecting invalidated and freed */ + spinlock_t status_lock; + + /* BOs moved, but not yet updated in the PT */ + struct list_head invalidated; + + /* BOs freed, but not yet updated in the PT */ + struct list_head freed; + + /* BOs cleared in the PT */ + struct list_head cleared; /* contains the page directory */ - struct radeon_sa_bo *page_directory; - uint64_t pd_gpu_addr; + struct radeon_bo *page_directory; + unsigned max_pde_used; /* array of page tables, one for each page directory entry */ - struct radeon_sa_bo **page_tables; + struct radeon_vm_pt *page_tables; - struct rwlock mutex; - /* last fence for cs using this vm */ - struct radeon_fence *fence; - /* last flush or NULL if we still need to flush */ - struct radeon_fence *last_flush; + struct radeon_bo_va *ib_bo_va; + + /* for id and flush management per ring */ + struct radeon_vm_id ids[RADEON_NUM_RINGS]; }; struct radeon_vm_manager { - struct rwlock lock; - struct list_head lru_vm; struct radeon_fence *active[RADEON_NUM_VM]; - struct radeon_sa_manager sa_manager; uint32_t max_pfn; /* number of VMIDs */ unsigned nvm; @@ -711,6 +969,8 @@ struct radeon_vm_manager { u64 vram_base_offset; /* is vm enabled? */ bool enabled; + /* for hw to save the PD addr on suspend/resume */ + uint32_t saved_table_addr[RADEON_NUM_VM]; }; /* @@ -734,45 +994,29 @@ struct r600_ih { bool enabled; }; -struct r600_blit_cp_primitives { - void (*set_render_target)(struct radeon_device *rdev, int format, - int w, int h, u64 gpu_addr); - void (*cp_set_surface_sync)(struct radeon_device *rdev, - u32 sync_type, u32 size, - u64 mc_addr); - void (*set_shaders)(struct radeon_device *rdev); - void (*set_vtx_resource)(struct radeon_device *rdev, u64 gpu_addr); - void (*set_tex_resource)(struct radeon_device *rdev, - int format, int w, int h, int pitch, - u64 gpu_addr, u32 size); - void (*set_scissors)(struct radeon_device *rdev, int x1, int y1, - int x2, int y2); - void (*draw_auto)(struct radeon_device *rdev); - void (*set_default_state)(struct radeon_device *rdev); -}; - -struct r600_blit { - struct radeon_bo *shader_obj; - struct r600_blit_cp_primitives primitives; - int max_dim; - int ring_size_common; - int ring_size_per_loop; - u64 shader_gpu_addr; - u32 vs_offset, ps_offset; - u32 state_offset; - u32 state_len; -}; - /* - * SI RLC stuff + * RLC stuff */ -struct si_rlc { +#include "clearstate_defs.h" + +struct radeon_rlc { /* for power gating */ struct radeon_bo *save_restore_obj; uint64_t save_restore_gpu_addr; + volatile uint32_t *sr_ptr; + const u32 *reg_list; + u32 reg_list_size; /* for clear state */ struct radeon_bo *clear_state_obj; uint64_t clear_state_gpu_addr; + volatile uint32_t *cs_ptr; + const struct cs_section_def *cs_data; + u32 clear_state_size; + /* for cp tables */ + struct radeon_bo *cp_table_obj; + uint64_t cp_table_gpu_addr; + volatile uint32_t *cp_table_ptr; + u32 cp_table_size; }; int radeon_ib_get(struct radeon_device *rdev, int ring, @@ -780,7 +1024,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring, unsigned size); void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib); int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib, - struct radeon_ib *const_ib); + struct radeon_ib *const_ib, bool hdp_flush); int radeon_ib_pool_init(struct radeon_device *rdev); void radeon_ib_pool_fini(struct radeon_device *rdev); int radeon_ib_ring_tests(struct radeon_device *rdev); @@ -790,21 +1034,22 @@ bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev, void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *cp); int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw); int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw); -void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *cp); -void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *cp); +void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *cp, + bool hdp_flush); +void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *cp, + bool hdp_flush); void radeon_ring_undo(struct radeon_ring *ring); void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *cp); int radeon_ring_test(struct radeon_device *rdev, struct radeon_ring *cp); -void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring *ring); -void radeon_ring_lockup_update(struct radeon_ring *ring); +void radeon_ring_lockup_update(struct radeon_device *rdev, + struct radeon_ring *ring); bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring); unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring, uint32_t **data); int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring, unsigned size, uint32_t *data); int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ring_size, - unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg, - u32 ptr_reg_shift, u32 ptr_reg_mask, u32 nop); + unsigned rptr_offs, u32 nop); void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *cp); @@ -820,23 +1065,10 @@ void cayman_dma_fini(struct radeon_device *rdev); /* * CS. */ -struct radeon_cs_reloc { - struct drm_gem_object *gobj; - struct radeon_bo *robj; - struct radeon_bo_list lobj; - uint32_t handle; - uint32_t flags; -}; - struct radeon_cs_chunk { - uint32_t chunk_id; uint32_t length_dw; - int kpage_idx[2]; - uint32_t *kpage[2]; uint32_t *kdata; void __user *user_ptr; - int last_copied_page; - int last_page_index; }; struct radeon_cs_parser { @@ -851,15 +1083,15 @@ struct radeon_cs_parser { unsigned idx; /* relocations */ unsigned nrelocs; - struct radeon_cs_reloc *relocs; - struct radeon_cs_reloc **relocs_ptr; + struct radeon_bo_list *relocs; + struct radeon_bo_list *vm_bos; struct list_head validated; unsigned dma_reloc_idx; /* indices of various chunks */ - int chunk_ib_idx; - int chunk_relocs_idx; - int chunk_flags_idx; - int chunk_const_ib_idx; + struct radeon_cs_chunk *chunk_ib; + struct radeon_cs_chunk *chunk_relocs; + struct radeon_cs_chunk *chunk_flags; + struct radeon_cs_chunk *chunk_const_ib; struct radeon_ib ib; struct radeon_ib const_ib; void *track; @@ -868,10 +1100,18 @@ struct radeon_cs_parser { u32 cs_flags; u32 ring; s32 priority; + struct ww_acquire_ctx ticket; }; -extern int radeon_cs_finish_pages(struct radeon_cs_parser *p); -extern u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx); +static inline u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx) +{ + struct radeon_cs_chunk *ibc = p->chunk_ib; + + if (ibc->kdata) + return ibc->kdata[idx]; + return p->ib.ptr[idx]; +} + struct radeon_cs_packet { unsigned idx; @@ -918,6 +1158,10 @@ struct radeon_wb { #define R600_WB_IH_WPTR_OFFSET 2048 #define CAYMAN_WB_DMA1_RPTR_OFFSET 2304 #define R600_WB_EVENT_OFFSET 3072 +#define CIK_WB_CP1_WPTR_OFFSET 3328 +#define CIK_WB_CP2_WPTR_OFFSET 3584 +#define R600_WB_DMA_RING_TEST_OFFSET 3588 +#define CAYMAN_WB_DMA1_RING_TEST_OFFSET 3592 /** * struct radeon_pm - power management datas @@ -942,6 +1186,7 @@ struct radeon_wb { enum radeon_pm_method { PM_METHOD_PROFILE, PM_METHOD_DYNPM, + PM_METHOD_DPM, }; enum radeon_dynpm_state { @@ -967,11 +1212,24 @@ enum radeon_voltage_type { }; enum radeon_pm_state_type { + /* not used for dpm */ POWER_STATE_TYPE_DEFAULT, POWER_STATE_TYPE_POWERSAVE, + /* user selectable states */ POWER_STATE_TYPE_BATTERY, POWER_STATE_TYPE_BALANCED, POWER_STATE_TYPE_PERFORMANCE, + /* internal states */ + POWER_STATE_TYPE_INTERNAL_UVD, + POWER_STATE_TYPE_INTERNAL_UVD_SD, + POWER_STATE_TYPE_INTERNAL_UVD_HD, + POWER_STATE_TYPE_INTERNAL_UVD_HD2, + POWER_STATE_TYPE_INTERNAL_UVD_MVC, + POWER_STATE_TYPE_INTERNAL_BOOT, + POWER_STATE_TYPE_INTERNAL_THERMAL, + POWER_STATE_TYPE_INTERNAL_ACPI, + POWER_STATE_TYPE_INTERNAL_ULV, + POWER_STATE_TYPE_INTERNAL_3DPERF, }; enum radeon_pm_profile_type { @@ -1000,12 +1258,18 @@ struct radeon_pm_profile { enum radeon_int_thermal_type { THERMAL_TYPE_NONE, + THERMAL_TYPE_EXTERNAL, + THERMAL_TYPE_EXTERNAL_GPIO, THERMAL_TYPE_RV6XX, THERMAL_TYPE_RV770, + THERMAL_TYPE_ADT7473_WITH_INTERNAL, THERMAL_TYPE_EVERGREEN, THERMAL_TYPE_SUMO, THERMAL_TYPE_NI, THERMAL_TYPE_SI, + THERMAL_TYPE_EMC2103_WITH_INTERNAL, + THERMAL_TYPE_CI, + THERMAL_TYPE_KV, }; struct radeon_voltage { @@ -1059,10 +1323,289 @@ struct radeon_power_state { */ #define RADEON_MODE_OVERCLOCK_MARGIN 500 /* 5 MHz */ +enum radeon_dpm_auto_throttle_src { + RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, + RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL +}; + +enum radeon_dpm_event_src { + RADEON_DPM_EVENT_SRC_ANALOG = 0, + RADEON_DPM_EVENT_SRC_EXTERNAL = 1, + RADEON_DPM_EVENT_SRC_DIGITAL = 2, + RADEON_DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3, + RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4 +}; + +#define RADEON_MAX_VCE_LEVELS 6 + +enum radeon_vce_level { + RADEON_VCE_LEVEL_AC_ALL = 0, /* AC, All cases */ + RADEON_VCE_LEVEL_DC_EE = 1, /* DC, entropy encoding */ + RADEON_VCE_LEVEL_DC_LL_LOW = 2, /* DC, low latency queue, res <= 720 */ + RADEON_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */ + RADEON_VCE_LEVEL_DC_GP_LOW = 4, /* DC, general purpose queue, res <= 720 */ + RADEON_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */ +}; + +struct radeon_ps { + u32 caps; /* vbios flags */ + u32 class; /* vbios flags */ + u32 class2; /* vbios flags */ + /* UVD clocks */ + u32 vclk; + u32 dclk; + /* VCE clocks */ + u32 evclk; + u32 ecclk; + bool vce_active; + enum radeon_vce_level vce_level; + /* asic priv */ + void *ps_priv; +}; + +struct radeon_dpm_thermal { + /* thermal interrupt work */ + struct work_struct work; + /* low temperature threshold */ + int min_temp; + /* high temperature threshold */ + int max_temp; + /* was interrupt low to high or high to low */ + bool high_to_low; +}; + +enum radeon_clk_action +{ + RADEON_SCLK_UP = 1, + RADEON_SCLK_DOWN +}; + +struct radeon_blacklist_clocks +{ + u32 sclk; + u32 mclk; + enum radeon_clk_action action; +}; + +struct radeon_clock_and_voltage_limits { + u32 sclk; + u32 mclk; + u16 vddc; + u16 vddci; +}; + +struct radeon_clock_array { + u32 count; + u32 *values; +}; + +struct radeon_clock_voltage_dependency_entry { + u32 clk; + u16 v; +}; + +struct radeon_clock_voltage_dependency_table { + u32 count; + struct radeon_clock_voltage_dependency_entry *entries; +}; + +union radeon_cac_leakage_entry { + struct { + u16 vddc; + u32 leakage; + }; + struct { + u16 vddc1; + u16 vddc2; + u16 vddc3; + }; +}; + +struct radeon_cac_leakage_table { + u32 count; + union radeon_cac_leakage_entry *entries; +}; + +struct radeon_phase_shedding_limits_entry { + u16 voltage; + u32 sclk; + u32 mclk; +}; + +struct radeon_phase_shedding_limits_table { + u32 count; + struct radeon_phase_shedding_limits_entry *entries; +}; + +struct radeon_uvd_clock_voltage_dependency_entry { + u32 vclk; + u32 dclk; + u16 v; +}; + +struct radeon_uvd_clock_voltage_dependency_table { + u8 count; + struct radeon_uvd_clock_voltage_dependency_entry *entries; +}; + +struct radeon_vce_clock_voltage_dependency_entry { + u32 ecclk; + u32 evclk; + u16 v; +}; + +struct radeon_vce_clock_voltage_dependency_table { + u8 count; + struct radeon_vce_clock_voltage_dependency_entry *entries; +}; + +struct radeon_ppm_table { + u8 ppm_design; + u16 cpu_core_number; + u32 platform_tdp; + u32 small_ac_platform_tdp; + u32 platform_tdc; + u32 small_ac_platform_tdc; + u32 apu_tdp; + u32 dgpu_tdp; + u32 dgpu_ulv_power; + u32 tj_max; +}; + +struct radeon_cac_tdp_table { + u16 tdp; + u16 configurable_tdp; + u16 tdc; + u16 battery_power_limit; + u16 small_power_limit; + u16 low_cac_leakage; + u16 high_cac_leakage; + u16 maximum_power_delivery_limit; +}; + +struct radeon_dpm_dynamic_state { + struct radeon_clock_voltage_dependency_table vddc_dependency_on_sclk; + struct radeon_clock_voltage_dependency_table vddci_dependency_on_mclk; + struct radeon_clock_voltage_dependency_table vddc_dependency_on_mclk; + struct radeon_clock_voltage_dependency_table mvdd_dependency_on_mclk; + struct radeon_clock_voltage_dependency_table vddc_dependency_on_dispclk; + struct radeon_uvd_clock_voltage_dependency_table uvd_clock_voltage_dependency_table; + struct radeon_vce_clock_voltage_dependency_table vce_clock_voltage_dependency_table; + struct radeon_clock_voltage_dependency_table samu_clock_voltage_dependency_table; + struct radeon_clock_voltage_dependency_table acp_clock_voltage_dependency_table; + struct radeon_clock_array valid_sclk_values; + struct radeon_clock_array valid_mclk_values; + struct radeon_clock_and_voltage_limits max_clock_voltage_on_dc; + struct radeon_clock_and_voltage_limits max_clock_voltage_on_ac; + u32 mclk_sclk_ratio; + u32 sclk_mclk_delta; + u16 vddc_vddci_delta; + u16 min_vddc_for_pcie_gen2; + struct radeon_cac_leakage_table cac_leakage_table; + struct radeon_phase_shedding_limits_table phase_shedding_limits_table; + struct radeon_ppm_table *ppm_table; + struct radeon_cac_tdp_table *cac_tdp_table; +}; + +struct radeon_dpm_fan { + u16 t_min; + u16 t_med; + u16 t_high; + u16 pwm_min; + u16 pwm_med; + u16 pwm_high; + u8 t_hyst; + u32 cycle_delay; + u16 t_max; + u8 control_mode; + u16 default_max_fan_pwm; + u16 default_fan_output_sensitivity; + u16 fan_output_sensitivity; + bool ucode_fan_control; +}; + +enum radeon_pcie_gen { + RADEON_PCIE_GEN1 = 0, + RADEON_PCIE_GEN2 = 1, + RADEON_PCIE_GEN3 = 2, + RADEON_PCIE_GEN_INVALID = 0xffff +}; + +enum radeon_dpm_forced_level { + RADEON_DPM_FORCED_LEVEL_AUTO = 0, + RADEON_DPM_FORCED_LEVEL_LOW = 1, + RADEON_DPM_FORCED_LEVEL_HIGH = 2, +}; + +struct radeon_vce_state { + /* vce clocks */ + u32 evclk; + u32 ecclk; + /* gpu clocks */ + u32 sclk; + u32 mclk; + u8 clk_idx; + u8 pstate; +}; + +struct radeon_dpm { + struct radeon_ps *ps; + /* number of valid power states */ + int num_ps; + /* current power state that is active */ + struct radeon_ps *current_ps; + /* requested power state */ + struct radeon_ps *requested_ps; + /* boot up power state */ + struct radeon_ps *boot_ps; + /* default uvd power state */ + struct radeon_ps *uvd_ps; + /* vce requirements */ + struct radeon_vce_state vce_states[RADEON_MAX_VCE_LEVELS]; + enum radeon_vce_level vce_level; + enum radeon_pm_state_type state; + enum radeon_pm_state_type user_state; + u32 platform_caps; + u32 voltage_response_time; + u32 backbias_response_time; + void *priv; + u32 new_active_crtcs; + int new_active_crtc_count; + u32 current_active_crtcs; + int current_active_crtc_count; + bool single_display; + struct radeon_dpm_dynamic_state dyn_state; + struct radeon_dpm_fan fan; + u32 tdp_limit; + u32 near_tdp_limit; + u32 near_tdp_limit_adjusted; + u32 sq_ramping_threshold; + u32 cac_leakage; + u16 tdp_od_limit; + u32 tdp_adjustment; + u16 load_line_slope; + bool power_control; + bool ac_power; + /* special states active */ + bool thermal_active; + bool uvd_active; + bool vce_active; + /* thermal handling */ + struct radeon_dpm_thermal thermal; + /* forced levels */ + enum radeon_dpm_forced_level forced_level; + /* track UVD streams */ + unsigned sd; + unsigned hd; +}; + +void radeon_dpm_enable_uvd(struct radeon_device *rdev, bool enable); +void radeon_dpm_enable_vce(struct radeon_device *rdev, bool enable); + struct radeon_pm { - struct rwlock mutex; + struct rwlock mutex; /* write locked while reprogramming mclk */ - struct rwlock mclk_lock; + struct rwlock mclk_lock; u32 active_crtcs; int active_crtc_count; int req_vblank; @@ -1112,18 +1655,120 @@ struct radeon_pm { /* internal thermal controller on rv6xx+ */ enum radeon_int_thermal_type int_thermal_type; struct device *int_hwmon_dev; + /* fan control parameters */ + bool no_fan; + u8 fan_pulses_per_revolution; + u8 fan_min_rpm; + u8 fan_max_rpm; + /* dpm */ + bool dpm_enabled; + bool sysfs_initialized; + struct radeon_dpm dpm; }; int radeon_pm_get_type_index(struct radeon_device *rdev, enum radeon_pm_state_type ps_type, int instance); +/* + * UVD + */ +#define RADEON_MAX_UVD_HANDLES 10 +#define RADEON_UVD_STACK_SIZE (1024*1024) +#define RADEON_UVD_HEAP_SIZE (1024*1024) -struct r600_audio { +struct radeon_uvd { + struct radeon_bo *vcpu_bo; + void *cpu_addr; + uint64_t gpu_addr; + atomic_t handles[RADEON_MAX_UVD_HANDLES]; + struct drm_file *filp[RADEON_MAX_UVD_HANDLES]; + unsigned img_size[RADEON_MAX_UVD_HANDLES]; + struct delayed_work idle_work; +}; + +int radeon_uvd_init(struct radeon_device *rdev); +void radeon_uvd_fini(struct radeon_device *rdev); +int radeon_uvd_suspend(struct radeon_device *rdev); +int radeon_uvd_resume(struct radeon_device *rdev); +int radeon_uvd_get_create_msg(struct radeon_device *rdev, int ring, + uint32_t handle, struct radeon_fence **fence); +int radeon_uvd_get_destroy_msg(struct radeon_device *rdev, int ring, + uint32_t handle, struct radeon_fence **fence); +void radeon_uvd_force_into_uvd_segment(struct radeon_bo *rbo, + uint32_t allowed_domains); +void radeon_uvd_free_handles(struct radeon_device *rdev, + struct drm_file *filp); +int radeon_uvd_cs_parse(struct radeon_cs_parser *parser); +void radeon_uvd_note_usage(struct radeon_device *rdev); +int radeon_uvd_calc_upll_dividers(struct radeon_device *rdev, + unsigned vclk, unsigned dclk, + unsigned vco_min, unsigned vco_max, + unsigned fb_factor, unsigned fb_mask, + unsigned pd_min, unsigned pd_max, + unsigned pd_even, + unsigned *optimal_fb_div, + unsigned *optimal_vclk_div, + unsigned *optimal_dclk_div); +int radeon_uvd_send_upll_ctlreq(struct radeon_device *rdev, + unsigned cg_upll_func_cntl); + +/* + * VCE + */ +#define RADEON_MAX_VCE_HANDLES 16 + +struct radeon_vce { + struct radeon_bo *vcpu_bo; + uint64_t gpu_addr; + unsigned fw_version; + unsigned fb_version; + atomic_t handles[RADEON_MAX_VCE_HANDLES]; + struct drm_file *filp[RADEON_MAX_VCE_HANDLES]; + unsigned img_size[RADEON_MAX_VCE_HANDLES]; + struct delayed_work idle_work; + uint32_t keyselect; +}; + +int radeon_vce_init(struct radeon_device *rdev); +void radeon_vce_fini(struct radeon_device *rdev); +int radeon_vce_suspend(struct radeon_device *rdev); +int radeon_vce_resume(struct radeon_device *rdev); +int radeon_vce_get_create_msg(struct radeon_device *rdev, int ring, + uint32_t handle, struct radeon_fence **fence); +int radeon_vce_get_destroy_msg(struct radeon_device *rdev, int ring, + uint32_t handle, struct radeon_fence **fence); +void radeon_vce_free_handles(struct radeon_device *rdev, struct drm_file *filp); +void radeon_vce_note_usage(struct radeon_device *rdev); +int radeon_vce_cs_reloc(struct radeon_cs_parser *p, int lo, int hi, unsigned size); +int radeon_vce_cs_parse(struct radeon_cs_parser *p); +bool radeon_vce_semaphore_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait); +void radeon_vce_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); +void radeon_vce_fence_emit(struct radeon_device *rdev, + struct radeon_fence *fence); +int radeon_vce_ring_test(struct radeon_device *rdev, struct radeon_ring *ring); +int radeon_vce_ib_test(struct radeon_device *rdev, struct radeon_ring *ring); + +struct r600_audio_pin { int channels; int rate; int bits_per_sample; u8 status_bits; u8 category_code; + u32 offset; + bool connected; + u32 id; +}; + +struct r600_audio { + bool enabled; + struct r600_audio_pin pin[RADEON_MAX_AFMT_BLOCKS]; + int num_pins; + struct radeon_audio_funcs *hdmi_funcs; + struct radeon_audio_funcs *dp_funcs; + struct radeon_audio_basic_funcs *funcs; }; /* @@ -1141,6 +1786,19 @@ void radeon_test_ring_sync(struct radeon_device *rdev, struct radeon_ring *cpB); void radeon_test_syncing(struct radeon_device *rdev); +/* + * MMU Notifier + */ +#if defined(CONFIG_MMU_NOTIFIER) +int radeon_mn_register(struct radeon_bo *bo, unsigned long addr); +void radeon_mn_unregister(struct radeon_bo *bo); +#else +static inline int radeon_mn_register(struct radeon_bo *bo, unsigned long addr) +{ + return -ENODEV; +} +static inline void radeon_mn_unregister(struct radeon_bo *bo) {} +#endif /* * Debugfs @@ -1155,6 +1813,36 @@ int radeon_debugfs_add_files(struct radeon_device *rdev, unsigned nfiles); int radeon_debugfs_fence_init(struct radeon_device *rdev); +/* + * ASIC ring specific functions. + */ +struct radeon_asic_ring { + /* ring read/write ptr handling */ + u32 (*get_rptr)(struct radeon_device *rdev, struct radeon_ring *ring); + u32 (*get_wptr)(struct radeon_device *rdev, struct radeon_ring *ring); + void (*set_wptr)(struct radeon_device *rdev, struct radeon_ring *ring); + + /* validating and patching of IBs */ + int (*ib_parse)(struct radeon_device *rdev, struct radeon_ib *ib); + int (*cs_parse)(struct radeon_cs_parser *p); + + /* command emmit functions */ + void (*ib_execute)(struct radeon_device *rdev, struct radeon_ib *ib); + void (*emit_fence)(struct radeon_device *rdev, struct radeon_fence *fence); + void (*hdp_flush)(struct radeon_device *rdev, struct radeon_ring *ring); + bool (*emit_semaphore)(struct radeon_device *rdev, struct radeon_ring *cp, + struct radeon_semaphore *semaphore, bool emit_wait); + void (*vm_flush)(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); + + /* testing functions */ + int (*ring_test)(struct radeon_device *rdev, struct radeon_ring *cp); + int (*ib_test)(struct radeon_device *rdev, struct radeon_ring *cp); + bool (*is_lockup)(struct radeon_device *rdev, struct radeon_ring *cp); + + /* deprecated */ + void (*ring_start)(struct radeon_device *rdev, struct radeon_ring *cp); +}; /* * ASIC specific functions. @@ -1166,45 +1854,46 @@ struct radeon_asic { int (*suspend)(struct radeon_device *rdev); void (*vga_set_state)(struct radeon_device *rdev, bool state); int (*asic_reset)(struct radeon_device *rdev); - /* ioctl hw specific callback. Some hw might want to perform special - * operation on specific ioctl. For instance on wait idle some hw - * might want to perform and HDP flush through MMIO as it seems that - * some R6XX/R7XX hw doesn't take HDP flush into account if programmed - * through ring. - */ - void (*ioctl_wait_idle)(struct radeon_device *rdev, struct radeon_bo *bo); + /* Flush the HDP cache via MMIO */ + void (*mmio_hdp_flush)(struct radeon_device *rdev); /* check if 3D engine is idle */ bool (*gui_idle)(struct radeon_device *rdev); /* wait for mc_idle */ int (*mc_wait_for_idle)(struct radeon_device *rdev); + /* get the reference clock */ + u32 (*get_xclk)(struct radeon_device *rdev); + /* get the gpu clock counter */ + uint64_t (*get_gpu_clock_counter)(struct radeon_device *rdev); + /* get register for info ioctl */ + int (*get_allowed_info_register)(struct radeon_device *rdev, u32 reg, u32 *val); /* gart */ struct { void (*tlb_flush)(struct radeon_device *rdev); - int (*set_page)(struct radeon_device *rdev, int i, uint64_t addr); + uint64_t (*get_page_entry)(uint64_t addr, uint32_t flags); + void (*set_page)(struct radeon_device *rdev, unsigned i, + uint64_t entry); } gart; struct { int (*init)(struct radeon_device *rdev); void (*fini)(struct radeon_device *rdev); - - u32 pt_ring_index; - void (*set_page)(struct radeon_device *rdev, uint64_t pe, - uint64_t addr, unsigned count, - uint32_t incr, uint32_t flags); + void (*copy_pages)(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, uint64_t src, + unsigned count); + void (*write_pages)(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags); + void (*set_pages)(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags); + void (*pad_ib)(struct radeon_ib *ib); } vm; /* ring specific callbacks */ - struct { - void (*ib_execute)(struct radeon_device *rdev, struct radeon_ib *ib); - int (*ib_parse)(struct radeon_device *rdev, struct radeon_ib *ib); - void (*emit_fence)(struct radeon_device *rdev, struct radeon_fence *fence); - void (*emit_semaphore)(struct radeon_device *rdev, struct radeon_ring *cp, - struct radeon_semaphore *semaphore, bool emit_wait); - int (*cs_parse)(struct radeon_cs_parser *p); - void (*ring_start)(struct radeon_device *rdev, struct radeon_ring *cp); - int (*ring_test)(struct radeon_device *rdev, struct radeon_ring *cp); - int (*ib_test)(struct radeon_device *rdev, struct radeon_ring *cp); - bool (*is_lockup)(struct radeon_device *rdev, struct radeon_ring *cp); - void (*vm_flush)(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); - } ring[RADEON_NUM_RINGS]; + struct radeon_asic_ring *ring[RADEON_NUM_RINGS]; /* irqs */ struct { int (*set)(struct radeon_device *rdev); @@ -1222,27 +1911,30 @@ struct radeon_asic { void (*set_backlight_level)(struct radeon_encoder *radeon_encoder, u8 level); /* get backlight level */ u8 (*get_backlight_level)(struct radeon_encoder *radeon_encoder); + /* audio callbacks */ + void (*hdmi_enable)(struct drm_encoder *encoder, bool enable); + void (*hdmi_setmode)(struct drm_encoder *encoder, struct drm_display_mode *mode); } display; /* copy functions for bo handling */ struct { - int (*blit)(struct radeon_device *rdev, - uint64_t src_offset, - uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence); + struct radeon_fence *(*blit)(struct radeon_device *rdev, + uint64_t src_offset, + uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); u32 blit_ring_index; - int (*dma)(struct radeon_device *rdev, - uint64_t src_offset, - uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence); + struct radeon_fence *(*dma)(struct radeon_device *rdev, + uint64_t src_offset, + uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); u32 dma_ring_index; /* method used for bo copy */ - int (*copy)(struct radeon_device *rdev, - uint64_t src_offset, - uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence); + struct radeon_fence *(*copy)(struct radeon_device *rdev, + uint64_t src_offset, + uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); /* ring used for bo copies */ u32 copy_ring_index; } copy; @@ -1260,7 +1952,7 @@ struct radeon_asic { bool (*sense)(struct radeon_device *rdev, enum radeon_hpd_id hpd); void (*set_polarity)(struct radeon_device *rdev, enum radeon_hpd_id hpd); } hpd; - /* power management */ + /* static power management */ struct { void (*misc)(struct radeon_device *rdev); void (*prepare)(struct radeon_device *rdev); @@ -1274,12 +1966,41 @@ struct radeon_asic { int (*get_pcie_lanes)(struct radeon_device *rdev); void (*set_pcie_lanes)(struct radeon_device *rdev, int lanes); void (*set_clock_gating)(struct radeon_device *rdev, int enable); + int (*set_uvd_clocks)(struct radeon_device *rdev, u32 vclk, u32 dclk); + int (*set_vce_clocks)(struct radeon_device *rdev, u32 evclk, u32 ecclk); + int (*get_temperature)(struct radeon_device *rdev); } pm; + /* dynamic power management */ + struct { + int (*init)(struct radeon_device *rdev); + void (*setup_asic)(struct radeon_device *rdev); + int (*enable)(struct radeon_device *rdev); + int (*late_enable)(struct radeon_device *rdev); + void (*disable)(struct radeon_device *rdev); + int (*pre_set_power_state)(struct radeon_device *rdev); + int (*set_power_state)(struct radeon_device *rdev); + void (*post_set_power_state)(struct radeon_device *rdev); + void (*display_configuration_changed)(struct radeon_device *rdev); + void (*fini)(struct radeon_device *rdev); + u32 (*get_sclk)(struct radeon_device *rdev, bool low); + u32 (*get_mclk)(struct radeon_device *rdev, bool low); + void (*print_power_state)(struct radeon_device *rdev, struct radeon_ps *ps); + void (*debugfs_print_current_performance_level)(struct radeon_device *rdev, struct seq_file *m); + int (*force_performance_level)(struct radeon_device *rdev, enum radeon_dpm_forced_level level); + bool (*vblank_too_short)(struct radeon_device *rdev); + void (*powergate_uvd)(struct radeon_device *rdev, bool gate); + void (*enable_bapm)(struct radeon_device *rdev, bool enable); + void (*fan_ctrl_set_mode)(struct radeon_device *rdev, u32 mode); + u32 (*fan_ctrl_get_mode)(struct radeon_device *rdev); + int (*set_fan_speed_percent)(struct radeon_device *rdev, u32 speed); + int (*get_fan_speed_percent)(struct radeon_device *rdev, u32 *speed); + u32 (*get_current_sclk)(struct radeon_device *rdev); + u32 (*get_current_mclk)(struct radeon_device *rdev); + } dpm; /* pageflipping */ struct { - void (*pre_page_flip)(struct radeon_device *rdev, int crtc); - u32 (*page_flip)(struct radeon_device *rdev, int crtc, u64 crtc_base); - void (*post_page_flip)(struct radeon_device *rdev, int crtc); + void (*page_flip)(struct radeon_device *rdev, int crtc, u64 crtc_base); + bool (*page_flip_pending)(struct radeon_device *rdev, int crtc); } pflip; }; @@ -1318,6 +2039,7 @@ struct r600_asic { unsigned tiling_group_size; unsigned tile_config; unsigned backend_map; + unsigned active_simds; }; struct rv770_asic { @@ -1343,6 +2065,7 @@ struct rv770_asic { unsigned tiling_group_size; unsigned tile_config; unsigned backend_map; + unsigned active_simds; }; struct evergreen_asic { @@ -1369,6 +2092,7 @@ struct evergreen_asic { unsigned tiling_group_size; unsigned tile_config; unsigned backend_map; + unsigned active_simds; }; struct cayman_asic { @@ -1407,6 +2131,7 @@ struct cayman_asic { unsigned multi_gpu_tile_size; unsigned tile_config; + unsigned active_simds; }; struct si_asic { @@ -1436,6 +2161,40 @@ struct si_asic { unsigned multi_gpu_tile_size; unsigned tile_config; + uint32_t tile_mode_array[32]; + uint32_t active_cus; +}; + +struct cik_asic { + unsigned max_shader_engines; + unsigned max_tile_pipes; + unsigned max_cu_per_sh; + unsigned max_sh_per_se; + unsigned max_backends_per_se; + unsigned max_texture_channel_caches; + unsigned max_gprs; + unsigned max_gs_threads; + unsigned max_hw_contexts; + unsigned sc_prim_fifo_size_frontend; + unsigned sc_prim_fifo_size_backend; + unsigned sc_hiz_tile_fifo_size; + unsigned sc_earlyz_tile_fifo_size; + + unsigned num_tile_pipes; + unsigned backend_enable_mask; + unsigned backend_disable_mask_per_asic; + unsigned backend_map; + unsigned num_texture_channel_caches; + unsigned mem_max_burst_length_bytes; + unsigned mem_row_size_in_kb; + unsigned shader_engine_tile_size; + unsigned num_gpus; + unsigned multi_gpu_tile_size; + + unsigned tile_config; + uint32_t tile_mode_array[32]; + uint32_t macrotile_mode_array[16]; + uint32_t active_cus; }; union radeon_asic_config { @@ -1446,6 +2205,7 @@ union radeon_asic_config { struct evergreen_asic evergreen; struct cayman_asic cayman; struct si_asic si; + struct cik_asic cik; }; /* @@ -1462,6 +2222,8 @@ int radeon_gem_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); int radeon_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); int radeon_gem_pin_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int radeon_gem_unpin_ioctl(struct drm_device *dev, void *data, @@ -1480,6 +2242,8 @@ int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); int radeon_gem_va_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int radeon_gem_op_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); int radeon_gem_set_tiling_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); @@ -1551,9 +2315,11 @@ typedef uint32_t (*radeon_rreg_t)(struct radeon_device*, uint32_t); typedef void (*radeon_wreg_t)(struct radeon_device*, uint32_t, uint32_t); struct radeon_device { - struct device dev; + struct device self; + struct device *dev; struct drm_device *ddev; struct pci_dev *pdev; + struct rwlock exclusive_lock; pci_chipset_tag_t pc; pcitag_t pa_tag; @@ -1579,8 +2345,6 @@ struct radeon_device { bus_space_handle_t memh; #endif - struct rwlock exclusive_lock; - unsigned long fb_aper_offset; unsigned long fb_aper_size; @@ -1599,11 +2363,33 @@ struct radeon_device { uint16_t bios_header_start; struct radeon_bo *stollen_vga_memory; /* Register mmio */ - bus_addr_t rmmio_base; - bus_size_t rmmio_size; + resource_size_t rmmio_base; + resource_size_t rmmio_size; /* protects concurrent MM_INDEX/DATA based register access */ spinlock_t mmio_idx_lock; - bus_space_handle_t rmmio; + /* protects concurrent SMC based register access */ + spinlock_t smc_idx_lock; + /* protects concurrent PLL register access */ + spinlock_t pll_idx_lock; + /* protects concurrent MC register access */ + spinlock_t mc_idx_lock; + /* protects concurrent PCIE register access */ + spinlock_t pcie_idx_lock; + /* protects concurrent PCIE_PORT register access */ + spinlock_t pciep_idx_lock; + /* protects concurrent PIF register access */ + spinlock_t pif_idx_lock; + /* protects concurrent CG register access */ + spinlock_t cg_idx_lock; + /* protects concurrent UVD register access */ + spinlock_t uvd_idx_lock; + /* protects concurrent RCU register access */ + spinlock_t rcu_idx_lock; + /* protects concurrent DIDT register access */ + spinlock_t didt_idx_lock; + /* protects concurrent ENDPOINT (audio) register access */ + spinlock_t end_idx_lock; + bus_space_handle_t rmmio_bsh; radeon_rreg_t mc_rreg; radeon_wreg_t mc_wreg; radeon_rreg_t pll_rreg; @@ -1613,16 +2399,18 @@ struct radeon_device { radeon_wreg_t pciep_wreg; /* io port */ bus_space_handle_t rio_mem; - bus_size_t rio_mem_size; + resource_size_t rio_mem_size; struct radeon_clock clock; struct radeon_mc mc; struct radeon_gart gart; struct radeon_mode_info mode_info; struct radeon_scratch scratch; + struct radeon_doorbell doorbell; struct radeon_mman mman; struct radeon_fence_driver fence_drv[RADEON_NUM_RINGS]; wait_queue_head_t fence_queue; - struct rwlock ring_lock; + unsigned fence_context; + struct rwlock ring_lock; struct radeon_ring ring[RADEON_NUM_RINGS]; bool ib_pool_ready; struct radeon_sa_manager ring_tmp_bo; @@ -1630,6 +2418,8 @@ struct radeon_device { struct radeon_asic *asic; struct radeon_gem gem; struct radeon_pm pm; + struct radeon_uvd uvd; + struct radeon_vce vce; uint32_t bios_scratch[RADEON_BIOS_NUM_SCRATCH]; struct radeon_wb wb; struct radeon_dummy_page dummy_page; @@ -1637,31 +2427,34 @@ struct radeon_device { bool suspend; bool need_dma32; bool accel_working; + bool fastfb_working; /* IGP feature*/ + bool needs_reset, in_reset; struct radeon_surface_reg surface_regs[RADEON_GEM_MAX_SURFACES]; - u_char *me_fw; /* all family ME firmware */ - size_t me_fw_size; - u_char *pfp_fw; /* r6/700 PFP firmware */ - size_t pfp_fw_size; - u_char *rlc_fw; /* r6/700 RLC firmware */ - size_t rlc_fw_size; - u_char *mc_fw; /* NI MC firmware */ - size_t mc_fw_size; - u_char *ce_fw; /* SI CE firmware */ - size_t ce_fw_size; - struct r600_blit r600_blit; + const struct firmware *me_fw; /* all family ME firmware */ + const struct firmware *pfp_fw; /* r6/700 PFP firmware */ + const struct firmware *rlc_fw; /* r6/700 RLC firmware */ + const struct firmware *mc_fw; /* NI MC firmware */ + const struct firmware *ce_fw; /* SI CE firmware */ + const struct firmware *mec_fw; /* CIK MEC firmware */ + const struct firmware *mec2_fw; /* KV MEC2 firmware */ + const struct firmware *sdma_fw; /* CIK SDMA firmware */ + const struct firmware *smc_fw; /* SMC firmware */ + const struct firmware *uvd_fw; /* UVD firmware */ + const struct firmware *vce_fw; /* VCE firmware */ + bool new_fw; struct r600_vram_scratch vram_scratch; int msi_enabled; /* msi enabled */ struct r600_ih ih; /* r6/700 interrupt ring */ - struct si_rlc rlc; - struct task hotplug_task; - struct task audio_task; + struct radeon_rlc rlc; + struct radeon_mec mec; + struct delayed_work hotplug_work; + struct work_struct dp_work; + struct work_struct audio_work; int num_crtc; /* number of crtcs */ struct rwlock dc_hw_i2c_mutex; /* display controller hw i2c mutex */ - bool audio_enabled; - struct r600_audio audio_status; /* audio stuff */ -#ifdef notyet + bool has_uvd; + struct r600_audio audio; /* audio stuff */ struct notifier_block acpi_nb; -#endif /* only one userspace can use Hyperz features or CMASK at a time */ struct drm_file *hyperz_filp; struct drm_file *cmask_filp; @@ -1672,41 +2465,102 @@ struct radeon_device { unsigned debugfs_count; /* virtual memory */ struct radeon_vm_manager vm_manager; - struct rwlock gpu_clock_mutex; + struct rwlock gpu_clock_mutex; + /* memory stats */ + atomic64_t vram_usage; + atomic64_t gtt_usage; + atomic64_t num_bytes_moved; + atomic_t gpu_reset_counter; /* ACPI interface */ struct radeon_atif atif; struct radeon_atcs atcs; + /* srbm instance registers */ + struct rwlock srbm_mutex; + /* GRBM index mutex. Protects concurrents access to GRBM index */ + struct rwlock grbm_idx_mutex; + /* clock, powergating flags */ + u32 cg_flags; + u32 pg_flags; + +#ifdef __linux__ + struct dev_pm_domain vga_pm_domain; +#endif + bool have_disp_power_ref; + u32 px_quirk_flags; + + /* tracking pinned memory */ + u64 vram_pin_size; + u64 gart_pin_size; + + /* amdkfd interface */ + struct kfd_dev *kfd; + + struct rwlock mn_lock; + DECLARE_HASHTABLE(mn_hash, 7); }; +bool radeon_is_px(struct drm_device *dev); int radeon_device_init(struct radeon_device *rdev, - struct drm_device *ddev); + struct drm_device *ddev, + struct pci_dev *pdev, + uint32_t flags); void radeon_device_fini(struct radeon_device *rdev); int radeon_gpu_wait_for_idle(struct radeon_device *rdev); -uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg, - bool always_indirect); -void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v, - bool always_indirect); +#define RADEON_MIN_MMIO_SIZE 0x10000 + +uint32_t r100_mm_rreg_slow(struct radeon_device *rdev, uint32_t reg); +void r100_mm_wreg_slow(struct radeon_device *rdev, uint32_t reg, uint32_t v); +static inline uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg, + bool always_indirect) +{ + /* The mmio size is 64kb at minimum. Allows the if to be optimized out. */ + if ((reg < rdev->rmmio_size || reg < RADEON_MIN_MMIO_SIZE) && !always_indirect) + return bus_space_read_4(rdev->memt, rdev->rmmio_bsh, reg); + else + return r100_mm_rreg_slow(rdev, reg); +} +static inline void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v, + bool always_indirect) +{ + if ((reg < rdev->rmmio_size || reg < RADEON_MIN_MMIO_SIZE) && !always_indirect) + bus_space_write_4(rdev->memt, rdev->rmmio_bsh, reg, v); + else + r100_mm_wreg_slow(rdev, reg, v); +} + u32 r100_io_rreg(struct radeon_device *rdev, u32 reg); void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v); +u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index); +void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v); + /* * Cast helper */ -#define to_radeon_fence(p) ((struct radeon_fence *)(p)) +extern const struct fence_ops radeon_fence_ops; + +static inline struct radeon_fence *to_radeon_fence(struct fence *f) +{ + struct radeon_fence *__f = container_of(f, struct radeon_fence, base); + + if (__f->base.ops == &radeon_fence_ops) + return __f; + + return NULL; +} /* * Registers read & write functions. */ #define RREG8(reg) \ - bus_space_read_1(rdev->memt, rdev->rmmio, (reg)) + bus_space_read_1(rdev->memt, rdev->rmmio_bsh, (reg)) #define WREG8(reg, v) \ - bus_space_write_1(rdev->memt, rdev->rmmio, (reg), (v)) + bus_space_write_1(rdev->memt, rdev->rmmio_bsh, (reg), (v)) #define RREG16(reg) \ - bus_space_read_2(rdev->memt, rdev->rmmio, (reg)) + bus_space_read_2(rdev->memt, rdev->rmmio_bsh, (reg)) #define WREG16(reg, v) \ - bus_space_write_2(rdev->memt, rdev->rmmio, (reg), (v)) - + bus_space_write_2(rdev->memt, rdev->rmmio_bsh, (reg), (v)) #define RREG32(reg) r100_mm_rreg(rdev, (reg), false) #define RREG32_IDX(reg) r100_mm_rreg(rdev, (reg), true) #define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", r100_mm_rreg(rdev, (reg), false)) @@ -1720,8 +2574,22 @@ void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v); #define WREG32_MC(reg, v) rdev->mc_wreg(rdev, (reg), (v)) #define RREG32_PCIE(reg) rv370_pcie_rreg(rdev, (reg)) #define WREG32_PCIE(reg, v) rv370_pcie_wreg(rdev, (reg), (v)) -#define RREG32_PCIE_P(reg) rdev->pciep_rreg(rdev, (reg)) -#define WREG32_PCIE_P(reg, v) rdev->pciep_wreg(rdev, (reg), (v)) +#define RREG32_PCIE_PORT(reg) rdev->pciep_rreg(rdev, (reg)) +#define WREG32_PCIE_PORT(reg, v) rdev->pciep_wreg(rdev, (reg), (v)) +#define RREG32_SMC(reg) tn_smc_rreg(rdev, (reg)) +#define WREG32_SMC(reg, v) tn_smc_wreg(rdev, (reg), (v)) +#define RREG32_RCU(reg) r600_rcu_rreg(rdev, (reg)) +#define WREG32_RCU(reg, v) r600_rcu_wreg(rdev, (reg), (v)) +#define RREG32_CG(reg) eg_cg_rreg(rdev, (reg)) +#define WREG32_CG(reg, v) eg_cg_wreg(rdev, (reg), (v)) +#define RREG32_PIF_PHY0(reg) eg_pif_phy0_rreg(rdev, (reg)) +#define WREG32_PIF_PHY0(reg, v) eg_pif_phy0_wreg(rdev, (reg), (v)) +#define RREG32_PIF_PHY1(reg) eg_pif_phy1_rreg(rdev, (reg)) +#define WREG32_PIF_PHY1(reg, v) eg_pif_phy1_wreg(rdev, (reg), (v)) +#define RREG32_UVD_CTX(reg) r600_uvd_ctx_rreg(rdev, (reg)) +#define WREG32_UVD_CTX(reg, v) r600_uvd_ctx_wreg(rdev, (reg), (v)) +#define RREG32_DIDT(reg) cik_didt_rreg(rdev, (reg)) +#define WREG32_DIDT(reg, v) cik_didt_wreg(rdev, (reg), (v)) #define WREG32_P(reg, val, mask) \ do { \ uint32_t tmp_ = RREG32(reg); \ @@ -1729,6 +2597,8 @@ void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v); tmp_ |= ((val) & ~(mask)); \ WREG32(reg, tmp_); \ } while (0) +#define WREG32_AND(reg, and) WREG32_P(reg, 0, and) +#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or)) #define WREG32_PLL_P(reg, val, mask) \ do { \ uint32_t tmp_ = RREG32_PLL(reg); \ @@ -1736,27 +2606,44 @@ void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v); tmp_ |= ((val) & ~(mask)); \ WREG32_PLL(reg, tmp_); \ } while (0) +#define WREG32_SMC_P(reg, val, mask) \ + do { \ + uint32_t tmp_ = RREG32_SMC(reg); \ + tmp_ &= (mask); \ + tmp_ |= ((val) & ~(mask)); \ + WREG32_SMC(reg, tmp_); \ + } while (0) #define DREG32_SYS(sqf, rdev, reg) seq_printf((sqf), #reg " : 0x%08X\n", r100_mm_rreg((rdev), (reg), false)) #define RREG32_IO(reg) r100_io_rreg(rdev, (reg)) #define WREG32_IO(reg, v) r100_io_wreg(rdev, (reg), (v)) +#define RDOORBELL32(index) cik_mm_rdoorbell(rdev, (index)) +#define WDOORBELL32(index, v) cik_mm_wdoorbell(rdev, (index), (v)) + /* - * Indirect registers accessor + * Indirect registers accessors. + * They used to be inlined, but this increases code size by ~65 kbytes. + * Since each performs a pair of MMIO ops + * within a spin_lock_irqsave/spin_unlock_irqrestore region, + * the cost of call+ret is almost negligible. MMIO and locking + * costs several dozens of cycles each at best, call+ret is ~5 cycles. */ -static inline uint32_t rv370_pcie_rreg(struct radeon_device *rdev, uint32_t reg) -{ - uint32_t r; - - WREG32(RADEON_PCIE_INDEX, ((reg) & rdev->pcie_reg_mask)); - r = RREG32(RADEON_PCIE_DATA); - return r; -} - -static inline void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v) -{ - WREG32(RADEON_PCIE_INDEX, ((reg) & rdev->pcie_reg_mask)); - WREG32(RADEON_PCIE_DATA, (v)); -} +uint32_t rv370_pcie_rreg(struct radeon_device *rdev, uint32_t reg); +void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); +u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg); +void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v); +u32 r600_rcu_rreg(struct radeon_device *rdev, u32 reg); +void r600_rcu_wreg(struct radeon_device *rdev, u32 reg, u32 v); +u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg); +void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v); +u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg); +void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v); +u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg); +void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v); +u32 r600_uvd_ctx_rreg(struct radeon_device *rdev, u32 reg); +void r600_uvd_ctx_wreg(struct radeon_device *rdev, u32 reg, u32 v); +u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg); +void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v); void r100_pll_errata_after_index(struct radeon_device *rdev); @@ -1804,6 +2691,22 @@ void r100_pll_errata_after_index(struct radeon_device *rdev); #define ASIC_IS_DCE6(rdev) ((rdev->family >= CHIP_ARUBA)) #define ASIC_IS_DCE61(rdev) ((rdev->family >= CHIP_ARUBA) && \ (rdev->flags & RADEON_IS_IGP)) +#define ASIC_IS_DCE64(rdev) ((rdev->family == CHIP_OLAND)) +#define ASIC_IS_NODCE(rdev) ((rdev->family == CHIP_HAINAN)) +#define ASIC_IS_DCE8(rdev) ((rdev->family >= CHIP_BONAIRE)) +#define ASIC_IS_DCE81(rdev) ((rdev->family == CHIP_KAVERI)) +#define ASIC_IS_DCE82(rdev) ((rdev->family == CHIP_BONAIRE)) +#define ASIC_IS_DCE83(rdev) ((rdev->family == CHIP_KABINI) || \ + (rdev->family == CHIP_MULLINS)) + +#define ASIC_IS_LOMBOK(rdev) ((rdev->ddev->pdev->device == 0x6849) || \ + (rdev->ddev->pdev->device == 0x6850) || \ + (rdev->ddev->pdev->device == 0x6858) || \ + (rdev->ddev->pdev->device == 0x6859) || \ + (rdev->ddev->pdev->device == 0x6840) || \ + (rdev->ddev->pdev->device == 0x6841) || \ + (rdev->ddev->pdev->device == 0x6842) || \ + (rdev->ddev->pdev->device == 0x6843)) /* * BIOS helpers. @@ -1821,18 +2724,25 @@ void radeon_atombios_fini(struct radeon_device *rdev); /* * RING helpers. */ -#if !defined(DRM_DEBUG_CODE) || DRM_DEBUG_CODE == 0 + +/** + * radeon_ring_write - write a value to the ring + * + * @ring: radeon_ring structure holding ring information + * @v: dword (dw) value to write + * + * Write a value to the requested ring buffer (all asics). + */ static inline void radeon_ring_write(struct radeon_ring *ring, uint32_t v) { + if (ring->count_dw <= 0) + DRM_ERROR("radeon: writing more dwords to the ring than expected!\n"); + ring->ring[ring->wptr++] = v; ring->wptr &= ring->ptr_mask; ring->count_dw--; ring->ring_free_dw--; } -#else -/* With debugging this is just too big to inline */ -void radeon_ring_write(struct radeon_ring *ring, uint32_t v); -#endif /* * ASICs macro. @@ -1841,31 +2751,40 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v); #define radeon_fini(rdev) (rdev)->asic->fini((rdev)) #define radeon_resume(rdev) (rdev)->asic->resume((rdev)) #define radeon_suspend(rdev) (rdev)->asic->suspend((rdev)) -#define radeon_cs_parse(rdev, r, p) (rdev)->asic->ring[(r)].cs_parse((p)) +#define radeon_cs_parse(rdev, r, p) (rdev)->asic->ring[(r)]->cs_parse((p)) #define radeon_vga_set_state(rdev, state) (rdev)->asic->vga_set_state((rdev), (state)) #define radeon_asic_reset(rdev) (rdev)->asic->asic_reset((rdev)) #define radeon_gart_tlb_flush(rdev) (rdev)->asic->gart.tlb_flush((rdev)) -#define radeon_gart_set_page(rdev, i, p) (rdev)->asic->gart.set_page((rdev), (i), (p)) +#define radeon_gart_get_page_entry(a, f) (rdev)->asic->gart.get_page_entry((a), (f)) +#define radeon_gart_set_page(rdev, i, e) (rdev)->asic->gart.set_page((rdev), (i), (e)) #define radeon_asic_vm_init(rdev) (rdev)->asic->vm.init((rdev)) #define radeon_asic_vm_fini(rdev) (rdev)->asic->vm.fini((rdev)) -#define radeon_asic_vm_set_page(rdev, pe, addr, count, incr, flags) ((rdev)->asic->vm.set_page((rdev), (pe), (addr), (count), (incr), (flags))) -#define radeon_ring_start(rdev, r, cp) (rdev)->asic->ring[(r)].ring_start((rdev), (cp)) -#define radeon_ring_test(rdev, r, cp) (rdev)->asic->ring[(r)].ring_test((rdev), (cp)) -#define radeon_ib_test(rdev, r, cp) (rdev)->asic->ring[(r)].ib_test((rdev), (cp)) -#define radeon_ring_ib_execute(rdev, r, ib) (rdev)->asic->ring[(r)].ib_execute((rdev), (ib)) -#define radeon_ring_ib_parse(rdev, r, ib) (rdev)->asic->ring[(r)].ib_parse((rdev), (ib)) -#define radeon_ring_is_lockup(rdev, r, cp) (rdev)->asic->ring[(r)].is_lockup((rdev), (cp)) -#define radeon_ring_vm_flush(rdev, r, vm) (rdev)->asic->ring[(r)].vm_flush((rdev), (r), (vm)) +#define radeon_asic_vm_copy_pages(rdev, ib, pe, src, count) ((rdev)->asic->vm.copy_pages((rdev), (ib), (pe), (src), (count))) +#define radeon_asic_vm_write_pages(rdev, ib, pe, addr, count, incr, flags) ((rdev)->asic->vm.write_pages((rdev), (ib), (pe), (addr), (count), (incr), (flags))) +#define radeon_asic_vm_set_pages(rdev, ib, pe, addr, count, incr, flags) ((rdev)->asic->vm.set_pages((rdev), (ib), (pe), (addr), (count), (incr), (flags))) +#define radeon_asic_vm_pad_ib(rdev, ib) ((rdev)->asic->vm.pad_ib((ib))) +#define radeon_ring_start(rdev, r, cp) (rdev)->asic->ring[(r)]->ring_start((rdev), (cp)) +#define radeon_ring_test(rdev, r, cp) (rdev)->asic->ring[(r)]->ring_test((rdev), (cp)) +#define radeon_ib_test(rdev, r, cp) (rdev)->asic->ring[(r)]->ib_test((rdev), (cp)) +#define radeon_ring_ib_execute(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_execute((rdev), (ib)) +#define radeon_ring_ib_parse(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_parse((rdev), (ib)) +#define radeon_ring_is_lockup(rdev, r, cp) (rdev)->asic->ring[(r)]->is_lockup((rdev), (cp)) +#define radeon_ring_vm_flush(rdev, r, vm_id, pd_addr) (rdev)->asic->ring[(r)->idx]->vm_flush((rdev), (r), (vm_id), (pd_addr)) +#define radeon_ring_get_rptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_rptr((rdev), (r)) +#define radeon_ring_get_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_wptr((rdev), (r)) +#define radeon_ring_set_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->set_wptr((rdev), (r)) #define radeon_irq_set(rdev) (rdev)->asic->irq.set((rdev)) #define radeon_irq_process(rdev) (rdev)->asic->irq.process((rdev)) #define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->display.get_vblank_counter((rdev), (crtc)) #define radeon_set_backlight_level(rdev, e, l) (rdev)->asic->display.set_backlight_level((e), (l)) #define radeon_get_backlight_level(rdev, e) (rdev)->asic->display.get_backlight_level((e)) -#define radeon_fence_ring_emit(rdev, r, fence) (rdev)->asic->ring[(r)].emit_fence((rdev), (fence)) -#define radeon_semaphore_ring_emit(rdev, r, cp, semaphore, emit_wait) (rdev)->asic->ring[(r)].emit_semaphore((rdev), (cp), (semaphore), (emit_wait)) -#define radeon_copy_blit(rdev, s, d, np, f) (rdev)->asic->copy.blit((rdev), (s), (d), (np), (f)) -#define radeon_copy_dma(rdev, s, d, np, f) (rdev)->asic->copy.dma((rdev), (s), (d), (np), (f)) -#define radeon_copy(rdev, s, d, np, f) (rdev)->asic->copy.copy((rdev), (s), (d), (np), (f)) +#define radeon_hdmi_enable(rdev, e, b) (rdev)->asic->display.hdmi_enable((e), (b)) +#define radeon_hdmi_setmode(rdev, e, m) (rdev)->asic->display.hdmi_setmode((e), (m)) +#define radeon_fence_ring_emit(rdev, r, fence) (rdev)->asic->ring[(r)]->emit_fence((rdev), (fence)) +#define radeon_semaphore_ring_emit(rdev, r, cp, semaphore, emit_wait) (rdev)->asic->ring[(r)]->emit_semaphore((rdev), (cp), (semaphore), (emit_wait)) +#define radeon_copy_blit(rdev, s, d, np, resv) (rdev)->asic->copy.blit((rdev), (s), (d), (np), (resv)) +#define radeon_copy_dma(rdev, s, d, np, resv) (rdev)->asic->copy.dma((rdev), (s), (d), (np), (resv)) +#define radeon_copy(rdev, s, d, np, resv) (rdev)->asic->copy.copy((rdev), (s), (d), (np), (resv)) #define radeon_copy_blit_ring_index(rdev) (rdev)->asic->copy.blit_ring_index #define radeon_copy_dma_ring_index(rdev) (rdev)->asic->copy.dma_ring_index #define radeon_copy_ring_index(rdev) (rdev)->asic->copy.copy_ring_index @@ -1876,6 +2795,9 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v); #define radeon_get_pcie_lanes(rdev) (rdev)->asic->pm.get_pcie_lanes((rdev)) #define radeon_set_pcie_lanes(rdev, l) (rdev)->asic->pm.set_pcie_lanes((rdev), (l)) #define radeon_set_clock_gating(rdev, e) (rdev)->asic->pm.set_clock_gating((rdev), (e)) +#define radeon_set_uvd_clocks(rdev, v, d) (rdev)->asic->pm.set_uvd_clocks((rdev), (v), (d)) +#define radeon_set_vce_clocks(rdev, ev, ec) (rdev)->asic->pm.set_vce_clocks((rdev), (ev), (ec)) +#define radeon_get_temperature(rdev) (rdev)->asic->pm.get_temperature((rdev)) #define radeon_set_surface_reg(rdev, r, f, p, o, s) ((rdev)->asic->surface.set_reg((rdev), (r), (f), (p), (o), (s))) #define radeon_clear_surface_reg(rdev, r) ((rdev)->asic->surface.clear_reg((rdev), (r))) #define radeon_bandwidth_update(rdev) (rdev)->asic->display.bandwidth_update((rdev)) @@ -1889,15 +2811,39 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v); #define radeon_pm_finish(rdev) (rdev)->asic->pm.finish((rdev)) #define radeon_pm_init_profile(rdev) (rdev)->asic->pm.init_profile((rdev)) #define radeon_pm_get_dynpm_state(rdev) (rdev)->asic->pm.get_dynpm_state((rdev)) -#define radeon_pre_page_flip(rdev, crtc) (rdev)->asic->pflip.pre_page_flip((rdev), (crtc)) #define radeon_page_flip(rdev, crtc, base) (rdev)->asic->pflip.page_flip((rdev), (crtc), (base)) -#define radeon_post_page_flip(rdev, crtc) (rdev)->asic->pflip.post_page_flip((rdev), (crtc)) +#define radeon_page_flip_pending(rdev, crtc) (rdev)->asic->pflip.page_flip_pending((rdev), (crtc)) #define radeon_wait_for_vblank(rdev, crtc) (rdev)->asic->display.wait_for_vblank((rdev), (crtc)) #define radeon_mc_wait_for_idle(rdev) (rdev)->asic->mc_wait_for_idle((rdev)) +#define radeon_get_xclk(rdev) (rdev)->asic->get_xclk((rdev)) +#define radeon_get_gpu_clock_counter(rdev) (rdev)->asic->get_gpu_clock_counter((rdev)) +#define radeon_get_allowed_info_register(rdev, r, v) (rdev)->asic->get_allowed_info_register((rdev), (r), (v)) +#define radeon_dpm_init(rdev) rdev->asic->dpm.init((rdev)) +#define radeon_dpm_setup_asic(rdev) rdev->asic->dpm.setup_asic((rdev)) +#define radeon_dpm_enable(rdev) rdev->asic->dpm.enable((rdev)) +#define radeon_dpm_late_enable(rdev) rdev->asic->dpm.late_enable((rdev)) +#define radeon_dpm_disable(rdev) rdev->asic->dpm.disable((rdev)) +#define radeon_dpm_pre_set_power_state(rdev) rdev->asic->dpm.pre_set_power_state((rdev)) +#define radeon_dpm_set_power_state(rdev) rdev->asic->dpm.set_power_state((rdev)) +#define radeon_dpm_post_set_power_state(rdev) rdev->asic->dpm.post_set_power_state((rdev)) +#define radeon_dpm_display_configuration_changed(rdev) rdev->asic->dpm.display_configuration_changed((rdev)) +#define radeon_dpm_fini(rdev) rdev->asic->dpm.fini((rdev)) +#define radeon_dpm_get_sclk(rdev, l) rdev->asic->dpm.get_sclk((rdev), (l)) +#define radeon_dpm_get_mclk(rdev, l) rdev->asic->dpm.get_mclk((rdev), (l)) +#define radeon_dpm_print_power_state(rdev, ps) rdev->asic->dpm.print_power_state((rdev), (ps)) +#define radeon_dpm_debugfs_print_current_performance_level(rdev, m) rdev->asic->dpm.debugfs_print_current_performance_level((rdev), (m)) +#define radeon_dpm_force_performance_level(rdev, l) rdev->asic->dpm.force_performance_level((rdev), (l)) +#define radeon_dpm_vblank_too_short(rdev) rdev->asic->dpm.vblank_too_short((rdev)) +#define radeon_dpm_powergate_uvd(rdev, g) rdev->asic->dpm.powergate_uvd((rdev), (g)) +#define radeon_dpm_enable_bapm(rdev, e) rdev->asic->dpm.enable_bapm((rdev), (e)) +#define radeon_dpm_get_current_sclk(rdev) rdev->asic->dpm.get_current_sclk((rdev)) +#define radeon_dpm_get_current_mclk(rdev) rdev->asic->dpm.get_current_mclk((rdev)) /* Common functions */ /* AGP */ extern int radeon_gpu_reset(struct radeon_device *rdev); +extern void radeon_pci_config_reset(struct radeon_device *rdev); +extern void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung); extern void radeon_agp_disable(struct radeon_device *rdev); extern int radeon_modeset_init(struct radeon_device *rdev); extern void radeon_modeset_fini(struct radeon_device *rdev); @@ -1915,32 +2861,47 @@ extern void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enabl extern void radeon_atom_set_clock_gating(struct radeon_device *rdev, int enable); extern void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain); extern bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo); +extern int radeon_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, + uint32_t flags); +extern bool radeon_ttm_tt_has_userptr(struct ttm_tt *ttm); +extern bool radeon_ttm_tt_is_readonly(struct ttm_tt *ttm); extern void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 base); extern void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc); -extern int radeon_resume_kms(struct drm_device *dev); -extern int radeon_suspend_kms(struct drm_device *dev); +extern int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon); +extern int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon); extern void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size); -extern struct uvm_object *radeon_mmap(struct drm_device *, voff_t, vsize_t); +extern void radeon_program_register_sequence(struct radeon_device *rdev, + const u32 *registers, + const u32 array_size); /* * vm */ int radeon_vm_manager_init(struct radeon_device *rdev); void radeon_vm_manager_fini(struct radeon_device *rdev); -void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm); +int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm); void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm); -int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm); -void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm); +struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev, + struct radeon_vm *vm, + struct list_head *head); struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, struct radeon_vm *vm, int ring); +void radeon_vm_flush(struct radeon_device *rdev, + struct radeon_vm *vm, + int ring, struct radeon_fence *fence); void radeon_vm_fence(struct radeon_device *rdev, struct radeon_vm *vm, struct radeon_fence *fence); uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr); -int radeon_vm_bo_update_pte(struct radeon_device *rdev, - struct radeon_vm *vm, - struct radeon_bo *bo, - struct ttm_mem_reg *mem); +int radeon_vm_update_page_directory(struct radeon_device *rdev, + struct radeon_vm *vm); +int radeon_vm_clear_freed(struct radeon_device *rdev, + struct radeon_vm *vm); +int radeon_vm_clear_invalids(struct radeon_device *rdev, + struct radeon_vm *vm); +int radeon_vm_bo_update(struct radeon_device *rdev, + struct radeon_bo_va *bo_va, + struct ttm_mem_reg *mem); void radeon_vm_bo_invalidate(struct radeon_device *rdev, struct radeon_bo *bo); struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm, @@ -1952,11 +2913,19 @@ int radeon_vm_bo_set_addr(struct radeon_device *rdev, struct radeon_bo_va *bo_va, uint64_t offset, uint32_t flags); -int radeon_vm_bo_rmv(struct radeon_device *rdev, - struct radeon_bo_va *bo_va); +void radeon_vm_bo_rmv(struct radeon_device *rdev, + struct radeon_bo_va *bo_va); /* audio */ -void r600_audio_update_hdmi(void *arg1); +void r600_audio_update_hdmi(struct work_struct *work); +struct r600_audio_pin *r600_audio_get_pin(struct radeon_device *rdev); +struct r600_audio_pin *dce6_audio_get_pin(struct radeon_device *rdev); +void r600_audio_enable(struct radeon_device *rdev, + struct r600_audio_pin *pin, + u8 enable_mask); +void dce6_audio_enable(struct radeon_device *rdev, + struct r600_audio_pin *pin, + u8 enable_mask); /* * R600 vram scratch functions @@ -1993,9 +2962,6 @@ struct radeon_hdmi_acr { extern struct radeon_hdmi_acr r600_hdmi_acr(uint32_t clock); -extern void r600_hdmi_enable(struct drm_encoder *encoder); -extern void r600_hdmi_disable(struct drm_encoder *encoder); -extern void r600_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode); extern u32 r6xx_remap_render_backend(struct radeon_device *rdev, u32 tiling_pipe_num, u32 max_rb_num, @@ -2006,8 +2972,6 @@ extern u32 r6xx_remap_render_backend(struct radeon_device *rdev, * evergreen functions used by radeon_encoder.c */ -extern void evergreen_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode); - extern int ni_init_microcode(struct radeon_device *rdev); extern int ni_mc_load_microcode(struct radeon_device *rdev); @@ -2015,11 +2979,28 @@ extern int ni_mc_load_microcode(struct radeon_device *rdev); #if defined(CONFIG_ACPI) extern int radeon_acpi_init(struct radeon_device *rdev); extern void radeon_acpi_fini(struct radeon_device *rdev); +extern bool radeon_acpi_is_pcie_performance_request_supported(struct radeon_device *rdev); +extern int radeon_acpi_pcie_performance_request(struct radeon_device *rdev, + u8 perf_req, bool advertise); +extern int radeon_acpi_pcie_notify_device_ready(struct radeon_device *rdev); #else static inline int radeon_acpi_init(struct radeon_device *rdev) { return 0; } static inline void radeon_acpi_fini(struct radeon_device *rdev) { } #endif +int radeon_cs_packet_parse(struct radeon_cs_parser *p, + struct radeon_cs_packet *pkt, + unsigned idx); +bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p); +void radeon_cs_dump_packet(struct radeon_cs_parser *p, + struct radeon_cs_packet *pkt); +int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p, + struct radeon_bo_list **cs_reloc, + int nomm); +int r600_cs_common_vline_parse(struct radeon_cs_parser *p, + uint32_t *vline_start_end, + uint32_t *vline_status); + #include "radeon_object.h" #endif diff --git a/sys/dev/pci/drm/radeon/radeon_acpi.c b/sys/dev/pci/drm/radeon/radeon_acpi.c index 260b5650976..4b0858c56f1 100644 --- a/sys/dev/pci/drm/radeon/radeon_acpi.c +++ b/sys/dev/pci/drm/radeon/radeon_acpi.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_acpi.c,v 1.4 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2012 Advanced Micro Devices, Inc. * @@ -22,6 +21,7 @@ * */ +#include #include #include #include "radeon.h" @@ -69,6 +69,22 @@ struct atcs_verify_interface { u32 function_bits; /* supported functions bit vector */ } __packed; +#define ATCS_VALID_FLAGS_MASK 0x3 + +struct atcs_pref_req_input { + u16 size; /* structure size in bytes (includes size field) */ + u16 client_id; /* client id (bit 2-0: func num, 7-3: dev num, 15-8: bus num) */ + u16 valid_flags_mask; /* valid flags mask */ + u16 flags; /* flags */ + u8 req_type; /* request type */ + u8 perf_req; /* performance request */ +} __packed; + +struct atcs_pref_req_output { + u16 size; /* structure size in bytes (includes size field) */ + u8 ret_val; /* return value */ +} __packed; + /* Call the ATIF method */ /** @@ -344,7 +360,7 @@ int radeon_atif_handler(struct radeon_device *rdev, return NOTIFY_DONE; /* Check pending SBIOS requests */ - handle = DEVICE_ACPI_HANDLE(&rdev->pdev->dev); + handle = ACPI_HANDLE(&rdev->pdev->dev); count = radeon_atif_get_sbios_requests(handle, &req); if (count <= 0) @@ -496,6 +512,135 @@ out: return err; } +/** + * radeon_acpi_is_pcie_performance_request_supported + * + * @rdev: radeon_device pointer + * + * Check if the ATCS pcie_perf_req and pcie_dev_rdy methods + * are supported (all asics). + * returns true if supported, false if not. + */ +bool radeon_acpi_is_pcie_performance_request_supported(struct radeon_device *rdev) +{ + struct radeon_atcs *atcs = &rdev->atcs; + + if (atcs->functions.pcie_perf_req && atcs->functions.pcie_dev_rdy) + return true; + + return false; +} + +/** + * radeon_acpi_pcie_notify_device_ready + * + * @rdev: radeon_device pointer + * + * Executes the PCIE_DEVICE_READY_NOTIFICATION method + * (all asics). + * returns 0 on success, error on failure. + */ +int radeon_acpi_pcie_notify_device_ready(struct radeon_device *rdev) +{ + acpi_handle handle; + union acpi_object *info; + struct radeon_atcs *atcs = &rdev->atcs; + + /* Get the device handle */ + handle = ACPI_HANDLE(&rdev->pdev->dev); + if (!handle) + return -EINVAL; + + if (!atcs->functions.pcie_dev_rdy) + return -EINVAL; + + info = radeon_atcs_call(handle, ATCS_FUNCTION_PCIE_DEVICE_READY_NOTIFICATION, NULL); + if (!info) + return -EIO; + + kfree(info); + + return 0; +} + +/** + * radeon_acpi_pcie_performance_request + * + * @rdev: radeon_device pointer + * @perf_req: requested perf level (pcie gen speed) + * @advertise: set advertise caps flag if set + * + * Executes the PCIE_PERFORMANCE_REQUEST method to + * change the pcie gen speed (all asics). + * returns 0 on success, error on failure. + */ +int radeon_acpi_pcie_performance_request(struct radeon_device *rdev, + u8 perf_req, bool advertise) +{ + acpi_handle handle; + union acpi_object *info; + struct radeon_atcs *atcs = &rdev->atcs; + struct atcs_pref_req_input atcs_input; + struct atcs_pref_req_output atcs_output; + struct acpi_buffer params; + size_t size; + u32 retry = 3; + + /* Get the device handle */ + handle = ACPI_HANDLE(&rdev->pdev->dev); + if (!handle) + return -EINVAL; + + if (!atcs->functions.pcie_perf_req) + return -EINVAL; + + atcs_input.size = sizeof(struct atcs_pref_req_input); + /* client id (bit 2-0: func num, 7-3: dev num, 15-8: bus num) */ + atcs_input.client_id = rdev->pdev->devfn | (rdev->pdev->bus->number << 8); + atcs_input.valid_flags_mask = ATCS_VALID_FLAGS_MASK; + atcs_input.flags = ATCS_WAIT_FOR_COMPLETION; + if (advertise) + atcs_input.flags |= ATCS_ADVERTISE_CAPS; + atcs_input.req_type = ATCS_PCIE_LINK_SPEED; + atcs_input.perf_req = perf_req; + + params.length = sizeof(struct atcs_pref_req_input); + params.pointer = &atcs_input; + + while (retry--) { + info = radeon_atcs_call(handle, ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST, ¶ms); + if (!info) + return -EIO; + + memset(&atcs_output, 0, sizeof(atcs_output)); + + size = *(u16 *) info->buffer.pointer; + if (size < 3) { + DRM_INFO("ATCS buffer is too small: %zu\n", size); + kfree(info); + return -EINVAL; + } + size = min(sizeof(atcs_output), size); + + memcpy(&atcs_output, info->buffer.pointer, size); + + kfree(info); + + switch (atcs_output.ret_val) { + case ATCS_REQUEST_REFUSED: + default: + return -EINVAL; + case ATCS_REQUEST_COMPLETE: + return 0; + case ATCS_REQUEST_IN_PROGRESS: + udelay(10); + break; + } + } + + return 0; +} + /** * radeon_acpi_event - handle notify events * @@ -545,7 +690,7 @@ int radeon_acpi_init(struct radeon_device *rdev) int ret; /* Get the device handle */ - handle = DEVICE_ACPI_HANDLE(&rdev->pdev->dev); + handle = ACPI_HANDLE(&rdev->pdev->dev); /* No need to proceed if we're sure that ATIF is not supported */ if (!ASIC_IS_AVIVO(rdev) || !rdev->bios || !handle) diff --git a/sys/dev/pci/drm/radeon/radeon_acpi.h b/sys/dev/pci/drm/radeon/radeon_acpi.h index 2487f07b52c..be4af76f213 100644 --- a/sys/dev/pci/drm/radeon/radeon_acpi.h +++ b/sys/dev/pci/drm/radeon/radeon_acpi.h @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_acpi.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2012 Advanced Micro Devices, Inc. * diff --git a/sys/dev/pci/drm/radeon/radeon_agp.c b/sys/dev/pci/drm/radeon/radeon_agp.c index a0be2dd4932..71192113c6f 100644 --- a/sys/dev/pci/drm/radeon/radeon_agp.c +++ b/sys/dev/pci/drm/radeon/radeon_agp.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_agp.c,v 1.7 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Red Hat Inc. * Copyright 2009 Jerome Glisse. @@ -55,6 +54,9 @@ static struct radeon_agpmode_quirk radeon_agpmode_quirk_list[] = { /* Intel 82855PM host bridge / Mobility 9600 M10 RV350 Needs AGPMode 1 (lp #195051) */ { PCI_VENDOR_ID_INTEL, 0x3340, PCI_VENDOR_ID_ATI, 0x4e50, PCI_VENDOR_ID_IBM, 0x0550, 1}, + /* Intel 82855PM host bridge / RV250/M9 GL [Mobility FireGL 9000/Radeon 9000] needs AGPMode 1 (Thinkpad T40p) */ + { PCI_VENDOR_ID_INTEL, 0x3340, PCI_VENDOR_ID_ATI, 0x4c66, + PCI_VENDOR_ID_IBM, 0x054d, 1}, /* Intel 82855PM host bridge / Mobility M7 needs AGPMode 1 */ { PCI_VENDOR_ID_INTEL, 0x3340, PCI_VENDOR_ID_ATI, 0x4c57, PCI_VENDOR_ID_IBM, 0x0530, 1}, @@ -118,9 +120,6 @@ static struct radeon_agpmode_quirk radeon_agpmode_quirk_list[] = { /* ATI Host Bridge / RV280 [M9+] Needs AGPMode 1 (phoronix forum) */ { PCI_VENDOR_ID_ATI, 0xcbb2, PCI_VENDOR_ID_ATI, 0x5c61, PCI_VENDOR_ID_SONY, 0x8175, 1}, - /* HP Host Bridge / R300 [FireGL X1] Needs AGPMode 2 (fdo #7770) */ - { PCI_VENDOR_ID_HP, 0x122e, PCI_VENDOR_ID_ATI, 0x4e47, - PCI_VENDOR_ID_ATI, 0x0152, 2}, { 0, 0, 0, 0, 0, 0, 0 }, }; #endif diff --git a/sys/dev/pci/drm/radeon/radeon_asic.c b/sys/dev/pci/drm/radeon/radeon_asic.c index b6b24a37dc1..d2d6a347846 100644 --- a/sys/dev/pci/drm/radeon/radeon_asic.c +++ b/sys/dev/pci/drm/radeon/radeon_asic.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_asic.c,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -120,12 +119,25 @@ static void radeon_register_accessor_init(struct radeon_device *rdev) rdev->mc_rreg = &rs600_mc_rreg; rdev->mc_wreg = &rs600_mc_wreg; } - if (rdev->family >= CHIP_R600) { + if (rdev->family == CHIP_RS780 || rdev->family == CHIP_RS880) { + rdev->mc_rreg = &rs780_mc_rreg; + rdev->mc_wreg = &rs780_mc_wreg; + } + + if (rdev->family >= CHIP_BONAIRE) { + rdev->pciep_rreg = &cik_pciep_rreg; + rdev->pciep_wreg = &cik_pciep_wreg; + } else if (rdev->family >= CHIP_R600) { rdev->pciep_rreg = &r600_pciep_rreg; rdev->pciep_wreg = &r600_pciep_wreg; } } +static int radeon_invalid_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val) +{ + return -EINVAL; +} /* helper to disable agp */ /** @@ -149,11 +161,13 @@ void radeon_agp_disable(struct radeon_device *rdev) DRM_INFO("Forcing AGP to PCIE mode\n"); rdev->flags |= RADEON_IS_PCIE; rdev->asic->gart.tlb_flush = &rv370_pcie_gart_tlb_flush; + rdev->asic->gart.get_page_entry = &rv370_pcie_gart_get_page_entry; rdev->asic->gart.set_page = &rv370_pcie_gart_set_page; } else { DRM_INFO("Forcing AGP to PCI mode\n"); rdev->flags |= RADEON_IS_PCI; rdev->asic->gart.tlb_flush = &r100_pci_gart_tlb_flush; + rdev->asic->gart.get_page_entry = &r100_pci_gart_get_page_entry; rdev->asic->gart.set_page = &r100_pci_gart_set_page; } rdev->mc.gtt_size = radeon_gart_size * 1024 * 1024; @@ -162,6 +176,21 @@ void radeon_agp_disable(struct radeon_device *rdev) /* * ASIC */ + +static struct radeon_asic_ring r100_gfx_ring = { + .ib_execute = &r100_ring_ib_execute, + .emit_fence = &r100_fence_ring_emit, + .emit_semaphore = &r100_semaphore_ring_emit, + .cs_parse = &r100_cs_parse, + .ring_start = &r100_ring_start, + .ring_test = &r100_ring_test, + .ib_test = &r100_ib_test, + .is_lockup = &r100_gpu_is_lockup, + .get_rptr = &r100_gfx_get_rptr, + .get_wptr = &r100_gfx_get_wptr, + .set_wptr = &r100_gfx_set_wptr, +}; + static struct radeon_asic r100_asic = { .init = &r100_init, .fini = &r100_fini, @@ -169,24 +198,17 @@ static struct radeon_asic r100_asic = { .resume = &r100_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &r100_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &r100_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &r100_pci_gart_tlb_flush, + .get_page_entry = &r100_pci_gart_get_page_entry, .set_page = &r100_pci_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r100_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r100_cs_parse, - .ring_start = &r100_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r100_gfx_ring }, .irq = { .set = &r100_irq_set, @@ -232,9 +254,8 @@ static struct radeon_asic r100_asic = { .set_clock_gating = &radeon_legacy_set_clock_gating, }, .pflip = { - .pre_page_flip = &r100_pre_page_flip, .page_flip = &r100_page_flip, - .post_page_flip = &r100_post_page_flip, + .page_flip_pending = &r100_page_flip_pending, }, }; @@ -245,24 +266,17 @@ static struct radeon_asic r200_asic = { .resume = &r100_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &r100_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &r100_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &r100_pci_gart_tlb_flush, + .get_page_entry = &r100_pci_gart_get_page_entry, .set_page = &r100_pci_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r100_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r100_cs_parse, - .ring_start = &r100_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r100_gfx_ring }, .irq = { .set = &r100_irq_set, @@ -308,12 +322,39 @@ static struct radeon_asic r200_asic = { .set_clock_gating = &radeon_legacy_set_clock_gating, }, .pflip = { - .pre_page_flip = &r100_pre_page_flip, .page_flip = &r100_page_flip, - .post_page_flip = &r100_post_page_flip, + .page_flip_pending = &r100_page_flip_pending, }, }; +static struct radeon_asic_ring r300_gfx_ring = { + .ib_execute = &r100_ring_ib_execute, + .emit_fence = &r300_fence_ring_emit, + .emit_semaphore = &r100_semaphore_ring_emit, + .cs_parse = &r300_cs_parse, + .ring_start = &r300_ring_start, + .ring_test = &r100_ring_test, + .ib_test = &r100_ib_test, + .is_lockup = &r100_gpu_is_lockup, + .get_rptr = &r100_gfx_get_rptr, + .get_wptr = &r100_gfx_get_wptr, + .set_wptr = &r100_gfx_set_wptr, +}; + +static struct radeon_asic_ring rv515_gfx_ring = { + .ib_execute = &r100_ring_ib_execute, + .emit_fence = &r300_fence_ring_emit, + .emit_semaphore = &r100_semaphore_ring_emit, + .cs_parse = &r300_cs_parse, + .ring_start = &rv515_ring_start, + .ring_test = &r100_ring_test, + .ib_test = &r100_ib_test, + .is_lockup = &r100_gpu_is_lockup, + .get_rptr = &r100_gfx_get_rptr, + .get_wptr = &r100_gfx_get_wptr, + .set_wptr = &r100_gfx_set_wptr, +}; + static struct radeon_asic r300_asic = { .init = &r300_init, .fini = &r300_fini, @@ -321,24 +362,17 @@ static struct radeon_asic r300_asic = { .resume = &r300_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &r300_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &r300_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &r100_pci_gart_tlb_flush, + .get_page_entry = &r100_pci_gart_get_page_entry, .set_page = &r100_pci_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r300_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r300_cs_parse, - .ring_start = &r300_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r300_gfx_ring }, .irq = { .set = &r100_irq_set, @@ -384,9 +418,8 @@ static struct radeon_asic r300_asic = { .set_clock_gating = &radeon_legacy_set_clock_gating, }, .pflip = { - .pre_page_flip = &r100_pre_page_flip, .page_flip = &r100_page_flip, - .post_page_flip = &r100_post_page_flip, + .page_flip_pending = &r100_page_flip_pending, }, }; @@ -397,24 +430,17 @@ static struct radeon_asic r300_asic_pcie = { .resume = &r300_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &r300_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &r300_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &rv370_pcie_gart_tlb_flush, + .get_page_entry = &rv370_pcie_gart_get_page_entry, .set_page = &rv370_pcie_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r300_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r300_cs_parse, - .ring_start = &r300_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r300_gfx_ring }, .irq = { .set = &r100_irq_set, @@ -460,9 +486,8 @@ static struct radeon_asic r300_asic_pcie = { .set_clock_gating = &radeon_legacy_set_clock_gating, }, .pflip = { - .pre_page_flip = &r100_pre_page_flip, .page_flip = &r100_page_flip, - .post_page_flip = &r100_post_page_flip, + .page_flip_pending = &r100_page_flip_pending, }, }; @@ -473,24 +498,17 @@ static struct radeon_asic r420_asic = { .resume = &r420_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &r300_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &r300_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &rv370_pcie_gart_tlb_flush, + .get_page_entry = &rv370_pcie_gart_get_page_entry, .set_page = &rv370_pcie_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r300_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r300_cs_parse, - .ring_start = &r300_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r300_gfx_ring }, .irq = { .set = &r100_irq_set, @@ -536,9 +554,8 @@ static struct radeon_asic r420_asic = { .set_clock_gating = &radeon_atom_set_clock_gating, }, .pflip = { - .pre_page_flip = &r100_pre_page_flip, .page_flip = &r100_page_flip, - .post_page_flip = &r100_post_page_flip, + .page_flip_pending = &r100_page_flip_pending, }, }; @@ -549,24 +566,17 @@ static struct radeon_asic rs400_asic = { .resume = &rs400_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &r300_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &rs400_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &rs400_gart_tlb_flush, + .get_page_entry = &rs400_gart_get_page_entry, .set_page = &rs400_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r300_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r300_cs_parse, - .ring_start = &r300_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r300_gfx_ring }, .irq = { .set = &r100_irq_set, @@ -612,9 +622,8 @@ static struct radeon_asic rs400_asic = { .set_clock_gating = &radeon_legacy_set_clock_gating, }, .pflip = { - .pre_page_flip = &r100_pre_page_flip, .page_flip = &r100_page_flip, - .post_page_flip = &r100_post_page_flip, + .page_flip_pending = &r100_page_flip_pending, }, }; @@ -625,24 +634,17 @@ static struct radeon_asic rs600_asic = { .resume = &rs600_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &rs600_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &rs600_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &rs600_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r300_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r300_cs_parse, - .ring_start = &r300_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r300_gfx_ring }, .irq = { .set = &rs600_irq_set, @@ -688,9 +690,8 @@ static struct radeon_asic rs600_asic = { .set_clock_gating = &radeon_atom_set_clock_gating, }, .pflip = { - .pre_page_flip = &rs600_pre_page_flip, .page_flip = &rs600_page_flip, - .post_page_flip = &rs600_post_page_flip, + .page_flip_pending = &rs600_page_flip_pending, }, }; @@ -701,24 +702,17 @@ static struct radeon_asic rs690_asic = { .resume = &rs690_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &rs600_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &rs690_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &rs400_gart_tlb_flush, + .get_page_entry = &rs400_gart_get_page_entry, .set_page = &rs400_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r300_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r300_cs_parse, - .ring_start = &r300_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r300_gfx_ring }, .irq = { .set = &rs600_irq_set, @@ -764,9 +758,8 @@ static struct radeon_asic rs690_asic = { .set_clock_gating = &radeon_atom_set_clock_gating, }, .pflip = { - .pre_page_flip = &rs600_pre_page_flip, .page_flip = &rs600_page_flip, - .post_page_flip = &rs600_post_page_flip, + .page_flip_pending = &rs600_page_flip_pending, }, }; @@ -777,24 +770,17 @@ static struct radeon_asic rv515_asic = { .resume = &rv515_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &rs600_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &rv515_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &rv370_pcie_gart_tlb_flush, + .get_page_entry = &rv370_pcie_gart_get_page_entry, .set_page = &rv370_pcie_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r300_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r300_cs_parse, - .ring_start = &rv515_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &rv515_gfx_ring }, .irq = { .set = &rs600_irq_set, @@ -840,9 +826,8 @@ static struct radeon_asic rv515_asic = { .set_clock_gating = &radeon_atom_set_clock_gating, }, .pflip = { - .pre_page_flip = &rs600_pre_page_flip, .page_flip = &rs600_page_flip, - .post_page_flip = &rs600_post_page_flip, + .page_flip_pending = &rs600_page_flip_pending, }, }; @@ -853,24 +838,17 @@ static struct radeon_asic r520_asic = { .resume = &r520_resume, .vga_set_state = &r100_vga_set_state, .asic_reset = &rs600_asic_reset, - .ioctl_wait_idle = NULL, + .mmio_hdp_flush = NULL, .gui_idle = &r100_gui_idle, .mc_wait_for_idle = &r520_mc_wait_for_idle, + .get_allowed_info_register = radeon_invalid_get_allowed_info_register, .gart = { .tlb_flush = &rv370_pcie_gart_tlb_flush, + .get_page_entry = &rv370_pcie_gart_get_page_entry, .set_page = &rv370_pcie_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r100_ring_ib_execute, - .emit_fence = &r300_fence_ring_emit, - .emit_semaphore = &r100_semaphore_ring_emit, - .cs_parse = &r300_cs_parse, - .ring_start = &rv515_ring_start, - .ring_test = &r100_ring_test, - .ib_test = &r100_ib_test, - .is_lockup = &r100_gpu_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &rv515_gfx_ring }, .irq = { .set = &rs600_irq_set, @@ -916,12 +894,37 @@ static struct radeon_asic r520_asic = { .set_clock_gating = &radeon_atom_set_clock_gating, }, .pflip = { - .pre_page_flip = &rs600_pre_page_flip, .page_flip = &rs600_page_flip, - .post_page_flip = &rs600_post_page_flip, + .page_flip_pending = &rs600_page_flip_pending, }, }; +static struct radeon_asic_ring r600_gfx_ring = { + .ib_execute = &r600_ring_ib_execute, + .emit_fence = &r600_fence_ring_emit, + .emit_semaphore = &r600_semaphore_ring_emit, + .cs_parse = &r600_cs_parse, + .ring_test = &r600_ring_test, + .ib_test = &r600_ib_test, + .is_lockup = &r600_gfx_is_lockup, + .get_rptr = &r600_gfx_get_rptr, + .get_wptr = &r600_gfx_get_wptr, + .set_wptr = &r600_gfx_set_wptr, +}; + +static struct radeon_asic_ring r600_dma_ring = { + .ib_execute = &r600_dma_ring_ib_execute, + .emit_fence = &r600_dma_fence_ring_emit, + .emit_semaphore = &r600_dma_semaphore_ring_emit, + .cs_parse = &r600_dma_cs_parse, + .ring_test = &r600_dma_ring_test, + .ib_test = &r600_dma_ib_test, + .is_lockup = &r600_dma_is_lockup, + .get_rptr = &r600_dma_get_rptr, + .get_wptr = &r600_dma_get_wptr, + .set_wptr = &r600_dma_set_wptr, +}; + static struct radeon_asic r600_asic = { .init = &r600_init, .fini = &r600_fini, @@ -929,32 +932,20 @@ static struct radeon_asic r600_asic = { .resume = &r600_resume, .vga_set_state = &r600_vga_set_state, .asic_reset = &r600_asic_reset, - .ioctl_wait_idle = r600_ioctl_wait_idle, + .mmio_hdp_flush = r600_mmio_hdp_flush, .gui_idle = &r600_gui_idle, .mc_wait_for_idle = &r600_mc_wait_for_idle, + .get_xclk = &r600_get_xclk, + .get_gpu_clock_counter = &r600_get_gpu_clock_counter, + .get_allowed_info_register = r600_get_allowed_info_register, .gart = { .tlb_flush = &r600_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r600_ring_ib_execute, - .emit_fence = &r600_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &r600_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &r600_gpu_is_lockup, - }, - [R600_RING_TYPE_DMA_INDEX] = { - .ib_execute = &r600_dma_ring_ib_execute, - .emit_fence = &r600_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &r600_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &r600_dma_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r600_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &r600_dma_ring, }, .irq = { .set = &r600_irq_set, @@ -968,12 +959,12 @@ static struct radeon_asic r600_asic = { .get_backlight_level = &atombios_get_backlight_level, }, .copy = { - .blit = &r600_copy_blit, + .blit = &r600_copy_cpdma, .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, .dma = &r600_copy_dma, .dma_ring_index = R600_RING_TYPE_DMA_INDEX, - .copy = &r600_copy_dma, - .copy_ring_index = R600_RING_TYPE_DMA_INDEX, + .copy = &r600_copy_cpdma, + .copy_ring_index = RADEON_RING_TYPE_GFX_INDEX, }, .surface = { .set_reg = r600_set_surface_reg, @@ -998,11 +989,117 @@ static struct radeon_asic r600_asic = { .get_pcie_lanes = &r600_get_pcie_lanes, .set_pcie_lanes = &r600_set_pcie_lanes, .set_clock_gating = NULL, + .get_temperature = &rv6xx_get_temp, }, .pflip = { - .pre_page_flip = &rs600_pre_page_flip, .page_flip = &rs600_page_flip, - .post_page_flip = &rs600_post_page_flip, + .page_flip_pending = &rs600_page_flip_pending, + }, +}; + +static struct radeon_asic_ring rv6xx_uvd_ring = { + .ib_execute = &uvd_v1_0_ib_execute, + .emit_fence = &uvd_v1_0_fence_emit, + .emit_semaphore = &uvd_v1_0_semaphore_emit, + .cs_parse = &radeon_uvd_cs_parse, + .ring_test = &uvd_v1_0_ring_test, + .ib_test = &uvd_v1_0_ib_test, + .is_lockup = &radeon_ring_test_lockup, + .get_rptr = &uvd_v1_0_get_rptr, + .get_wptr = &uvd_v1_0_get_wptr, + .set_wptr = &uvd_v1_0_set_wptr, +}; + +static struct radeon_asic rv6xx_asic = { + .init = &r600_init, + .fini = &r600_fini, + .suspend = &r600_suspend, + .resume = &r600_resume, + .vga_set_state = &r600_vga_set_state, + .asic_reset = &r600_asic_reset, + .mmio_hdp_flush = r600_mmio_hdp_flush, + .gui_idle = &r600_gui_idle, + .mc_wait_for_idle = &r600_mc_wait_for_idle, + .get_xclk = &r600_get_xclk, + .get_gpu_clock_counter = &r600_get_gpu_clock_counter, + .get_allowed_info_register = r600_get_allowed_info_register, + .gart = { + .tlb_flush = &r600_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, + .set_page = &rs600_gart_set_page, + }, + .ring = { + [RADEON_RING_TYPE_GFX_INDEX] = &r600_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &r600_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &rv6xx_uvd_ring, + }, + .irq = { + .set = &r600_irq_set, + .process = &r600_irq_process, + }, + .display = { + .bandwidth_update = &rv515_bandwidth_update, + .get_vblank_counter = &rs600_get_vblank_counter, + .wait_for_vblank = &avivo_wait_for_vblank, + .set_backlight_level = &atombios_set_backlight_level, + .get_backlight_level = &atombios_get_backlight_level, + }, + .copy = { + .blit = &r600_copy_cpdma, + .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, + .dma = &r600_copy_dma, + .dma_ring_index = R600_RING_TYPE_DMA_INDEX, + .copy = &r600_copy_cpdma, + .copy_ring_index = RADEON_RING_TYPE_GFX_INDEX, + }, + .surface = { + .set_reg = r600_set_surface_reg, + .clear_reg = r600_clear_surface_reg, + }, + .hpd = { + .init = &r600_hpd_init, + .fini = &r600_hpd_fini, + .sense = &r600_hpd_sense, + .set_polarity = &r600_hpd_set_polarity, + }, + .pm = { + .misc = &r600_pm_misc, + .prepare = &rs600_pm_prepare, + .finish = &rs600_pm_finish, + .init_profile = &r600_pm_init_profile, + .get_dynpm_state = &r600_pm_get_dynpm_state, + .get_engine_clock = &radeon_atom_get_engine_clock, + .set_engine_clock = &radeon_atom_set_engine_clock, + .get_memory_clock = &radeon_atom_get_memory_clock, + .set_memory_clock = &radeon_atom_set_memory_clock, + .get_pcie_lanes = &r600_get_pcie_lanes, + .set_pcie_lanes = &r600_set_pcie_lanes, + .set_clock_gating = NULL, + .get_temperature = &rv6xx_get_temp, + .set_uvd_clocks = &r600_set_uvd_clocks, + }, + .dpm = { + .init = &rv6xx_dpm_init, + .setup_asic = &rv6xx_setup_asic, + .enable = &rv6xx_dpm_enable, + .late_enable = &r600_dpm_late_enable, + .disable = &rv6xx_dpm_disable, + .pre_set_power_state = &r600_dpm_pre_set_power_state, + .set_power_state = &rv6xx_dpm_set_power_state, + .post_set_power_state = &r600_dpm_post_set_power_state, + .display_configuration_changed = &rv6xx_dpm_display_configuration_changed, + .fini = &rv6xx_dpm_fini, + .get_sclk = &rv6xx_dpm_get_sclk, + .get_mclk = &rv6xx_dpm_get_mclk, + .print_power_state = &rv6xx_dpm_print_power_state, + .debugfs_print_current_performance_level = &rv6xx_dpm_debugfs_print_current_performance_level, + .force_performance_level = &rv6xx_dpm_force_performance_level, + .get_current_sclk = &rv6xx_dpm_get_current_sclk, + .get_current_mclk = &rv6xx_dpm_get_current_mclk, + }, + .pflip = { + .page_flip = &rs600_page_flip, + .page_flip_pending = &rs600_page_flip_pending, }, }; @@ -1013,32 +1110,21 @@ static struct radeon_asic rs780_asic = { .resume = &r600_resume, .vga_set_state = &r600_vga_set_state, .asic_reset = &r600_asic_reset, - .ioctl_wait_idle = r600_ioctl_wait_idle, + .mmio_hdp_flush = r600_mmio_hdp_flush, .gui_idle = &r600_gui_idle, .mc_wait_for_idle = &r600_mc_wait_for_idle, + .get_xclk = &r600_get_xclk, + .get_gpu_clock_counter = &r600_get_gpu_clock_counter, + .get_allowed_info_register = r600_get_allowed_info_register, .gart = { .tlb_flush = &r600_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r600_ring_ib_execute, - .emit_fence = &r600_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &r600_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &r600_gpu_is_lockup, - }, - [R600_RING_TYPE_DMA_INDEX] = { - .ib_execute = &r600_dma_ring_ib_execute, - .emit_fence = &r600_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &r600_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &r600_dma_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r600_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &r600_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &rv6xx_uvd_ring, }, .irq = { .set = &r600_irq_set, @@ -1052,12 +1138,12 @@ static struct radeon_asic rs780_asic = { .get_backlight_level = &atombios_get_backlight_level, }, .copy = { - .blit = &r600_copy_blit, + .blit = &r600_copy_cpdma, .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, .dma = &r600_copy_dma, .dma_ring_index = R600_RING_TYPE_DMA_INDEX, - .copy = &r600_copy_dma, - .copy_ring_index = R600_RING_TYPE_DMA_INDEX, + .copy = &r600_copy_cpdma, + .copy_ring_index = RADEON_RING_TYPE_GFX_INDEX, }, .surface = { .set_reg = r600_set_surface_reg, @@ -1082,14 +1168,47 @@ static struct radeon_asic rs780_asic = { .get_pcie_lanes = NULL, .set_pcie_lanes = NULL, .set_clock_gating = NULL, + .get_temperature = &rv6xx_get_temp, + .set_uvd_clocks = &r600_set_uvd_clocks, + }, + .dpm = { + .init = &rs780_dpm_init, + .setup_asic = &rs780_dpm_setup_asic, + .enable = &rs780_dpm_enable, + .late_enable = &r600_dpm_late_enable, + .disable = &rs780_dpm_disable, + .pre_set_power_state = &r600_dpm_pre_set_power_state, + .set_power_state = &rs780_dpm_set_power_state, + .post_set_power_state = &r600_dpm_post_set_power_state, + .display_configuration_changed = &rs780_dpm_display_configuration_changed, + .fini = &rs780_dpm_fini, + .get_sclk = &rs780_dpm_get_sclk, + .get_mclk = &rs780_dpm_get_mclk, + .print_power_state = &rs780_dpm_print_power_state, + .debugfs_print_current_performance_level = &rs780_dpm_debugfs_print_current_performance_level, + .force_performance_level = &rs780_dpm_force_performance_level, + .get_current_sclk = &rs780_dpm_get_current_sclk, + .get_current_mclk = &rs780_dpm_get_current_mclk, }, .pflip = { - .pre_page_flip = &rs600_pre_page_flip, .page_flip = &rs600_page_flip, - .post_page_flip = &rs600_post_page_flip, + .page_flip_pending = &rs600_page_flip_pending, }, }; +static struct radeon_asic_ring rv770_uvd_ring = { + .ib_execute = &uvd_v1_0_ib_execute, + .emit_fence = &uvd_v2_2_fence_emit, + .emit_semaphore = &uvd_v2_2_semaphore_emit, + .cs_parse = &radeon_uvd_cs_parse, + .ring_test = &uvd_v1_0_ring_test, + .ib_test = &uvd_v1_0_ib_test, + .is_lockup = &radeon_ring_test_lockup, + .get_rptr = &uvd_v1_0_get_rptr, + .get_wptr = &uvd_v1_0_get_wptr, + .set_wptr = &uvd_v1_0_set_wptr, +}; + static struct radeon_asic rv770_asic = { .init = &rv770_init, .fini = &rv770_fini, @@ -1097,32 +1216,21 @@ static struct radeon_asic rv770_asic = { .resume = &rv770_resume, .asic_reset = &r600_asic_reset, .vga_set_state = &r600_vga_set_state, - .ioctl_wait_idle = r600_ioctl_wait_idle, + .mmio_hdp_flush = r600_mmio_hdp_flush, .gui_idle = &r600_gui_idle, .mc_wait_for_idle = &r600_mc_wait_for_idle, + .get_xclk = &rv770_get_xclk, + .get_gpu_clock_counter = &r600_get_gpu_clock_counter, + .get_allowed_info_register = r600_get_allowed_info_register, .gart = { .tlb_flush = &r600_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &r600_ring_ib_execute, - .emit_fence = &r600_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &r600_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &r600_gpu_is_lockup, - }, - [R600_RING_TYPE_DMA_INDEX] = { - .ib_execute = &r600_dma_ring_ib_execute, - .emit_fence = &r600_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &r600_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &r600_dma_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &r600_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &r600_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &rv770_uvd_ring, }, .irq = { .set = &r600_irq_set, @@ -1136,7 +1244,7 @@ static struct radeon_asic rv770_asic = { .get_backlight_level = &atombios_get_backlight_level, }, .copy = { - .blit = &r600_copy_blit, + .blit = &r600_copy_cpdma, .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, .dma = &rv770_copy_dma, .dma_ring_index = R600_RING_TYPE_DMA_INDEX, @@ -1166,14 +1274,61 @@ static struct radeon_asic rv770_asic = { .get_pcie_lanes = &r600_get_pcie_lanes, .set_pcie_lanes = &r600_set_pcie_lanes, .set_clock_gating = &radeon_atom_set_clock_gating, + .set_uvd_clocks = &rv770_set_uvd_clocks, + .get_temperature = &rv770_get_temp, + }, + .dpm = { + .init = &rv770_dpm_init, + .setup_asic = &rv770_dpm_setup_asic, + .enable = &rv770_dpm_enable, + .late_enable = &rv770_dpm_late_enable, + .disable = &rv770_dpm_disable, + .pre_set_power_state = &r600_dpm_pre_set_power_state, + .set_power_state = &rv770_dpm_set_power_state, + .post_set_power_state = &r600_dpm_post_set_power_state, + .display_configuration_changed = &rv770_dpm_display_configuration_changed, + .fini = &rv770_dpm_fini, + .get_sclk = &rv770_dpm_get_sclk, + .get_mclk = &rv770_dpm_get_mclk, + .print_power_state = &rv770_dpm_print_power_state, + .debugfs_print_current_performance_level = &rv770_dpm_debugfs_print_current_performance_level, + .force_performance_level = &rv770_dpm_force_performance_level, + .vblank_too_short = &rv770_dpm_vblank_too_short, + .get_current_sclk = &rv770_dpm_get_current_sclk, + .get_current_mclk = &rv770_dpm_get_current_mclk, }, .pflip = { - .pre_page_flip = &rs600_pre_page_flip, .page_flip = &rv770_page_flip, - .post_page_flip = &rs600_post_page_flip, + .page_flip_pending = &rv770_page_flip_pending, }, }; +static struct radeon_asic_ring evergreen_gfx_ring = { + .ib_execute = &evergreen_ring_ib_execute, + .emit_fence = &r600_fence_ring_emit, + .emit_semaphore = &r600_semaphore_ring_emit, + .cs_parse = &evergreen_cs_parse, + .ring_test = &r600_ring_test, + .ib_test = &r600_ib_test, + .is_lockup = &evergreen_gfx_is_lockup, + .get_rptr = &r600_gfx_get_rptr, + .get_wptr = &r600_gfx_get_wptr, + .set_wptr = &r600_gfx_set_wptr, +}; + +static struct radeon_asic_ring evergreen_dma_ring = { + .ib_execute = &evergreen_dma_ring_ib_execute, + .emit_fence = &evergreen_dma_fence_ring_emit, + .emit_semaphore = &r600_dma_semaphore_ring_emit, + .cs_parse = &evergreen_dma_cs_parse, + .ring_test = &r600_dma_ring_test, + .ib_test = &r600_dma_ib_test, + .is_lockup = &evergreen_dma_is_lockup, + .get_rptr = &r600_dma_get_rptr, + .get_wptr = &r600_dma_get_wptr, + .set_wptr = &r600_dma_set_wptr, +}; + static struct radeon_asic evergreen_asic = { .init = &evergreen_init, .fini = &evergreen_fini, @@ -1181,32 +1336,21 @@ static struct radeon_asic evergreen_asic = { .resume = &evergreen_resume, .asic_reset = &evergreen_asic_reset, .vga_set_state = &r600_vga_set_state, - .ioctl_wait_idle = r600_ioctl_wait_idle, + .mmio_hdp_flush = r600_mmio_hdp_flush, .gui_idle = &r600_gui_idle, .mc_wait_for_idle = &evergreen_mc_wait_for_idle, + .get_xclk = &rv770_get_xclk, + .get_gpu_clock_counter = &r600_get_gpu_clock_counter, + .get_allowed_info_register = evergreen_get_allowed_info_register, .gart = { .tlb_flush = &evergreen_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &evergreen_ring_ib_execute, - .emit_fence = &r600_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &evergreen_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &evergreen_gpu_is_lockup, - }, - [R600_RING_TYPE_DMA_INDEX] = { - .ib_execute = &evergreen_dma_ring_ib_execute, - .emit_fence = &evergreen_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &evergreen_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &r600_dma_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &evergreen_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &evergreen_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &rv770_uvd_ring, }, .irq = { .set = &evergreen_irq_set, @@ -1220,7 +1364,7 @@ static struct radeon_asic evergreen_asic = { .get_backlight_level = &atombios_get_backlight_level, }, .copy = { - .blit = &r600_copy_blit, + .blit = &r600_copy_cpdma, .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, .dma = &evergreen_copy_dma, .dma_ring_index = R600_RING_TYPE_DMA_INDEX, @@ -1250,11 +1394,32 @@ static struct radeon_asic evergreen_asic = { .get_pcie_lanes = &r600_get_pcie_lanes, .set_pcie_lanes = &r600_set_pcie_lanes, .set_clock_gating = NULL, + .set_uvd_clocks = &evergreen_set_uvd_clocks, + .get_temperature = &evergreen_get_temp, + }, + .dpm = { + .init = &cypress_dpm_init, + .setup_asic = &cypress_dpm_setup_asic, + .enable = &cypress_dpm_enable, + .late_enable = &rv770_dpm_late_enable, + .disable = &cypress_dpm_disable, + .pre_set_power_state = &r600_dpm_pre_set_power_state, + .set_power_state = &cypress_dpm_set_power_state, + .post_set_power_state = &r600_dpm_post_set_power_state, + .display_configuration_changed = &cypress_dpm_display_configuration_changed, + .fini = &cypress_dpm_fini, + .get_sclk = &rv770_dpm_get_sclk, + .get_mclk = &rv770_dpm_get_mclk, + .print_power_state = &rv770_dpm_print_power_state, + .debugfs_print_current_performance_level = &rv770_dpm_debugfs_print_current_performance_level, + .force_performance_level = &rv770_dpm_force_performance_level, + .vblank_too_short = &cypress_dpm_vblank_too_short, + .get_current_sclk = &rv770_dpm_get_current_sclk, + .get_current_mclk = &rv770_dpm_get_current_mclk, }, .pflip = { - .pre_page_flip = &evergreen_pre_page_flip, .page_flip = &evergreen_page_flip, - .post_page_flip = &evergreen_post_page_flip, + .page_flip_pending = &evergreen_page_flip_pending, }, }; @@ -1265,32 +1430,21 @@ static struct radeon_asic sumo_asic = { .resume = &evergreen_resume, .asic_reset = &evergreen_asic_reset, .vga_set_state = &r600_vga_set_state, - .ioctl_wait_idle = r600_ioctl_wait_idle, + .mmio_hdp_flush = r600_mmio_hdp_flush, .gui_idle = &r600_gui_idle, .mc_wait_for_idle = &evergreen_mc_wait_for_idle, + .get_xclk = &r600_get_xclk, + .get_gpu_clock_counter = &r600_get_gpu_clock_counter, + .get_allowed_info_register = evergreen_get_allowed_info_register, .gart = { .tlb_flush = &evergreen_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &evergreen_ring_ib_execute, - .emit_fence = &r600_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &evergreen_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &evergreen_gpu_is_lockup, - }, - [R600_RING_TYPE_DMA_INDEX] = { - .ib_execute = &evergreen_dma_ring_ib_execute, - .emit_fence = &evergreen_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &evergreen_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &r600_dma_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &evergreen_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &evergreen_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &rv770_uvd_ring, }, .irq = { .set = &evergreen_irq_set, @@ -1304,7 +1458,7 @@ static struct radeon_asic sumo_asic = { .get_backlight_level = &atombios_get_backlight_level, }, .copy = { - .blit = &r600_copy_blit, + .blit = &r600_copy_cpdma, .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, .dma = &evergreen_copy_dma, .dma_ring_index = R600_RING_TYPE_DMA_INDEX, @@ -1334,11 +1488,31 @@ static struct radeon_asic sumo_asic = { .get_pcie_lanes = NULL, .set_pcie_lanes = NULL, .set_clock_gating = NULL, + .set_uvd_clocks = &sumo_set_uvd_clocks, + .get_temperature = &sumo_get_temp, + }, + .dpm = { + .init = &sumo_dpm_init, + .setup_asic = &sumo_dpm_setup_asic, + .enable = &sumo_dpm_enable, + .late_enable = &sumo_dpm_late_enable, + .disable = &sumo_dpm_disable, + .pre_set_power_state = &sumo_dpm_pre_set_power_state, + .set_power_state = &sumo_dpm_set_power_state, + .post_set_power_state = &sumo_dpm_post_set_power_state, + .display_configuration_changed = &sumo_dpm_display_configuration_changed, + .fini = &sumo_dpm_fini, + .get_sclk = &sumo_dpm_get_sclk, + .get_mclk = &sumo_dpm_get_mclk, + .print_power_state = &sumo_dpm_print_power_state, + .debugfs_print_current_performance_level = &sumo_dpm_debugfs_print_current_performance_level, + .force_performance_level = &sumo_dpm_force_performance_level, + .get_current_sclk = &sumo_dpm_get_current_sclk, + .get_current_mclk = &sumo_dpm_get_current_mclk, }, .pflip = { - .pre_page_flip = &evergreen_pre_page_flip, .page_flip = &evergreen_page_flip, - .post_page_flip = &evergreen_post_page_flip, + .page_flip_pending = &evergreen_page_flip_pending, }, }; @@ -1349,32 +1523,21 @@ static struct radeon_asic btc_asic = { .resume = &evergreen_resume, .asic_reset = &evergreen_asic_reset, .vga_set_state = &r600_vga_set_state, - .ioctl_wait_idle = r600_ioctl_wait_idle, + .mmio_hdp_flush = r600_mmio_hdp_flush, .gui_idle = &r600_gui_idle, .mc_wait_for_idle = &evergreen_mc_wait_for_idle, + .get_xclk = &rv770_get_xclk, + .get_gpu_clock_counter = &r600_get_gpu_clock_counter, + .get_allowed_info_register = evergreen_get_allowed_info_register, .gart = { .tlb_flush = &evergreen_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &evergreen_ring_ib_execute, - .emit_fence = &r600_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &evergreen_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &evergreen_gpu_is_lockup, - }, - [R600_RING_TYPE_DMA_INDEX] = { - .ib_execute = &evergreen_dma_ring_ib_execute, - .emit_fence = &evergreen_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &evergreen_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &r600_dma_is_lockup, - } + [RADEON_RING_TYPE_GFX_INDEX] = &evergreen_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &evergreen_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &rv770_uvd_ring, }, .irq = { .set = &evergreen_irq_set, @@ -1388,7 +1551,7 @@ static struct radeon_asic btc_asic = { .get_backlight_level = &atombios_get_backlight_level, }, .copy = { - .blit = &r600_copy_blit, + .blit = &r600_copy_cpdma, .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, .dma = &evergreen_copy_dma, .dma_ring_index = R600_RING_TYPE_DMA_INDEX, @@ -1415,17 +1578,81 @@ static struct radeon_asic btc_asic = { .set_engine_clock = &radeon_atom_set_engine_clock, .get_memory_clock = &radeon_atom_get_memory_clock, .set_memory_clock = &radeon_atom_set_memory_clock, - .get_pcie_lanes = NULL, - .set_pcie_lanes = NULL, + .get_pcie_lanes = &r600_get_pcie_lanes, + .set_pcie_lanes = &r600_set_pcie_lanes, .set_clock_gating = NULL, + .set_uvd_clocks = &evergreen_set_uvd_clocks, + .get_temperature = &evergreen_get_temp, + }, + .dpm = { + .init = &btc_dpm_init, + .setup_asic = &btc_dpm_setup_asic, + .enable = &btc_dpm_enable, + .late_enable = &rv770_dpm_late_enable, + .disable = &btc_dpm_disable, + .pre_set_power_state = &btc_dpm_pre_set_power_state, + .set_power_state = &btc_dpm_set_power_state, + .post_set_power_state = &btc_dpm_post_set_power_state, + .display_configuration_changed = &cypress_dpm_display_configuration_changed, + .fini = &btc_dpm_fini, + .get_sclk = &btc_dpm_get_sclk, + .get_mclk = &btc_dpm_get_mclk, + .print_power_state = &rv770_dpm_print_power_state, + .debugfs_print_current_performance_level = &btc_dpm_debugfs_print_current_performance_level, + .force_performance_level = &rv770_dpm_force_performance_level, + .vblank_too_short = &btc_dpm_vblank_too_short, + .get_current_sclk = &btc_dpm_get_current_sclk, + .get_current_mclk = &btc_dpm_get_current_mclk, }, .pflip = { - .pre_page_flip = &evergreen_pre_page_flip, .page_flip = &evergreen_page_flip, - .post_page_flip = &evergreen_post_page_flip, + .page_flip_pending = &evergreen_page_flip_pending, }, }; +static struct radeon_asic_ring cayman_gfx_ring = { + .ib_execute = &cayman_ring_ib_execute, + .ib_parse = &evergreen_ib_parse, + .emit_fence = &cayman_fence_ring_emit, + .emit_semaphore = &r600_semaphore_ring_emit, + .cs_parse = &evergreen_cs_parse, + .ring_test = &r600_ring_test, + .ib_test = &r600_ib_test, + .is_lockup = &cayman_gfx_is_lockup, + .vm_flush = &cayman_vm_flush, + .get_rptr = &cayman_gfx_get_rptr, + .get_wptr = &cayman_gfx_get_wptr, + .set_wptr = &cayman_gfx_set_wptr, +}; + +static struct radeon_asic_ring cayman_dma_ring = { + .ib_execute = &cayman_dma_ring_ib_execute, + .ib_parse = &evergreen_dma_ib_parse, + .emit_fence = &evergreen_dma_fence_ring_emit, + .emit_semaphore = &r600_dma_semaphore_ring_emit, + .cs_parse = &evergreen_dma_cs_parse, + .ring_test = &r600_dma_ring_test, + .ib_test = &r600_dma_ib_test, + .is_lockup = &cayman_dma_is_lockup, + .vm_flush = &cayman_dma_vm_flush, + .get_rptr = &cayman_dma_get_rptr, + .get_wptr = &cayman_dma_get_wptr, + .set_wptr = &cayman_dma_set_wptr +}; + +static struct radeon_asic_ring cayman_uvd_ring = { + .ib_execute = &uvd_v1_0_ib_execute, + .emit_fence = &uvd_v2_2_fence_emit, + .emit_semaphore = &uvd_v3_1_semaphore_emit, + .cs_parse = &radeon_uvd_cs_parse, + .ring_test = &uvd_v1_0_ring_test, + .ib_test = &uvd_v1_0_ib_test, + .is_lockup = &radeon_ring_test_lockup, + .get_rptr = &uvd_v1_0_get_rptr, + .get_wptr = &uvd_v1_0_get_wptr, + .set_wptr = &uvd_v1_0_set_wptr, +}; + static struct radeon_asic cayman_asic = { .init = &cayman_init, .fini = &cayman_fini, @@ -1433,75 +1660,32 @@ static struct radeon_asic cayman_asic = { .resume = &cayman_resume, .asic_reset = &cayman_asic_reset, .vga_set_state = &r600_vga_set_state, - .ioctl_wait_idle = r600_ioctl_wait_idle, + .mmio_hdp_flush = r600_mmio_hdp_flush, .gui_idle = &r600_gui_idle, .mc_wait_for_idle = &evergreen_mc_wait_for_idle, + .get_xclk = &rv770_get_xclk, + .get_gpu_clock_counter = &r600_get_gpu_clock_counter, + .get_allowed_info_register = cayman_get_allowed_info_register, .gart = { .tlb_flush = &cayman_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .vm = { .init = &cayman_vm_init, .fini = &cayman_vm_fini, - .pt_ring_index = RADEON_RING_TYPE_GFX_INDEX, - .set_page = &cayman_vm_set_page, + .copy_pages = &cayman_dma_vm_copy_pages, + .write_pages = &cayman_dma_vm_write_pages, + .set_pages = &cayman_dma_vm_set_pages, + .pad_ib = &cayman_dma_vm_pad_ib, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &cayman_ring_ib_execute, - .ib_parse = &evergreen_ib_parse, - .emit_fence = &cayman_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &evergreen_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &evergreen_gpu_is_lockup, - .vm_flush = &cayman_vm_flush, - }, - [CAYMAN_RING_TYPE_CP1_INDEX] = { - .ib_execute = &cayman_ring_ib_execute, - .ib_parse = &evergreen_ib_parse, - .emit_fence = &cayman_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &evergreen_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &evergreen_gpu_is_lockup, - .vm_flush = &cayman_vm_flush, - }, - [CAYMAN_RING_TYPE_CP2_INDEX] = { - .ib_execute = &cayman_ring_ib_execute, - .ib_parse = &evergreen_ib_parse, - .emit_fence = &cayman_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &evergreen_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &evergreen_gpu_is_lockup, - .vm_flush = &cayman_vm_flush, - }, - [R600_RING_TYPE_DMA_INDEX] = { - .ib_execute = &cayman_dma_ring_ib_execute, - .ib_parse = &evergreen_dma_ib_parse, - .emit_fence = &evergreen_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &evergreen_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &cayman_dma_is_lockup, - .vm_flush = &cayman_dma_vm_flush, - }, - [CAYMAN_RING_TYPE_DMA1_INDEX] = { - .ib_execute = &cayman_dma_ring_ib_execute, - .ib_parse = &evergreen_dma_ib_parse, - .emit_fence = &evergreen_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &evergreen_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &cayman_dma_is_lockup, - .vm_flush = &cayman_dma_vm_flush, - } + [RADEON_RING_TYPE_GFX_INDEX] = &cayman_gfx_ring, + [CAYMAN_RING_TYPE_CP1_INDEX] = &cayman_gfx_ring, + [CAYMAN_RING_TYPE_CP2_INDEX] = &cayman_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &cayman_dma_ring, + [CAYMAN_RING_TYPE_DMA1_INDEX] = &cayman_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &cayman_uvd_ring, }, .irq = { .set = &evergreen_irq_set, @@ -1515,7 +1699,7 @@ static struct radeon_asic cayman_asic = { .get_backlight_level = &atombios_get_backlight_level, }, .copy = { - .blit = &r600_copy_blit, + .blit = &r600_copy_cpdma, .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, .dma = &evergreen_copy_dma, .dma_ring_index = R600_RING_TYPE_DMA_INDEX, @@ -1542,17 +1726,51 @@ static struct radeon_asic cayman_asic = { .set_engine_clock = &radeon_atom_set_engine_clock, .get_memory_clock = &radeon_atom_get_memory_clock, .set_memory_clock = &radeon_atom_set_memory_clock, - .get_pcie_lanes = NULL, - .set_pcie_lanes = NULL, + .get_pcie_lanes = &r600_get_pcie_lanes, + .set_pcie_lanes = &r600_set_pcie_lanes, .set_clock_gating = NULL, + .set_uvd_clocks = &evergreen_set_uvd_clocks, + .get_temperature = &evergreen_get_temp, + }, + .dpm = { + .init = &ni_dpm_init, + .setup_asic = &ni_dpm_setup_asic, + .enable = &ni_dpm_enable, + .late_enable = &rv770_dpm_late_enable, + .disable = &ni_dpm_disable, + .pre_set_power_state = &ni_dpm_pre_set_power_state, + .set_power_state = &ni_dpm_set_power_state, + .post_set_power_state = &ni_dpm_post_set_power_state, + .display_configuration_changed = &cypress_dpm_display_configuration_changed, + .fini = &ni_dpm_fini, + .get_sclk = &ni_dpm_get_sclk, + .get_mclk = &ni_dpm_get_mclk, + .print_power_state = &ni_dpm_print_power_state, + .debugfs_print_current_performance_level = &ni_dpm_debugfs_print_current_performance_level, + .force_performance_level = &ni_dpm_force_performance_level, + .vblank_too_short = &ni_dpm_vblank_too_short, + .get_current_sclk = &ni_dpm_get_current_sclk, + .get_current_mclk = &ni_dpm_get_current_mclk, }, .pflip = { - .pre_page_flip = &evergreen_pre_page_flip, .page_flip = &evergreen_page_flip, - .post_page_flip = &evergreen_post_page_flip, + .page_flip_pending = &evergreen_page_flip_pending, }, }; +static struct radeon_asic_ring trinity_vce_ring = { + .ib_execute = &radeon_vce_ib_execute, + .emit_fence = &radeon_vce_fence_emit, + .emit_semaphore = &radeon_vce_semaphore_emit, + .cs_parse = &radeon_vce_cs_parse, + .ring_test = &radeon_vce_ring_test, + .ib_test = &radeon_vce_ib_test, + .is_lockup = &radeon_ring_test_lockup, + .get_rptr = &vce_v1_0_get_rptr, + .get_wptr = &vce_v1_0_get_wptr, + .set_wptr = &vce_v1_0_set_wptr, +}; + static struct radeon_asic trinity_asic = { .init = &cayman_init, .fini = &cayman_fini, @@ -1560,75 +1778,34 @@ static struct radeon_asic trinity_asic = { .resume = &cayman_resume, .asic_reset = &cayman_asic_reset, .vga_set_state = &r600_vga_set_state, - .ioctl_wait_idle = r600_ioctl_wait_idle, + .mmio_hdp_flush = r600_mmio_hdp_flush, .gui_idle = &r600_gui_idle, .mc_wait_for_idle = &evergreen_mc_wait_for_idle, + .get_xclk = &r600_get_xclk, + .get_gpu_clock_counter = &r600_get_gpu_clock_counter, + .get_allowed_info_register = cayman_get_allowed_info_register, .gart = { .tlb_flush = &cayman_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .vm = { .init = &cayman_vm_init, .fini = &cayman_vm_fini, - .pt_ring_index = RADEON_RING_TYPE_GFX_INDEX, - .set_page = &cayman_vm_set_page, + .copy_pages = &cayman_dma_vm_copy_pages, + .write_pages = &cayman_dma_vm_write_pages, + .set_pages = &cayman_dma_vm_set_pages, + .pad_ib = &cayman_dma_vm_pad_ib, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &cayman_ring_ib_execute, - .ib_parse = &evergreen_ib_parse, - .emit_fence = &cayman_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &evergreen_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &evergreen_gpu_is_lockup, - .vm_flush = &cayman_vm_flush, - }, - [CAYMAN_RING_TYPE_CP1_INDEX] = { - .ib_execute = &cayman_ring_ib_execute, - .ib_parse = &evergreen_ib_parse, - .emit_fence = &cayman_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &evergreen_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &evergreen_gpu_is_lockup, - .vm_flush = &cayman_vm_flush, - }, - [CAYMAN_RING_TYPE_CP2_INDEX] = { - .ib_execute = &cayman_ring_ib_execute, - .ib_parse = &evergreen_ib_parse, - .emit_fence = &cayman_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = &evergreen_cs_parse, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &evergreen_gpu_is_lockup, - .vm_flush = &cayman_vm_flush, - }, - [R600_RING_TYPE_DMA_INDEX] = { - .ib_execute = &cayman_dma_ring_ib_execute, - .ib_parse = &evergreen_dma_ib_parse, - .emit_fence = &evergreen_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &evergreen_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &cayman_dma_is_lockup, - .vm_flush = &cayman_dma_vm_flush, - }, - [CAYMAN_RING_TYPE_DMA1_INDEX] = { - .ib_execute = &cayman_dma_ring_ib_execute, - .ib_parse = &evergreen_dma_ib_parse, - .emit_fence = &evergreen_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = &evergreen_dma_cs_parse, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &cayman_dma_is_lockup, - .vm_flush = &cayman_dma_vm_flush, - } + [RADEON_RING_TYPE_GFX_INDEX] = &cayman_gfx_ring, + [CAYMAN_RING_TYPE_CP1_INDEX] = &cayman_gfx_ring, + [CAYMAN_RING_TYPE_CP2_INDEX] = &cayman_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &cayman_dma_ring, + [CAYMAN_RING_TYPE_DMA1_INDEX] = &cayman_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &cayman_uvd_ring, + [TN_RING_TYPE_VCE1_INDEX] = &trinity_vce_ring, + [TN_RING_TYPE_VCE2_INDEX] = &trinity_vce_ring, }, .irq = { .set = &evergreen_irq_set, @@ -1642,7 +1819,7 @@ static struct radeon_asic trinity_asic = { .get_backlight_level = &atombios_get_backlight_level, }, .copy = { - .blit = &r600_copy_blit, + .blit = &r600_copy_cpdma, .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, .dma = &evergreen_copy_dma, .dma_ring_index = R600_RING_TYPE_DMA_INDEX, @@ -1672,14 +1849,66 @@ static struct radeon_asic trinity_asic = { .get_pcie_lanes = NULL, .set_pcie_lanes = NULL, .set_clock_gating = NULL, + .set_uvd_clocks = &sumo_set_uvd_clocks, + .set_vce_clocks = &tn_set_vce_clocks, + .get_temperature = &tn_get_temp, + }, + .dpm = { + .init = &trinity_dpm_init, + .setup_asic = &trinity_dpm_setup_asic, + .enable = &trinity_dpm_enable, + .late_enable = &trinity_dpm_late_enable, + .disable = &trinity_dpm_disable, + .pre_set_power_state = &trinity_dpm_pre_set_power_state, + .set_power_state = &trinity_dpm_set_power_state, + .post_set_power_state = &trinity_dpm_post_set_power_state, + .display_configuration_changed = &trinity_dpm_display_configuration_changed, + .fini = &trinity_dpm_fini, + .get_sclk = &trinity_dpm_get_sclk, + .get_mclk = &trinity_dpm_get_mclk, + .print_power_state = &trinity_dpm_print_power_state, + .debugfs_print_current_performance_level = &trinity_dpm_debugfs_print_current_performance_level, + .force_performance_level = &trinity_dpm_force_performance_level, + .enable_bapm = &trinity_dpm_enable_bapm, + .get_current_sclk = &trinity_dpm_get_current_sclk, + .get_current_mclk = &trinity_dpm_get_current_mclk, }, .pflip = { - .pre_page_flip = &evergreen_pre_page_flip, .page_flip = &evergreen_page_flip, - .post_page_flip = &evergreen_post_page_flip, + .page_flip_pending = &evergreen_page_flip_pending, }, }; +static struct radeon_asic_ring si_gfx_ring = { + .ib_execute = &si_ring_ib_execute, + .ib_parse = &si_ib_parse, + .emit_fence = &si_fence_ring_emit, + .emit_semaphore = &r600_semaphore_ring_emit, + .cs_parse = NULL, + .ring_test = &r600_ring_test, + .ib_test = &r600_ib_test, + .is_lockup = &si_gfx_is_lockup, + .vm_flush = &si_vm_flush, + .get_rptr = &cayman_gfx_get_rptr, + .get_wptr = &cayman_gfx_get_wptr, + .set_wptr = &cayman_gfx_set_wptr, +}; + +static struct radeon_asic_ring si_dma_ring = { + .ib_execute = &cayman_dma_ring_ib_execute, + .ib_parse = &evergreen_dma_ib_parse, + .emit_fence = &evergreen_dma_fence_ring_emit, + .emit_semaphore = &r600_dma_semaphore_ring_emit, + .cs_parse = NULL, + .ring_test = &r600_dma_ring_test, + .ib_test = &r600_dma_ib_test, + .is_lockup = &si_dma_is_lockup, + .vm_flush = &si_dma_vm_flush, + .get_rptr = &cayman_dma_get_rptr, + .get_wptr = &cayman_dma_get_wptr, + .set_wptr = &cayman_dma_set_wptr, +}; + static struct radeon_asic si_asic = { .init = &si_init, .fini = &si_fini, @@ -1687,75 +1916,34 @@ static struct radeon_asic si_asic = { .resume = &si_resume, .asic_reset = &si_asic_reset, .vga_set_state = &r600_vga_set_state, - .ioctl_wait_idle = r600_ioctl_wait_idle, + .mmio_hdp_flush = r600_mmio_hdp_flush, .gui_idle = &r600_gui_idle, .mc_wait_for_idle = &evergreen_mc_wait_for_idle, + .get_xclk = &si_get_xclk, + .get_gpu_clock_counter = &si_get_gpu_clock_counter, + .get_allowed_info_register = si_get_allowed_info_register, .gart = { .tlb_flush = &si_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, .set_page = &rs600_gart_set_page, }, .vm = { .init = &si_vm_init, .fini = &si_vm_fini, - .pt_ring_index = RADEON_RING_TYPE_GFX_INDEX, - .set_page = &si_vm_set_page, + .copy_pages = &si_dma_vm_copy_pages, + .write_pages = &si_dma_vm_write_pages, + .set_pages = &si_dma_vm_set_pages, + .pad_ib = &cayman_dma_vm_pad_ib, }, .ring = { - [RADEON_RING_TYPE_GFX_INDEX] = { - .ib_execute = &si_ring_ib_execute, - .ib_parse = &si_ib_parse, - .emit_fence = &si_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = NULL, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &si_gpu_is_lockup, - .vm_flush = &si_vm_flush, - }, - [CAYMAN_RING_TYPE_CP1_INDEX] = { - .ib_execute = &si_ring_ib_execute, - .ib_parse = &si_ib_parse, - .emit_fence = &si_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = NULL, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &si_gpu_is_lockup, - .vm_flush = &si_vm_flush, - }, - [CAYMAN_RING_TYPE_CP2_INDEX] = { - .ib_execute = &si_ring_ib_execute, - .ib_parse = &si_ib_parse, - .emit_fence = &si_fence_ring_emit, - .emit_semaphore = &r600_semaphore_ring_emit, - .cs_parse = NULL, - .ring_test = &r600_ring_test, - .ib_test = &r600_ib_test, - .is_lockup = &si_gpu_is_lockup, - .vm_flush = &si_vm_flush, - }, - [R600_RING_TYPE_DMA_INDEX] = { - .ib_execute = &cayman_dma_ring_ib_execute, - .ib_parse = &evergreen_dma_ib_parse, - .emit_fence = &evergreen_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = NULL, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &cayman_dma_is_lockup, - .vm_flush = &si_dma_vm_flush, - }, - [CAYMAN_RING_TYPE_DMA1_INDEX] = { - .ib_execute = &cayman_dma_ring_ib_execute, - .ib_parse = &evergreen_dma_ib_parse, - .emit_fence = &evergreen_dma_fence_ring_emit, - .emit_semaphore = &r600_dma_semaphore_ring_emit, - .cs_parse = NULL, - .ring_test = &r600_dma_ring_test, - .ib_test = &r600_dma_ib_test, - .is_lockup = &cayman_dma_is_lockup, - .vm_flush = &si_dma_vm_flush, - } + [RADEON_RING_TYPE_GFX_INDEX] = &si_gfx_ring, + [CAYMAN_RING_TYPE_CP1_INDEX] = &si_gfx_ring, + [CAYMAN_RING_TYPE_CP2_INDEX] = &si_gfx_ring, + [R600_RING_TYPE_DMA_INDEX] = &si_dma_ring, + [CAYMAN_RING_TYPE_DMA1_INDEX] = &si_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &cayman_uvd_ring, + [TN_RING_TYPE_VCE1_INDEX] = &trinity_vce_ring, + [TN_RING_TYPE_VCE2_INDEX] = &trinity_vce_ring, }, .irq = { .set = &si_irq_set, @@ -1769,7 +1957,7 @@ static struct radeon_asic si_asic = { .get_backlight_level = &atombios_get_backlight_level, }, .copy = { - .blit = NULL, + .blit = &r600_copy_cpdma, .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, .dma = &si_copy_dma, .dma_ring_index = R600_RING_TYPE_DMA_INDEX, @@ -1786,6 +1974,176 @@ static struct radeon_asic si_asic = { .sense = &evergreen_hpd_sense, .set_polarity = &evergreen_hpd_set_polarity, }, + .pm = { + .misc = &evergreen_pm_misc, + .prepare = &evergreen_pm_prepare, + .finish = &evergreen_pm_finish, + .init_profile = &sumo_pm_init_profile, + .get_dynpm_state = &r600_pm_get_dynpm_state, + .get_engine_clock = &radeon_atom_get_engine_clock, + .set_engine_clock = &radeon_atom_set_engine_clock, + .get_memory_clock = &radeon_atom_get_memory_clock, + .set_memory_clock = &radeon_atom_set_memory_clock, + .get_pcie_lanes = &r600_get_pcie_lanes, + .set_pcie_lanes = &r600_set_pcie_lanes, + .set_clock_gating = NULL, + .set_uvd_clocks = &si_set_uvd_clocks, + .set_vce_clocks = &si_set_vce_clocks, + .get_temperature = &si_get_temp, + }, + .dpm = { + .init = &si_dpm_init, + .setup_asic = &si_dpm_setup_asic, + .enable = &si_dpm_enable, + .late_enable = &si_dpm_late_enable, + .disable = &si_dpm_disable, + .pre_set_power_state = &si_dpm_pre_set_power_state, + .set_power_state = &si_dpm_set_power_state, + .post_set_power_state = &si_dpm_post_set_power_state, + .display_configuration_changed = &si_dpm_display_configuration_changed, + .fini = &si_dpm_fini, + .get_sclk = &ni_dpm_get_sclk, + .get_mclk = &ni_dpm_get_mclk, + .print_power_state = &ni_dpm_print_power_state, + .debugfs_print_current_performance_level = &si_dpm_debugfs_print_current_performance_level, + .force_performance_level = &si_dpm_force_performance_level, + .vblank_too_short = &ni_dpm_vblank_too_short, + .fan_ctrl_set_mode = &si_fan_ctrl_set_mode, + .fan_ctrl_get_mode = &si_fan_ctrl_get_mode, + .get_fan_speed_percent = &si_fan_ctrl_get_fan_speed_percent, + .set_fan_speed_percent = &si_fan_ctrl_set_fan_speed_percent, + .get_current_sclk = &si_dpm_get_current_sclk, + .get_current_mclk = &si_dpm_get_current_mclk, + }, + .pflip = { + .page_flip = &evergreen_page_flip, + .page_flip_pending = &evergreen_page_flip_pending, + }, +}; + +static struct radeon_asic_ring ci_gfx_ring = { + .ib_execute = &cik_ring_ib_execute, + .ib_parse = &cik_ib_parse, + .emit_fence = &cik_fence_gfx_ring_emit, + .emit_semaphore = &cik_semaphore_ring_emit, + .cs_parse = NULL, + .ring_test = &cik_ring_test, + .ib_test = &cik_ib_test, + .is_lockup = &cik_gfx_is_lockup, + .vm_flush = &cik_vm_flush, + .get_rptr = &cik_gfx_get_rptr, + .get_wptr = &cik_gfx_get_wptr, + .set_wptr = &cik_gfx_set_wptr, +}; + +static struct radeon_asic_ring ci_cp_ring = { + .ib_execute = &cik_ring_ib_execute, + .ib_parse = &cik_ib_parse, + .emit_fence = &cik_fence_compute_ring_emit, + .emit_semaphore = &cik_semaphore_ring_emit, + .cs_parse = NULL, + .ring_test = &cik_ring_test, + .ib_test = &cik_ib_test, + .is_lockup = &cik_gfx_is_lockup, + .vm_flush = &cik_vm_flush, + .get_rptr = &cik_compute_get_rptr, + .get_wptr = &cik_compute_get_wptr, + .set_wptr = &cik_compute_set_wptr, +}; + +static struct radeon_asic_ring ci_dma_ring = { + .ib_execute = &cik_sdma_ring_ib_execute, + .ib_parse = &cik_ib_parse, + .emit_fence = &cik_sdma_fence_ring_emit, + .emit_semaphore = &cik_sdma_semaphore_ring_emit, + .cs_parse = NULL, + .ring_test = &cik_sdma_ring_test, + .ib_test = &cik_sdma_ib_test, + .is_lockup = &cik_sdma_is_lockup, + .vm_flush = &cik_dma_vm_flush, + .get_rptr = &cik_sdma_get_rptr, + .get_wptr = &cik_sdma_get_wptr, + .set_wptr = &cik_sdma_set_wptr, +}; + +static struct radeon_asic_ring ci_vce_ring = { + .ib_execute = &radeon_vce_ib_execute, + .emit_fence = &radeon_vce_fence_emit, + .emit_semaphore = &radeon_vce_semaphore_emit, + .cs_parse = &radeon_vce_cs_parse, + .ring_test = &radeon_vce_ring_test, + .ib_test = &radeon_vce_ib_test, + .is_lockup = &radeon_ring_test_lockup, + .get_rptr = &vce_v1_0_get_rptr, + .get_wptr = &vce_v1_0_get_wptr, + .set_wptr = &vce_v1_0_set_wptr, +}; + +static struct radeon_asic ci_asic = { + .init = &cik_init, + .fini = &cik_fini, + .suspend = &cik_suspend, + .resume = &cik_resume, + .asic_reset = &cik_asic_reset, + .vga_set_state = &r600_vga_set_state, + .mmio_hdp_flush = &r600_mmio_hdp_flush, + .gui_idle = &r600_gui_idle, + .mc_wait_for_idle = &evergreen_mc_wait_for_idle, + .get_xclk = &cik_get_xclk, + .get_gpu_clock_counter = &cik_get_gpu_clock_counter, + .get_allowed_info_register = cik_get_allowed_info_register, + .gart = { + .tlb_flush = &cik_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, + .set_page = &rs600_gart_set_page, + }, + .vm = { + .init = &cik_vm_init, + .fini = &cik_vm_fini, + .copy_pages = &cik_sdma_vm_copy_pages, + .write_pages = &cik_sdma_vm_write_pages, + .set_pages = &cik_sdma_vm_set_pages, + .pad_ib = &cik_sdma_vm_pad_ib, + }, + .ring = { + [RADEON_RING_TYPE_GFX_INDEX] = &ci_gfx_ring, + [CAYMAN_RING_TYPE_CP1_INDEX] = &ci_cp_ring, + [CAYMAN_RING_TYPE_CP2_INDEX] = &ci_cp_ring, + [R600_RING_TYPE_DMA_INDEX] = &ci_dma_ring, + [CAYMAN_RING_TYPE_DMA1_INDEX] = &ci_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &cayman_uvd_ring, + [TN_RING_TYPE_VCE1_INDEX] = &ci_vce_ring, + [TN_RING_TYPE_VCE2_INDEX] = &ci_vce_ring, + }, + .irq = { + .set = &cik_irq_set, + .process = &cik_irq_process, + }, + .display = { + .bandwidth_update = &dce8_bandwidth_update, + .get_vblank_counter = &evergreen_get_vblank_counter, + .wait_for_vblank = &dce4_wait_for_vblank, + .set_backlight_level = &atombios_set_backlight_level, + .get_backlight_level = &atombios_get_backlight_level, + }, + .copy = { + .blit = &cik_copy_cpdma, + .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, + .dma = &cik_copy_dma, + .dma_ring_index = R600_RING_TYPE_DMA_INDEX, + .copy = &cik_copy_dma, + .copy_ring_index = R600_RING_TYPE_DMA_INDEX, + }, + .surface = { + .set_reg = r600_set_surface_reg, + .clear_reg = r600_clear_surface_reg, + }, + .hpd = { + .init = &evergreen_hpd_init, + .fini = &evergreen_hpd_fini, + .sense = &evergreen_hpd_sense, + .set_polarity = &evergreen_hpd_set_polarity, + }, .pm = { .misc = &evergreen_pm_misc, .prepare = &evergreen_pm_prepare, @@ -1799,11 +2157,147 @@ static struct radeon_asic si_asic = { .get_pcie_lanes = NULL, .set_pcie_lanes = NULL, .set_clock_gating = NULL, + .set_uvd_clocks = &cik_set_uvd_clocks, + .set_vce_clocks = &cik_set_vce_clocks, + .get_temperature = &ci_get_temp, + }, + .dpm = { + .init = &ci_dpm_init, + .setup_asic = &ci_dpm_setup_asic, + .enable = &ci_dpm_enable, + .late_enable = &ci_dpm_late_enable, + .disable = &ci_dpm_disable, + .pre_set_power_state = &ci_dpm_pre_set_power_state, + .set_power_state = &ci_dpm_set_power_state, + .post_set_power_state = &ci_dpm_post_set_power_state, + .display_configuration_changed = &ci_dpm_display_configuration_changed, + .fini = &ci_dpm_fini, + .get_sclk = &ci_dpm_get_sclk, + .get_mclk = &ci_dpm_get_mclk, + .print_power_state = &ci_dpm_print_power_state, + .debugfs_print_current_performance_level = &ci_dpm_debugfs_print_current_performance_level, + .force_performance_level = &ci_dpm_force_performance_level, + .vblank_too_short = &ci_dpm_vblank_too_short, + .powergate_uvd = &ci_dpm_powergate_uvd, + .fan_ctrl_set_mode = &ci_fan_ctrl_set_mode, + .fan_ctrl_get_mode = &ci_fan_ctrl_get_mode, + .get_fan_speed_percent = &ci_fan_ctrl_get_fan_speed_percent, + .set_fan_speed_percent = &ci_fan_ctrl_set_fan_speed_percent, + .get_current_sclk = &ci_dpm_get_current_sclk, + .get_current_mclk = &ci_dpm_get_current_mclk, }, .pflip = { - .pre_page_flip = &evergreen_pre_page_flip, .page_flip = &evergreen_page_flip, - .post_page_flip = &evergreen_post_page_flip, + .page_flip_pending = &evergreen_page_flip_pending, + }, +}; + +static struct radeon_asic kv_asic = { + .init = &cik_init, + .fini = &cik_fini, + .suspend = &cik_suspend, + .resume = &cik_resume, + .asic_reset = &cik_asic_reset, + .vga_set_state = &r600_vga_set_state, + .mmio_hdp_flush = &r600_mmio_hdp_flush, + .gui_idle = &r600_gui_idle, + .mc_wait_for_idle = &evergreen_mc_wait_for_idle, + .get_xclk = &cik_get_xclk, + .get_gpu_clock_counter = &cik_get_gpu_clock_counter, + .get_allowed_info_register = cik_get_allowed_info_register, + .gart = { + .tlb_flush = &cik_pcie_gart_tlb_flush, + .get_page_entry = &rs600_gart_get_page_entry, + .set_page = &rs600_gart_set_page, + }, + .vm = { + .init = &cik_vm_init, + .fini = &cik_vm_fini, + .copy_pages = &cik_sdma_vm_copy_pages, + .write_pages = &cik_sdma_vm_write_pages, + .set_pages = &cik_sdma_vm_set_pages, + .pad_ib = &cik_sdma_vm_pad_ib, + }, + .ring = { + [RADEON_RING_TYPE_GFX_INDEX] = &ci_gfx_ring, + [CAYMAN_RING_TYPE_CP1_INDEX] = &ci_cp_ring, + [CAYMAN_RING_TYPE_CP2_INDEX] = &ci_cp_ring, + [R600_RING_TYPE_DMA_INDEX] = &ci_dma_ring, + [CAYMAN_RING_TYPE_DMA1_INDEX] = &ci_dma_ring, + [R600_RING_TYPE_UVD_INDEX] = &cayman_uvd_ring, + [TN_RING_TYPE_VCE1_INDEX] = &ci_vce_ring, + [TN_RING_TYPE_VCE2_INDEX] = &ci_vce_ring, + }, + .irq = { + .set = &cik_irq_set, + .process = &cik_irq_process, + }, + .display = { + .bandwidth_update = &dce8_bandwidth_update, + .get_vblank_counter = &evergreen_get_vblank_counter, + .wait_for_vblank = &dce4_wait_for_vblank, + .set_backlight_level = &atombios_set_backlight_level, + .get_backlight_level = &atombios_get_backlight_level, + }, + .copy = { + .blit = &cik_copy_cpdma, + .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX, + .dma = &cik_copy_dma, + .dma_ring_index = R600_RING_TYPE_DMA_INDEX, + .copy = &cik_copy_dma, + .copy_ring_index = R600_RING_TYPE_DMA_INDEX, + }, + .surface = { + .set_reg = r600_set_surface_reg, + .clear_reg = r600_clear_surface_reg, + }, + .hpd = { + .init = &evergreen_hpd_init, + .fini = &evergreen_hpd_fini, + .sense = &evergreen_hpd_sense, + .set_polarity = &evergreen_hpd_set_polarity, + }, + .pm = { + .misc = &evergreen_pm_misc, + .prepare = &evergreen_pm_prepare, + .finish = &evergreen_pm_finish, + .init_profile = &sumo_pm_init_profile, + .get_dynpm_state = &r600_pm_get_dynpm_state, + .get_engine_clock = &radeon_atom_get_engine_clock, + .set_engine_clock = &radeon_atom_set_engine_clock, + .get_memory_clock = &radeon_atom_get_memory_clock, + .set_memory_clock = &radeon_atom_set_memory_clock, + .get_pcie_lanes = NULL, + .set_pcie_lanes = NULL, + .set_clock_gating = NULL, + .set_uvd_clocks = &cik_set_uvd_clocks, + .set_vce_clocks = &cik_set_vce_clocks, + .get_temperature = &kv_get_temp, + }, + .dpm = { + .init = &kv_dpm_init, + .setup_asic = &kv_dpm_setup_asic, + .enable = &kv_dpm_enable, + .late_enable = &kv_dpm_late_enable, + .disable = &kv_dpm_disable, + .pre_set_power_state = &kv_dpm_pre_set_power_state, + .set_power_state = &kv_dpm_set_power_state, + .post_set_power_state = &kv_dpm_post_set_power_state, + .display_configuration_changed = &kv_dpm_display_configuration_changed, + .fini = &kv_dpm_fini, + .get_sclk = &kv_dpm_get_sclk, + .get_mclk = &kv_dpm_get_mclk, + .print_power_state = &kv_dpm_print_power_state, + .debugfs_print_current_performance_level = &kv_dpm_debugfs_print_current_performance_level, + .force_performance_level = &kv_dpm_force_performance_level, + .powergate_uvd = &kv_dpm_powergate_uvd, + .enable_bapm = &kv_dpm_enable_bapm, + .get_current_sclk = &kv_dpm_get_current_sclk, + .get_current_mclk = &kv_dpm_get_current_mclk, + }, + .pflip = { + .page_flip = &evergreen_page_flip, + .page_flip_pending = &evergreen_page_flip_pending, }, }; @@ -1827,6 +2321,8 @@ int radeon_asic_init(struct radeon_device *rdev) else rdev->num_crtc = 2; + rdev->has_uvd = false; + switch (rdev->family) { case CHIP_R100: case CHIP_RV100: @@ -1885,22 +2381,35 @@ int radeon_asic_init(struct radeon_device *rdev) rdev->asic = &r520_asic; break; case CHIP_R600: + rdev->asic = &r600_asic; + break; case CHIP_RV610: case CHIP_RV630: case CHIP_RV620: case CHIP_RV635: case CHIP_RV670: - rdev->asic = &r600_asic; + rdev->asic = &rv6xx_asic; + rdev->has_uvd = true; break; case CHIP_RS780: case CHIP_RS880: rdev->asic = &rs780_asic; + /* 760G/780V/880V don't have UVD */ + if ((rdev->pdev->device == 0x9616)|| + (rdev->pdev->device == 0x9611)|| + (rdev->pdev->device == 0x9613)|| + (rdev->pdev->device == 0x9711)|| + (rdev->pdev->device == 0x9713)) + rdev->has_uvd = false; + else + rdev->has_uvd = true; break; case CHIP_RV770: case CHIP_RV730: case CHIP_RV710: case CHIP_RV740: rdev->asic = &rv770_asic; + rdev->has_uvd = true; break; case CHIP_CEDAR: case CHIP_REDWOOD: @@ -1913,11 +2422,13 @@ int radeon_asic_init(struct radeon_device *rdev) else rdev->num_crtc = 6; rdev->asic = &evergreen_asic; + rdev->has_uvd = true; break; case CHIP_PALM: case CHIP_SUMO: case CHIP_SUMO2: rdev->asic = &sumo_asic; + rdev->has_uvd = true; break; case CHIP_BARTS: case CHIP_TURKS: @@ -1928,23 +2439,243 @@ int radeon_asic_init(struct radeon_device *rdev) else rdev->num_crtc = 6; rdev->asic = &btc_asic; + rdev->has_uvd = true; break; case CHIP_CAYMAN: rdev->asic = &cayman_asic; /* set num crtcs */ rdev->num_crtc = 6; + rdev->has_uvd = true; break; case CHIP_ARUBA: rdev->asic = &trinity_asic; /* set num crtcs */ rdev->num_crtc = 4; + rdev->has_uvd = true; + rdev->cg_flags = + RADEON_CG_SUPPORT_VCE_MGCG; break; case CHIP_TAHITI: case CHIP_PITCAIRN: case CHIP_VERDE: + case CHIP_OLAND: + case CHIP_HAINAN: rdev->asic = &si_asic; /* set num crtcs */ + if (rdev->family == CHIP_HAINAN) + rdev->num_crtc = 0; + else if (rdev->family == CHIP_OLAND) + rdev->num_crtc = 2; + else + rdev->num_crtc = 6; + if (rdev->family == CHIP_HAINAN) + rdev->has_uvd = false; + else + rdev->has_uvd = true; + switch (rdev->family) { + case CHIP_TAHITI: + rdev->cg_flags = + RADEON_CG_SUPPORT_GFX_MGCG | + RADEON_CG_SUPPORT_GFX_MGLS | + /*RADEON_CG_SUPPORT_GFX_CGCG |*/ + RADEON_CG_SUPPORT_GFX_CGLS | + RADEON_CG_SUPPORT_GFX_CGTS | + RADEON_CG_SUPPORT_GFX_CP_LS | + RADEON_CG_SUPPORT_MC_MGCG | + RADEON_CG_SUPPORT_SDMA_MGCG | + RADEON_CG_SUPPORT_BIF_LS | + RADEON_CG_SUPPORT_VCE_MGCG | + RADEON_CG_SUPPORT_UVD_MGCG | + RADEON_CG_SUPPORT_HDP_LS | + RADEON_CG_SUPPORT_HDP_MGCG; + rdev->pg_flags = 0; + break; + case CHIP_PITCAIRN: + rdev->cg_flags = + RADEON_CG_SUPPORT_GFX_MGCG | + RADEON_CG_SUPPORT_GFX_MGLS | + /*RADEON_CG_SUPPORT_GFX_CGCG |*/ + RADEON_CG_SUPPORT_GFX_CGLS | + RADEON_CG_SUPPORT_GFX_CGTS | + RADEON_CG_SUPPORT_GFX_CP_LS | + RADEON_CG_SUPPORT_GFX_RLC_LS | + RADEON_CG_SUPPORT_MC_LS | + RADEON_CG_SUPPORT_MC_MGCG | + RADEON_CG_SUPPORT_SDMA_MGCG | + RADEON_CG_SUPPORT_BIF_LS | + RADEON_CG_SUPPORT_VCE_MGCG | + RADEON_CG_SUPPORT_UVD_MGCG | + RADEON_CG_SUPPORT_HDP_LS | + RADEON_CG_SUPPORT_HDP_MGCG; + rdev->pg_flags = 0; + break; + case CHIP_VERDE: + rdev->cg_flags = + RADEON_CG_SUPPORT_GFX_MGCG | + RADEON_CG_SUPPORT_GFX_MGLS | + /*RADEON_CG_SUPPORT_GFX_CGCG |*/ + RADEON_CG_SUPPORT_GFX_CGLS | + RADEON_CG_SUPPORT_GFX_CGTS | + RADEON_CG_SUPPORT_GFX_CP_LS | + RADEON_CG_SUPPORT_GFX_RLC_LS | + RADEON_CG_SUPPORT_MC_LS | + RADEON_CG_SUPPORT_MC_MGCG | + RADEON_CG_SUPPORT_SDMA_MGCG | + RADEON_CG_SUPPORT_BIF_LS | + RADEON_CG_SUPPORT_VCE_MGCG | + RADEON_CG_SUPPORT_UVD_MGCG | + RADEON_CG_SUPPORT_HDP_LS | + RADEON_CG_SUPPORT_HDP_MGCG; + rdev->pg_flags = 0 | + /*RADEON_PG_SUPPORT_GFX_PG | */ + RADEON_PG_SUPPORT_SDMA; + break; + case CHIP_OLAND: + rdev->cg_flags = + RADEON_CG_SUPPORT_GFX_MGCG | + RADEON_CG_SUPPORT_GFX_MGLS | + /*RADEON_CG_SUPPORT_GFX_CGCG |*/ + RADEON_CG_SUPPORT_GFX_CGLS | + RADEON_CG_SUPPORT_GFX_CGTS | + RADEON_CG_SUPPORT_GFX_CP_LS | + RADEON_CG_SUPPORT_GFX_RLC_LS | + RADEON_CG_SUPPORT_MC_LS | + RADEON_CG_SUPPORT_MC_MGCG | + RADEON_CG_SUPPORT_SDMA_MGCG | + RADEON_CG_SUPPORT_BIF_LS | + RADEON_CG_SUPPORT_UVD_MGCG | + RADEON_CG_SUPPORT_HDP_LS | + RADEON_CG_SUPPORT_HDP_MGCG; + rdev->pg_flags = 0; + break; + case CHIP_HAINAN: + rdev->cg_flags = + RADEON_CG_SUPPORT_GFX_MGCG | + RADEON_CG_SUPPORT_GFX_MGLS | + /*RADEON_CG_SUPPORT_GFX_CGCG |*/ + RADEON_CG_SUPPORT_GFX_CGLS | + RADEON_CG_SUPPORT_GFX_CGTS | + RADEON_CG_SUPPORT_GFX_CP_LS | + RADEON_CG_SUPPORT_GFX_RLC_LS | + RADEON_CG_SUPPORT_MC_LS | + RADEON_CG_SUPPORT_MC_MGCG | + RADEON_CG_SUPPORT_SDMA_MGCG | + RADEON_CG_SUPPORT_BIF_LS | + RADEON_CG_SUPPORT_HDP_LS | + RADEON_CG_SUPPORT_HDP_MGCG; + rdev->pg_flags = 0; + break; + default: + rdev->cg_flags = 0; + rdev->pg_flags = 0; + break; + } + break; + case CHIP_BONAIRE: + case CHIP_HAWAII: + rdev->asic = &ci_asic; rdev->num_crtc = 6; + rdev->has_uvd = true; + if (rdev->family == CHIP_BONAIRE) { + rdev->cg_flags = + RADEON_CG_SUPPORT_GFX_MGCG | + RADEON_CG_SUPPORT_GFX_MGLS | + /*RADEON_CG_SUPPORT_GFX_CGCG |*/ + RADEON_CG_SUPPORT_GFX_CGLS | + RADEON_CG_SUPPORT_GFX_CGTS | + RADEON_CG_SUPPORT_GFX_CGTS_LS | + RADEON_CG_SUPPORT_GFX_CP_LS | + RADEON_CG_SUPPORT_MC_LS | + RADEON_CG_SUPPORT_MC_MGCG | + RADEON_CG_SUPPORT_SDMA_MGCG | + RADEON_CG_SUPPORT_SDMA_LS | + RADEON_CG_SUPPORT_BIF_LS | + RADEON_CG_SUPPORT_VCE_MGCG | + RADEON_CG_SUPPORT_UVD_MGCG | + RADEON_CG_SUPPORT_HDP_LS | + RADEON_CG_SUPPORT_HDP_MGCG; + rdev->pg_flags = 0; + } else { + rdev->cg_flags = + RADEON_CG_SUPPORT_GFX_MGCG | + RADEON_CG_SUPPORT_GFX_MGLS | + /*RADEON_CG_SUPPORT_GFX_CGCG |*/ + RADEON_CG_SUPPORT_GFX_CGLS | + RADEON_CG_SUPPORT_GFX_CGTS | + RADEON_CG_SUPPORT_GFX_CP_LS | + RADEON_CG_SUPPORT_MC_LS | + RADEON_CG_SUPPORT_MC_MGCG | + RADEON_CG_SUPPORT_SDMA_MGCG | + RADEON_CG_SUPPORT_SDMA_LS | + RADEON_CG_SUPPORT_BIF_LS | + RADEON_CG_SUPPORT_VCE_MGCG | + RADEON_CG_SUPPORT_UVD_MGCG | + RADEON_CG_SUPPORT_HDP_LS | + RADEON_CG_SUPPORT_HDP_MGCG; + rdev->pg_flags = 0; + } + break; + case CHIP_KAVERI: + case CHIP_KABINI: + case CHIP_MULLINS: + rdev->asic = &kv_asic; + /* set num crtcs */ + if (rdev->family == CHIP_KAVERI) { + rdev->num_crtc = 4; + rdev->cg_flags = + RADEON_CG_SUPPORT_GFX_MGCG | + RADEON_CG_SUPPORT_GFX_MGLS | + /*RADEON_CG_SUPPORT_GFX_CGCG |*/ + RADEON_CG_SUPPORT_GFX_CGLS | + RADEON_CG_SUPPORT_GFX_CGTS | + RADEON_CG_SUPPORT_GFX_CGTS_LS | + RADEON_CG_SUPPORT_GFX_CP_LS | + RADEON_CG_SUPPORT_SDMA_MGCG | + RADEON_CG_SUPPORT_SDMA_LS | + RADEON_CG_SUPPORT_BIF_LS | + RADEON_CG_SUPPORT_VCE_MGCG | + RADEON_CG_SUPPORT_UVD_MGCG | + RADEON_CG_SUPPORT_HDP_LS | + RADEON_CG_SUPPORT_HDP_MGCG; + rdev->pg_flags = 0; + /*RADEON_PG_SUPPORT_GFX_PG | + RADEON_PG_SUPPORT_GFX_SMG | + RADEON_PG_SUPPORT_GFX_DMG | + RADEON_PG_SUPPORT_UVD | + RADEON_PG_SUPPORT_VCE | + RADEON_PG_SUPPORT_CP | + RADEON_PG_SUPPORT_GDS | + RADEON_PG_SUPPORT_RLC_SMU_HS | + RADEON_PG_SUPPORT_ACP | + RADEON_PG_SUPPORT_SAMU;*/ + } else { + rdev->num_crtc = 2; + rdev->cg_flags = + RADEON_CG_SUPPORT_GFX_MGCG | + RADEON_CG_SUPPORT_GFX_MGLS | + /*RADEON_CG_SUPPORT_GFX_CGCG |*/ + RADEON_CG_SUPPORT_GFX_CGLS | + RADEON_CG_SUPPORT_GFX_CGTS | + RADEON_CG_SUPPORT_GFX_CGTS_LS | + RADEON_CG_SUPPORT_GFX_CP_LS | + RADEON_CG_SUPPORT_SDMA_MGCG | + RADEON_CG_SUPPORT_SDMA_LS | + RADEON_CG_SUPPORT_BIF_LS | + RADEON_CG_SUPPORT_VCE_MGCG | + RADEON_CG_SUPPORT_UVD_MGCG | + RADEON_CG_SUPPORT_HDP_LS | + RADEON_CG_SUPPORT_HDP_MGCG; + rdev->pg_flags = 0; + /*RADEON_PG_SUPPORT_GFX_PG | + RADEON_PG_SUPPORT_GFX_SMG | + RADEON_PG_SUPPORT_UVD | + RADEON_PG_SUPPORT_VCE | + RADEON_PG_SUPPORT_CP | + RADEON_PG_SUPPORT_GDS | + RADEON_PG_SUPPORT_RLC_SMU_HS | + RADEON_PG_SUPPORT_SAMU;*/ + } + rdev->has_uvd = true; break; default: /* FIXME: not supported yet */ diff --git a/sys/dev/pci/drm/radeon/radeon_asic.h b/sys/dev/pci/drm/radeon/radeon_asic.h index f6483d9c574..e0aa33262ea 100644 --- a/sys/dev/pci/drm/radeon/radeon_asic.h +++ b/sys/dev/pci/drm/radeon/radeon_asic.h @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_asic.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -48,7 +47,6 @@ u8 atombios_get_backlight_level(struct radeon_encoder *radeon_encoder); void radeon_legacy_set_backlight_level(struct radeon_encoder *radeon_encoder, u8 level); u8 radeon_legacy_get_backlight_level(struct radeon_encoder *radeon_encoder); - /* * r100,rv100,rs100,rv200,rs200 */ @@ -69,24 +67,26 @@ bool r100_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); int r100_asic_reset(struct radeon_device *rdev); u32 r100_get_vblank_counter(struct radeon_device *rdev, int crtc); void r100_pci_gart_tlb_flush(struct radeon_device *rdev); -int r100_pci_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr); +uint64_t r100_pci_gart_get_page_entry(uint64_t addr, uint32_t flags); +void r100_pci_gart_set_page(struct radeon_device *rdev, unsigned i, + uint64_t entry); void r100_ring_start(struct radeon_device *rdev, struct radeon_ring *ring); int r100_irq_set(struct radeon_device *rdev); int r100_irq_process(struct radeon_device *rdev); void r100_fence_ring_emit(struct radeon_device *rdev, struct radeon_fence *fence); -void r100_semaphore_ring_emit(struct radeon_device *rdev, +bool r100_semaphore_ring_emit(struct radeon_device *rdev, struct radeon_ring *cp, struct radeon_semaphore *semaphore, bool emit_wait); int r100_cs_parse(struct radeon_cs_parser *p); void r100_pll_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); uint32_t r100_pll_rreg(struct radeon_device *rdev, uint32_t reg); -int r100_copy_blit(struct radeon_device *rdev, - uint64_t src_offset, - uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence); +struct radeon_fence *r100_copy_blit(struct radeon_device *rdev, + uint64_t src_offset, + uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); int r100_set_surface_reg(struct radeon_device *rdev, int reg, uint32_t tiling_flags, uint32_t pitch, uint32_t offset, uint32_t obj_size); @@ -137,20 +137,27 @@ extern void r100_pm_prepare(struct radeon_device *rdev); extern void r100_pm_finish(struct radeon_device *rdev); extern void r100_pm_init_profile(struct radeon_device *rdev); extern void r100_pm_get_dynpm_state(struct radeon_device *rdev); -extern void r100_pre_page_flip(struct radeon_device *rdev, int crtc); -extern u32 r100_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base); -extern void r100_post_page_flip(struct radeon_device *rdev, int crtc); +extern void r100_page_flip(struct radeon_device *rdev, int crtc, + u64 crtc_base); +extern bool r100_page_flip_pending(struct radeon_device *rdev, int crtc); extern void r100_wait_for_vblank(struct radeon_device *rdev, int crtc); extern int r100_mc_wait_for_idle(struct radeon_device *rdev); +u32 r100_gfx_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +u32 r100_gfx_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void r100_gfx_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); + /* * r200,rv250,rs300,rv280 */ -extern int r200_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, - uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence); +struct radeon_fence *r200_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, + uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); void r200_set_safe_registers(struct radeon_device *rdev); /* @@ -166,7 +173,9 @@ extern void r300_fence_ring_emit(struct radeon_device *rdev, struct radeon_fence *fence); extern int r300_cs_parse(struct radeon_cs_parser *p); extern void rv370_pcie_gart_tlb_flush(struct radeon_device *rdev); -extern int rv370_pcie_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr); +extern uint64_t rv370_pcie_gart_get_page_entry(uint64_t addr, uint32_t flags); +extern void rv370_pcie_gart_set_page(struct radeon_device *rdev, unsigned i, + uint64_t entry); extern void rv370_set_pcie_lanes(struct radeon_device *rdev, int lanes); extern int rv370_get_pcie_lanes(struct radeon_device *rdev); extern void r300_set_reg_safe(struct radeon_device *rdev); @@ -201,7 +210,9 @@ extern void rs400_fini(struct radeon_device *rdev); extern int rs400_suspend(struct radeon_device *rdev); extern int rs400_resume(struct radeon_device *rdev); void rs400_gart_tlb_flush(struct radeon_device *rdev); -int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr); +uint64_t rs400_gart_get_page_entry(uint64_t addr, uint32_t flags); +void rs400_gart_set_page(struct radeon_device *rdev, unsigned i, + uint64_t entry); uint32_t rs400_mc_rreg(struct radeon_device *rdev, uint32_t reg); void rs400_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); int rs400_gart_init(struct radeon_device *rdev); @@ -224,7 +235,9 @@ int rs600_irq_process(struct radeon_device *rdev); void rs600_irq_disable(struct radeon_device *rdev); u32 rs600_get_vblank_counter(struct radeon_device *rdev, int crtc); void rs600_gart_tlb_flush(struct radeon_device *rdev); -int rs600_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr); +uint64_t rs600_gart_get_page_entry(uint64_t addr, uint32_t flags); +void rs600_gart_set_page(struct radeon_device *rdev, unsigned i, + uint64_t entry); uint32_t rs600_mc_rreg(struct radeon_device *rdev, uint32_t reg); void rs600_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); void rs600_bandwidth_update(struct radeon_device *rdev); @@ -236,9 +249,9 @@ void rs600_hpd_set_polarity(struct radeon_device *rdev, extern void rs600_pm_misc(struct radeon_device *rdev); extern void rs600_pm_prepare(struct radeon_device *rdev); extern void rs600_pm_finish(struct radeon_device *rdev); -extern void rs600_pre_page_flip(struct radeon_device *rdev, int crtc); -extern u32 rs600_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base); -extern void rs600_post_page_flip(struct radeon_device *rdev, int crtc); +extern void rs600_page_flip(struct radeon_device *rdev, int crtc, + u64 crtc_base); +extern bool rs600_page_flip_pending(struct radeon_device *rdev, int crtc); void rs600_set_safe_registers(struct radeon_device *rdev); extern void avivo_wait_for_vblank(struct radeon_device *rdev, int crtc); extern int rs600_mc_wait_for_idle(struct radeon_device *rdev); @@ -308,19 +321,19 @@ int r600_cs_parse(struct radeon_cs_parser *p); int r600_dma_cs_parse(struct radeon_cs_parser *p); void r600_fence_ring_emit(struct radeon_device *rdev, struct radeon_fence *fence); -void r600_semaphore_ring_emit(struct radeon_device *rdev, +bool r600_semaphore_ring_emit(struct radeon_device *rdev, struct radeon_ring *cp, struct radeon_semaphore *semaphore, bool emit_wait); void r600_dma_fence_ring_emit(struct radeon_device *rdev, struct radeon_fence *fence); -void r600_dma_semaphore_ring_emit(struct radeon_device *rdev, +bool r600_dma_semaphore_ring_emit(struct radeon_device *rdev, struct radeon_ring *ring, struct radeon_semaphore *semaphore, bool emit_wait); void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring); -bool r600_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); +bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); int r600_asic_reset(struct radeon_device *rdev); int r600_set_surface_reg(struct radeon_device *rdev, int reg, uint32_t tiling_flags, uint32_t pitch, @@ -331,22 +344,26 @@ int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring); void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *cp); int r600_dma_ring_test(struct radeon_device *rdev, struct radeon_ring *cp); -int r600_copy_blit(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, struct radeon_fence **fence); -int r600_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, struct radeon_fence **fence); +struct radeon_fence *r600_copy_cpdma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); +struct radeon_fence *r600_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); void r600_hpd_init(struct radeon_device *rdev); void r600_hpd_fini(struct radeon_device *rdev); bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd); void r600_hpd_set_polarity(struct radeon_device *rdev, enum radeon_hpd_id hpd); -extern void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo); +extern void r600_mmio_hdp_flush(struct radeon_device *rdev); extern bool r600_gui_idle(struct radeon_device *rdev); extern void r600_pm_misc(struct radeon_device *rdev); extern void r600_pm_init_profile(struct radeon_device *rdev); extern void rs780_pm_init_profile(struct radeon_device *rdev); +extern uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg); +extern void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); extern void r600_pm_get_dynpm_state(struct radeon_device *rdev); extern void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes); extern int r600_get_pcie_lanes(struct radeon_device *rdev); @@ -360,9 +377,15 @@ int r600_count_pipe_bits(uint32_t val); int r600_mc_wait_for_idle(struct radeon_device *rdev); int r600_pcie_gart_init(struct radeon_device *rdev); void r600_scratch_init(struct radeon_device *rdev); -int r600_blit_init(struct radeon_device *rdev); -void r600_blit_fini(struct radeon_device *rdev); int r600_init_microcode(struct radeon_device *rdev); +u32 r600_gfx_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +u32 r600_gfx_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void r600_gfx_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +int r600_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val); /* r600 irq */ int r600_irq_process(struct radeon_device *rdev); int r600_irq_init(struct radeon_device *rdev); @@ -373,24 +396,65 @@ void r600_irq_suspend(struct radeon_device *rdev); void r600_disable_interrupts(struct radeon_device *rdev); void r600_rlc_stop(struct radeon_device *rdev); /* r600 audio */ -int r600_audio_init(struct radeon_device *rdev); -void r600_audio_set_clock(struct drm_encoder *encoder, int clock); -struct r600_audio r600_audio_status(struct radeon_device *rdev); void r600_audio_fini(struct radeon_device *rdev); +void r600_audio_set_dto(struct drm_encoder *encoder, u32 clock); +void r600_hdmi_update_avi_infoframe(struct drm_encoder *encoder, void *buffer, + size_t size); +void r600_hdmi_update_ACR(struct drm_encoder *encoder, uint32_t clock); +void r600_hdmi_audio_workaround(struct drm_encoder *encoder); int r600_hdmi_buffer_status_changed(struct drm_encoder *encoder); void r600_hdmi_update_audio_settings(struct drm_encoder *encoder); -/* r600 blit */ -int r600_blit_prepare_copy(struct radeon_device *rdev, unsigned num_gpu_pages, - struct radeon_fence **fence, struct radeon_sa_bo **vb, - struct radeon_semaphore **sem); -void r600_blit_done_copy(struct radeon_device *rdev, struct radeon_fence **fence, - struct radeon_sa_bo *vb, struct radeon_semaphore *sem); -void r600_kms_blit_copy(struct radeon_device *rdev, - u64 src_gpu_addr, u64 dst_gpu_addr, - unsigned num_gpu_pages, - struct radeon_sa_bo *vb); int r600_mc_wait_for_idle(struct radeon_device *rdev); -uint64_t r600_get_gpu_clock(struct radeon_device *rdev); +u32 r600_get_xclk(struct radeon_device *rdev); +uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev); +int rv6xx_get_temp(struct radeon_device *rdev); +int r600_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk); +int r600_dpm_pre_set_power_state(struct radeon_device *rdev); +void r600_dpm_post_set_power_state(struct radeon_device *rdev); +int r600_dpm_late_enable(struct radeon_device *rdev); +/* r600 dma */ +uint32_t r600_dma_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +uint32_t r600_dma_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void r600_dma_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +/* rv6xx dpm */ +int rv6xx_dpm_init(struct radeon_device *rdev); +int rv6xx_dpm_enable(struct radeon_device *rdev); +void rv6xx_dpm_disable(struct radeon_device *rdev); +int rv6xx_dpm_set_power_state(struct radeon_device *rdev); +void rv6xx_setup_asic(struct radeon_device *rdev); +void rv6xx_dpm_display_configuration_changed(struct radeon_device *rdev); +void rv6xx_dpm_fini(struct radeon_device *rdev); +u32 rv6xx_dpm_get_sclk(struct radeon_device *rdev, bool low); +u32 rv6xx_dpm_get_mclk(struct radeon_device *rdev, bool low); +void rv6xx_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *ps); +void rv6xx_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +int rv6xx_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +u32 rv6xx_dpm_get_current_sclk(struct radeon_device *rdev); +u32 rv6xx_dpm_get_current_mclk(struct radeon_device *rdev); +/* rs780 dpm */ +int rs780_dpm_init(struct radeon_device *rdev); +int rs780_dpm_enable(struct radeon_device *rdev); +void rs780_dpm_disable(struct radeon_device *rdev); +int rs780_dpm_set_power_state(struct radeon_device *rdev); +void rs780_dpm_setup_asic(struct radeon_device *rdev); +void rs780_dpm_display_configuration_changed(struct radeon_device *rdev); +void rs780_dpm_fini(struct radeon_device *rdev); +u32 rs780_dpm_get_sclk(struct radeon_device *rdev, bool low); +u32 rs780_dpm_get_mclk(struct radeon_device *rdev, bool low); +void rs780_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *ps); +void rs780_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +int rs780_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +u32 rs780_dpm_get_current_sclk(struct radeon_device *rdev); +u32 rs780_dpm_get_current_mclk(struct radeon_device *rdev); /* * rv770,rv730,rv710,rv740 @@ -400,14 +464,38 @@ void rv770_fini(struct radeon_device *rdev); int rv770_suspend(struct radeon_device *rdev); int rv770_resume(struct radeon_device *rdev); void rv770_pm_misc(struct radeon_device *rdev); -u32 rv770_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base); +void rv770_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base); +bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc); void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc); void r700_cp_stop(struct radeon_device *rdev); void r700_cp_fini(struct radeon_device *rdev); -int rv770_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence); +struct radeon_fence *rv770_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); +u32 rv770_get_xclk(struct radeon_device *rdev); +int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk); +int rv770_get_temp(struct radeon_device *rdev); +/* rv7xx pm */ +int rv770_dpm_init(struct radeon_device *rdev); +int rv770_dpm_enable(struct radeon_device *rdev); +int rv770_dpm_late_enable(struct radeon_device *rdev); +void rv770_dpm_disable(struct radeon_device *rdev); +int rv770_dpm_set_power_state(struct radeon_device *rdev); +void rv770_dpm_setup_asic(struct radeon_device *rdev); +void rv770_dpm_display_configuration_changed(struct radeon_device *rdev); +void rv770_dpm_fini(struct radeon_device *rdev); +u32 rv770_dpm_get_sclk(struct radeon_device *rdev, bool low); +u32 rv770_dpm_get_mclk(struct radeon_device *rdev, bool low); +void rv770_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *ps); +void rv770_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +int rv770_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +bool rv770_dpm_vblank_too_short(struct radeon_device *rdev); +u32 rv770_dpm_get_current_sclk(struct radeon_device *rdev); +u32 rv770_dpm_get_current_mclk(struct radeon_device *rdev); /* * evergreen @@ -423,7 +511,8 @@ int evergreen_init(struct radeon_device *rdev); void evergreen_fini(struct radeon_device *rdev); int evergreen_suspend(struct radeon_device *rdev); int evergreen_resume(struct radeon_device *rdev); -bool evergreen_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); +bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); +bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); int evergreen_asic_reset(struct radeon_device *rdev); void evergreen_bandwidth_update(struct radeon_device *rdev); void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); @@ -442,21 +531,70 @@ extern void evergreen_pm_prepare(struct radeon_device *rdev); extern void evergreen_pm_finish(struct radeon_device *rdev); extern void sumo_pm_init_profile(struct radeon_device *rdev); extern void btc_pm_init_profile(struct radeon_device *rdev); -extern void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc); -extern u32 evergreen_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base); -extern void evergreen_post_page_flip(struct radeon_device *rdev, int crtc); +int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk); +int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk); +extern void evergreen_page_flip(struct radeon_device *rdev, int crtc, + u64 crtc_base); +extern bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc); extern void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc); void evergreen_disable_interrupt_state(struct radeon_device *rdev); -int evergreen_blit_init(struct radeon_device *rdev); int evergreen_mc_wait_for_idle(struct radeon_device *rdev); void evergreen_dma_fence_ring_emit(struct radeon_device *rdev, struct radeon_fence *fence); void evergreen_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); -int evergreen_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence); +struct radeon_fence *evergreen_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); +int evergreen_get_temp(struct radeon_device *rdev); +int evergreen_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val); +int sumo_get_temp(struct radeon_device *rdev); +int tn_get_temp(struct radeon_device *rdev); +int cypress_dpm_init(struct radeon_device *rdev); +void cypress_dpm_setup_asic(struct radeon_device *rdev); +int cypress_dpm_enable(struct radeon_device *rdev); +void cypress_dpm_disable(struct radeon_device *rdev); +int cypress_dpm_set_power_state(struct radeon_device *rdev); +void cypress_dpm_display_configuration_changed(struct radeon_device *rdev); +void cypress_dpm_fini(struct radeon_device *rdev); +bool cypress_dpm_vblank_too_short(struct radeon_device *rdev); +int btc_dpm_init(struct radeon_device *rdev); +void btc_dpm_setup_asic(struct radeon_device *rdev); +int btc_dpm_enable(struct radeon_device *rdev); +void btc_dpm_disable(struct radeon_device *rdev); +int btc_dpm_pre_set_power_state(struct radeon_device *rdev); +int btc_dpm_set_power_state(struct radeon_device *rdev); +void btc_dpm_post_set_power_state(struct radeon_device *rdev); +void btc_dpm_fini(struct radeon_device *rdev); +u32 btc_dpm_get_sclk(struct radeon_device *rdev, bool low); +u32 btc_dpm_get_mclk(struct radeon_device *rdev, bool low); +bool btc_dpm_vblank_too_short(struct radeon_device *rdev); +void btc_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +u32 btc_dpm_get_current_sclk(struct radeon_device *rdev); +u32 btc_dpm_get_current_mclk(struct radeon_device *rdev); +int sumo_dpm_init(struct radeon_device *rdev); +int sumo_dpm_enable(struct radeon_device *rdev); +int sumo_dpm_late_enable(struct radeon_device *rdev); +void sumo_dpm_disable(struct radeon_device *rdev); +int sumo_dpm_pre_set_power_state(struct radeon_device *rdev); +int sumo_dpm_set_power_state(struct radeon_device *rdev); +void sumo_dpm_post_set_power_state(struct radeon_device *rdev); +void sumo_dpm_setup_asic(struct radeon_device *rdev); +void sumo_dpm_display_configuration_changed(struct radeon_device *rdev); +void sumo_dpm_fini(struct radeon_device *rdev); +u32 sumo_dpm_get_sclk(struct radeon_device *rdev, bool low); +u32 sumo_dpm_get_mclk(struct radeon_device *rdev, bool low); +void sumo_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *ps); +void sumo_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +int sumo_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +u32 sumo_dpm_get_current_sclk(struct radeon_device *rdev); +u32 sumo_dpm_get_current_mclk(struct radeon_device *rdev); /* * cayman @@ -472,20 +610,95 @@ int cayman_asic_reset(struct radeon_device *rdev); void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); int cayman_vm_init(struct radeon_device *rdev); void cayman_vm_fini(struct radeon_device *rdev); -void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); +void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags); -void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe, - uint64_t addr, unsigned count, - uint32_t incr, uint32_t flags); int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); int evergreen_dma_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); void cayman_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); +bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring); bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring); -void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); + +void cayman_dma_vm_copy_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, uint64_t src, + unsigned count); +void cayman_dma_vm_write_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags); +void cayman_dma_vm_set_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags); +void cayman_dma_vm_pad_ib(struct radeon_ib *ib); + +void cayman_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); + +u32 cayman_gfx_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +u32 cayman_gfx_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void cayman_gfx_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +uint32_t cayman_dma_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +uint32_t cayman_dma_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void cayman_dma_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +int cayman_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val); + +int ni_dpm_init(struct radeon_device *rdev); +void ni_dpm_setup_asic(struct radeon_device *rdev); +int ni_dpm_enable(struct radeon_device *rdev); +void ni_dpm_disable(struct radeon_device *rdev); +int ni_dpm_pre_set_power_state(struct radeon_device *rdev); +int ni_dpm_set_power_state(struct radeon_device *rdev); +void ni_dpm_post_set_power_state(struct radeon_device *rdev); +void ni_dpm_fini(struct radeon_device *rdev); +u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low); +u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low); +void ni_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *ps); +void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +int ni_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +bool ni_dpm_vblank_too_short(struct radeon_device *rdev); +u32 ni_dpm_get_current_sclk(struct radeon_device *rdev); +u32 ni_dpm_get_current_mclk(struct radeon_device *rdev); +int trinity_dpm_init(struct radeon_device *rdev); +int trinity_dpm_enable(struct radeon_device *rdev); +int trinity_dpm_late_enable(struct radeon_device *rdev); +void trinity_dpm_disable(struct radeon_device *rdev); +int trinity_dpm_pre_set_power_state(struct radeon_device *rdev); +int trinity_dpm_set_power_state(struct radeon_device *rdev); +void trinity_dpm_post_set_power_state(struct radeon_device *rdev); +void trinity_dpm_setup_asic(struct radeon_device *rdev); +void trinity_dpm_display_configuration_changed(struct radeon_device *rdev); +void trinity_dpm_fini(struct radeon_device *rdev); +u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low); +u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low); +void trinity_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *ps); +void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +int trinity_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +void trinity_dpm_enable_bapm(struct radeon_device *rdev, bool enable); +u32 trinity_dpm_get_current_sclk(struct radeon_device *rdev); +u32 trinity_dpm_get_current_mclk(struct radeon_device *rdev); +int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk); /* DCE6 - SI */ void dce6_bandwidth_update(struct radeon_device *rdev); +void dce6_audio_fini(struct radeon_device *rdev); /* * si @@ -497,23 +710,276 @@ int si_init(struct radeon_device *rdev); void si_fini(struct radeon_device *rdev); int si_suspend(struct radeon_device *rdev); int si_resume(struct radeon_device *rdev); -bool si_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); +bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); +bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); int si_asic_reset(struct radeon_device *rdev); void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); int si_irq_set(struct radeon_device *rdev); int si_irq_process(struct radeon_device *rdev); int si_vm_init(struct radeon_device *rdev); void si_vm_fini(struct radeon_device *rdev); -void si_vm_set_page(struct radeon_device *rdev, uint64_t pe, - uint64_t addr, unsigned count, - uint32_t incr, uint32_t flags); -void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); +void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); -uint64_t si_get_gpu_clock(struct radeon_device *rdev); -int si_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence); -void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); +struct radeon_fence *si_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); + +void si_dma_vm_copy_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, uint64_t src, + unsigned count); +void si_dma_vm_write_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags); +void si_dma_vm_set_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags); + +void si_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); +u32 si_get_xclk(struct radeon_device *rdev); +uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev); +int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk); +int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk); +int si_get_temp(struct radeon_device *rdev); +int si_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val); +int si_dpm_init(struct radeon_device *rdev); +void si_dpm_setup_asic(struct radeon_device *rdev); +int si_dpm_enable(struct radeon_device *rdev); +int si_dpm_late_enable(struct radeon_device *rdev); +void si_dpm_disable(struct radeon_device *rdev); +int si_dpm_pre_set_power_state(struct radeon_device *rdev); +int si_dpm_set_power_state(struct radeon_device *rdev); +void si_dpm_post_set_power_state(struct radeon_device *rdev); +void si_dpm_fini(struct radeon_device *rdev); +void si_dpm_display_configuration_changed(struct radeon_device *rdev); +void si_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +int si_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +int si_fan_ctrl_get_fan_speed_percent(struct radeon_device *rdev, + u32 *speed); +int si_fan_ctrl_set_fan_speed_percent(struct radeon_device *rdev, + u32 speed); +u32 si_fan_ctrl_get_mode(struct radeon_device *rdev); +void si_fan_ctrl_set_mode(struct radeon_device *rdev, u32 mode); +u32 si_dpm_get_current_sclk(struct radeon_device *rdev); +u32 si_dpm_get_current_mclk(struct radeon_device *rdev); + +/* DCE8 - CIK */ +void dce8_bandwidth_update(struct radeon_device *rdev); + +/* + * cik + */ +uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev); +u32 cik_get_xclk(struct radeon_device *rdev); +uint32_t cik_pciep_rreg(struct radeon_device *rdev, uint32_t reg); +void cik_pciep_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); +int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk); +int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk); +void cik_sdma_fence_ring_emit(struct radeon_device *rdev, + struct radeon_fence *fence); +bool cik_sdma_semaphore_ring_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait); +void cik_sdma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); +struct radeon_fence *cik_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); +struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv); +int cik_sdma_ring_test(struct radeon_device *rdev, struct radeon_ring *ring); +int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring); +bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring); +void cik_fence_gfx_ring_emit(struct radeon_device *rdev, + struct radeon_fence *fence); +void cik_fence_compute_ring_emit(struct radeon_device *rdev, + struct radeon_fence *fence); +bool cik_semaphore_ring_emit(struct radeon_device *rdev, + struct radeon_ring *cp, + struct radeon_semaphore *semaphore, + bool emit_wait); +void cik_pcie_gart_tlb_flush(struct radeon_device *rdev); +int cik_init(struct radeon_device *rdev); +void cik_fini(struct radeon_device *rdev); +int cik_suspend(struct radeon_device *rdev); +int cik_resume(struct radeon_device *rdev); +bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp); +int cik_asic_reset(struct radeon_device *rdev); +void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); +int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring); +int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring); +int cik_irq_set(struct radeon_device *rdev); +int cik_irq_process(struct radeon_device *rdev); +int cik_vm_init(struct radeon_device *rdev); +void cik_vm_fini(struct radeon_device *rdev); +void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); + +void cik_sdma_vm_copy_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, uint64_t src, + unsigned count); +void cik_sdma_vm_write_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags); +void cik_sdma_vm_set_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags); +void cik_sdma_vm_pad_ib(struct radeon_ib *ib); + +void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); +int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); +u32 cik_gfx_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +u32 cik_gfx_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void cik_gfx_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +u32 cik_compute_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +u32 cik_compute_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void cik_compute_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +u32 cik_sdma_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +u32 cik_sdma_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void cik_sdma_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +int ci_get_temp(struct radeon_device *rdev); +int kv_get_temp(struct radeon_device *rdev); +int cik_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val); + +int ci_dpm_init(struct radeon_device *rdev); +int ci_dpm_enable(struct radeon_device *rdev); +int ci_dpm_late_enable(struct radeon_device *rdev); +void ci_dpm_disable(struct radeon_device *rdev); +int ci_dpm_pre_set_power_state(struct radeon_device *rdev); +int ci_dpm_set_power_state(struct radeon_device *rdev); +void ci_dpm_post_set_power_state(struct radeon_device *rdev); +void ci_dpm_setup_asic(struct radeon_device *rdev); +void ci_dpm_display_configuration_changed(struct radeon_device *rdev); +void ci_dpm_fini(struct radeon_device *rdev); +u32 ci_dpm_get_sclk(struct radeon_device *rdev, bool low); +u32 ci_dpm_get_mclk(struct radeon_device *rdev, bool low); +void ci_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *ps); +void ci_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +int ci_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +bool ci_dpm_vblank_too_short(struct radeon_device *rdev); +void ci_dpm_powergate_uvd(struct radeon_device *rdev, bool gate); +u32 ci_dpm_get_current_sclk(struct radeon_device *rdev); +u32 ci_dpm_get_current_mclk(struct radeon_device *rdev); + +int ci_fan_ctrl_get_fan_speed_percent(struct radeon_device *rdev, + u32 *speed); +int ci_fan_ctrl_set_fan_speed_percent(struct radeon_device *rdev, + u32 speed); +u32 ci_fan_ctrl_get_mode(struct radeon_device *rdev); +void ci_fan_ctrl_set_mode(struct radeon_device *rdev, u32 mode); + +int kv_dpm_init(struct radeon_device *rdev); +int kv_dpm_enable(struct radeon_device *rdev); +int kv_dpm_late_enable(struct radeon_device *rdev); +void kv_dpm_disable(struct radeon_device *rdev); +int kv_dpm_pre_set_power_state(struct radeon_device *rdev); +int kv_dpm_set_power_state(struct radeon_device *rdev); +void kv_dpm_post_set_power_state(struct radeon_device *rdev); +void kv_dpm_setup_asic(struct radeon_device *rdev); +void kv_dpm_display_configuration_changed(struct radeon_device *rdev); +void kv_dpm_fini(struct radeon_device *rdev); +u32 kv_dpm_get_sclk(struct radeon_device *rdev, bool low); +u32 kv_dpm_get_mclk(struct radeon_device *rdev, bool low); +void kv_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *ps); +void kv_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m); +int kv_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +void kv_dpm_powergate_uvd(struct radeon_device *rdev, bool gate); +void kv_dpm_enable_bapm(struct radeon_device *rdev, bool enable); +u32 kv_dpm_get_current_sclk(struct radeon_device *rdev); +u32 kv_dpm_get_current_mclk(struct radeon_device *rdev); + +/* uvd v1.0 */ +uint32_t uvd_v1_0_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +uint32_t uvd_v1_0_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void uvd_v1_0_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +int uvd_v1_0_resume(struct radeon_device *rdev); + +int uvd_v1_0_init(struct radeon_device *rdev); +void uvd_v1_0_fini(struct radeon_device *rdev); +int uvd_v1_0_start(struct radeon_device *rdev); +void uvd_v1_0_stop(struct radeon_device *rdev); + +int uvd_v1_0_ring_test(struct radeon_device *rdev, struct radeon_ring *ring); +void uvd_v1_0_fence_emit(struct radeon_device *rdev, + struct radeon_fence *fence); +int uvd_v1_0_ib_test(struct radeon_device *rdev, struct radeon_ring *ring); +bool uvd_v1_0_semaphore_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait); +void uvd_v1_0_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); + +/* uvd v2.2 */ +int uvd_v2_2_resume(struct radeon_device *rdev); +void uvd_v2_2_fence_emit(struct radeon_device *rdev, + struct radeon_fence *fence); +bool uvd_v2_2_semaphore_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait); + +/* uvd v3.1 */ +bool uvd_v3_1_semaphore_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait); + +/* uvd v4.2 */ +int uvd_v4_2_resume(struct radeon_device *rdev); + +/* vce v1.0 */ +uint32_t vce_v1_0_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring); +uint32_t vce_v1_0_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +void vce_v1_0_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring); +int vce_v1_0_load_fw(struct radeon_device *rdev, uint32_t *data); +unsigned vce_v1_0_bo_size(struct radeon_device *rdev); +int vce_v1_0_resume(struct radeon_device *rdev); +int vce_v1_0_init(struct radeon_device *rdev); +int vce_v1_0_start(struct radeon_device *rdev); + +/* vce v2.0 */ +unsigned vce_v2_0_bo_size(struct radeon_device *rdev); +int vce_v2_0_resume(struct radeon_device *rdev); #endif diff --git a/sys/dev/pci/drm/radeon/radeon_atombios.c b/sys/dev/pci/drm/radeon/radeon_atombios.c index 683419cf064..e817bcb2daa 100644 --- a/sys/dev/pci/drm/radeon/radeon_atombios.c +++ b/sys/dev/pci/drm/radeon/radeon_atombios.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_atombios.c,v 1.9 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2007-8 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -31,36 +30,15 @@ #include "atom.h" #include "atom-bits.h" -/* from radeon_encoder.c */ -extern uint32_t -radeon_get_encoder_enum(struct drm_device *dev, uint32_t supported_device, - uint8_t dac); -extern void radeon_link_encoder_connector(struct drm_device *dev); extern void radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t supported_device, u16 caps); -/* from radeon_connector.c */ -extern void -radeon_add_atom_connector(struct drm_device *dev, - uint32_t connector_id, - uint32_t supported_device, - int connector_type, - struct radeon_i2c_bus_rec *i2c_bus, - uint32_t igp_lane_info, - uint16_t connector_object_id, - struct radeon_hpd *hpd, - struct radeon_router *router); - /* from radeon_legacy_encoder.c */ extern void radeon_add_legacy_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t supported_device); -/* local */ -static int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type, - u16 voltage_id, u16 *voltage); - union atom_supported_devices { struct _ATOM_SUPPORTED_DEVICES_INFO info; struct _ATOM_SUPPORTED_DEVICES_INFO_2 info_2; @@ -168,8 +146,8 @@ static struct radeon_i2c_bus_rec radeon_lookup_i2c_gpio(struct radeon_device *rd num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / sizeof(ATOM_GPIO_I2C_ASSIGMENT); + gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { - gpio = &i2c_info->asGPIO_Info[i]; radeon_lookup_i2c_gpio_quirks(rdev, gpio, i); @@ -177,6 +155,8 @@ static struct radeon_i2c_bus_rec radeon_lookup_i2c_gpio(struct radeon_device *rd i2c = radeon_get_bus_rec_for_i2c_gpio(gpio); break; } + gpio = (ATOM_GPIO_I2C_ASSIGMENT *) + ((u8 *)gpio + sizeof(ATOM_GPIO_I2C_ASSIGMENT)); } } @@ -200,9 +180,8 @@ void radeon_atombios_i2c_init(struct radeon_device *rdev) num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / sizeof(ATOM_GPIO_I2C_ASSIGMENT); + gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { - gpio = &i2c_info->asGPIO_Info[i]; - radeon_lookup_i2c_gpio_quirks(rdev, gpio, i); i2c = radeon_get_bus_rec_for_i2c_gpio(gpio); @@ -211,12 +190,14 @@ void radeon_atombios_i2c_init(struct radeon_device *rdev) snprintf(stmp, sizeof(stmp), "0x%x", i2c.i2c_id); rdev->i2c_bus[i] = radeon_i2c_create(rdev->ddev, &i2c, stmp); } + gpio = (ATOM_GPIO_I2C_ASSIGMENT *) + ((u8 *)gpio + sizeof(ATOM_GPIO_I2C_ASSIGMENT)); } } } -static struct radeon_gpio_rec radeon_lookup_gpio(struct radeon_device *rdev, - u8 id) +struct radeon_gpio_rec radeon_atombios_lookup_gpio(struct radeon_device *rdev, + u8 id) { struct atom_context *ctx = rdev->mode_info.atom_context; struct radeon_gpio_rec gpio; @@ -235,15 +216,18 @@ static struct radeon_gpio_rec radeon_lookup_gpio(struct radeon_device *rdev, num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / sizeof(ATOM_GPIO_PIN_ASSIGNMENT); + pin = gpio_info->asGPIO_Pin; for (i = 0; i < num_indices; i++) { - pin = &gpio_info->asGPIO_Pin[i]; if (id == pin->ucGPIO_ID) { gpio.id = pin->ucGPIO_ID; gpio.reg = le16_to_cpu(pin->usGpioPin_AIndex) * 4; + gpio.shift = pin->ucGpioPinBitShift; gpio.mask = (1 << pin->ucGpioPinBitShift); gpio.valid = true; break; } + pin = (ATOM_GPIO_PIN_ASSIGNMENT *) + ((u8 *)pin + sizeof(ATOM_GPIO_PIN_ASSIGNMENT)); } } @@ -453,7 +437,9 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev, } /* Fujitsu D3003-S2 board lists DVI-I as DVI-D and VGA */ - if (((dev->pdev->device == 0x9802) || (dev->pdev->device == 0x9806)) && + if (((dev->pdev->device == 0x9802) || + (dev->pdev->device == 0x9805) || + (dev->pdev->device == 0x9806)) && (dev->pdev->subsystem_vendor == 0x1734) && (dev->pdev->subsystem_device == 0x11bd)) { if (*connector_type == DRM_MODE_CONNECTOR_VGA) { @@ -464,11 +450,10 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev, } } - return true; } -const int supported_devices_connector_convert[] = { +static const int supported_devices_connector_convert[] = { DRM_MODE_CONNECTOR_Unknown, DRM_MODE_CONNECTOR_VGA, DRM_MODE_CONNECTOR_DVII, @@ -487,7 +472,7 @@ const int supported_devices_connector_convert[] = { DRM_MODE_CONNECTOR_DisplayPort }; -const uint16_t supported_devices_connector_object_id_convert[] = { +static const uint16_t supported_devices_connector_object_id_convert[] = { CONNECTOR_OBJECT_ID_NONE, CONNECTOR_OBJECT_ID_VGA, CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_I, /* not all boards support DL */ @@ -504,7 +489,7 @@ const uint16_t supported_devices_connector_object_id_convert[] = { CONNECTOR_OBJECT_ID_SVIDEO }; -const int object_connector_convert[] = { +static const int object_connector_convert[] = { DRM_MODE_CONNECTOR_Unknown, DRM_MODE_CONNECTOR_DVII, DRM_MODE_CONNECTOR_DVII, @@ -811,7 +796,7 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev) hpd_record = (ATOM_HPD_INT_RECORD *) record; - gpio = radeon_lookup_gpio(rdev, + gpio = radeon_atombios_lookup_gpio(rdev, hpd_record->ucHPDIntGPIOID); hpd = radeon_atom_get_hpd_info_from_gpio(rdev, &gpio); hpd.plugged_state = hpd_record->ucPlugged_PinState; @@ -854,6 +839,7 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev) radeon_link_encoder_connector(dev); + radeon_setup_mst_connector(dev); return true; } @@ -1120,6 +1106,31 @@ union firmware_info { ATOM_FIRMWARE_INFO_V2_2 info_22; }; +union igp_info { + struct _ATOM_INTEGRATED_SYSTEM_INFO info; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_8 info_8; +}; + +static void radeon_atombios_get_dentist_vco_freq(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo); + union igp_info *igp_info; + u8 frev, crev; + u16 data_offset; + + if (atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + igp_info = (union igp_info *)(mode_info->atom_context->bios + + data_offset); + rdev->clock.vco_freq = + le32_to_cpu(igp_info->info_6.ulDentistVCOFreq); + } +} + bool radeon_atom_get_clock_info(struct drm_device *dev) { struct radeon_device *rdev = dev->dev_private; @@ -1144,7 +1155,7 @@ bool radeon_atom_get_clock_info(struct drm_device *dev) le16_to_cpu(firmware_info->info.usReferenceClock); p1pll->reference_div = 0; - if (crev < 2) + if ((frev < 2) && (crev < 2)) p1pll->pll_out_min = le16_to_cpu(firmware_info->info.usMinPixelClockPLL_Output); else @@ -1153,7 +1164,7 @@ bool radeon_atom_get_clock_info(struct drm_device *dev) p1pll->pll_out_max = le32_to_cpu(firmware_info->info.ulMaxPixelClockPLL_Output); - if (crev >= 4) { + if (((frev < 2) && (crev >= 4)) || (frev >= 2)) { p1pll->lcd_pll_out_min = le16_to_cpu(firmware_info->info_14.usLcdMinPixelClockPLL_Output) * 100; if (p1pll->lcd_pll_out_min == 0) @@ -1244,13 +1255,22 @@ bool radeon_atom_get_clock_info(struct drm_device *dev) rdev->clock.default_dispclk = le32_to_cpu(firmware_info->info_21.ulDefaultDispEngineClkFreq); if (rdev->clock.default_dispclk == 0) { - if (ASIC_IS_DCE5(rdev)) + if (ASIC_IS_DCE6(rdev)) + rdev->clock.default_dispclk = 60000; /* 600 Mhz */ + else if (ASIC_IS_DCE5(rdev)) rdev->clock.default_dispclk = 54000; /* 540 Mhz */ else rdev->clock.default_dispclk = 60000; /* 600 Mhz */ } + /* set a reasonable default for DP */ + if (ASIC_IS_DCE6(rdev) && (rdev->clock.default_dispclk < 53900)) { + DRM_INFO("Changing default dispclk from %dMhz to 600Mhz\n", + rdev->clock.default_dispclk / 100); + rdev->clock.default_dispclk = 60000; + } rdev->clock.dp_extclk = le16_to_cpu(firmware_info->info_21.usUniphyDPModeExtClkFreq); + rdev->clock.current_dispclk = rdev->clock.default_dispclk; } *dcpll = *p1pll; @@ -1262,19 +1282,25 @@ bool radeon_atom_get_clock_info(struct drm_device *dev) rdev->mode_info.firmware_flags = le16_to_cpu(firmware_info->info.usFirmwareCapability.susAccess); + if (ASIC_IS_DCE8(rdev)) + rdev->clock.vco_freq = + le32_to_cpu(firmware_info->info_22.ulGPUPLL_OutputFreq); + else if (ASIC_IS_DCE5(rdev)) + rdev->clock.vco_freq = rdev->clock.current_dispclk; + else if (ASIC_IS_DCE41(rdev)) + radeon_atombios_get_dentist_vco_freq(rdev); + else + rdev->clock.vco_freq = rdev->clock.current_dispclk; + + if (rdev->clock.vco_freq == 0) + rdev->clock.vco_freq = 360000; /* 3.6 GHz */ + return true; } return false; } -union igp_info { - struct _ATOM_INTEGRATED_SYSTEM_INFO info; - struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2; - struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6; - struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7; -}; - bool radeon_atombios_sideport_present(struct radeon_device *rdev) { struct radeon_mode_info *mode_info = &rdev->mode_info; @@ -1365,6 +1391,7 @@ bool radeon_atombios_get_ppll_ss_info(struct radeon_device *rdev, int index = GetIndexIntoMasterTable(DATA, PPLL_SS_Info); uint16_t data_offset, size; struct _ATOM_SPREAD_SPECTRUM_INFO *ss_info; + struct _ATOM_SPREAD_SPECTRUM_ASSIGNMENT *ss_assign; uint8_t frev, crev; int i, num_indices; @@ -1376,18 +1403,21 @@ bool radeon_atombios_get_ppll_ss_info(struct radeon_device *rdev, num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / sizeof(ATOM_SPREAD_SPECTRUM_ASSIGNMENT); - + ss_assign = (struct _ATOM_SPREAD_SPECTRUM_ASSIGNMENT*) + ((u8 *)&ss_info->asSS_Info[0]); for (i = 0; i < num_indices; i++) { - if (ss_info->asSS_Info[i].ucSS_Id == id) { + if (ss_assign->ucSS_Id == id) { ss->percentage = - le16_to_cpu(ss_info->asSS_Info[i].usSpreadSpectrumPercentage); - ss->type = ss_info->asSS_Info[i].ucSpreadSpectrumType; - ss->step = ss_info->asSS_Info[i].ucSS_Step; - ss->delay = ss_info->asSS_Info[i].ucSS_Delay; - ss->range = ss_info->asSS_Info[i].ucSS_Range; - ss->refdiv = ss_info->asSS_Info[i].ucRecommendedRef_Div; + le16_to_cpu(ss_assign->usSpreadSpectrumPercentage); + ss->type = ss_assign->ucSpreadSpectrumType; + ss->step = ss_assign->ucSS_Step; + ss->delay = ss_assign->ucSS_Delay; + ss->range = ss_assign->ucSS_Range; + ss->refdiv = ss_assign->ucRecommendedRef_Div; return true; } + ss_assign = (struct _ATOM_SPREAD_SPECTRUM_ASSIGNMENT*) + ((u8 *)ss_assign + sizeof(struct _ATOM_SPREAD_SPECTRUM_ASSIGNMENT)); } } return false; @@ -1442,6 +1472,22 @@ static void radeon_atombios_get_igp_ss_overrides(struct radeon_device *rdev, break; } break; + case 8: + switch (id) { + case ASIC_INTERNAL_SS_ON_TMDS: + percentage = le16_to_cpu(igp_info->info_8.usDVISSPercentage); + rate = le16_to_cpu(igp_info->info_8.usDVISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_HDMI: + percentage = le16_to_cpu(igp_info->info_8.usHDMISSPercentage); + rate = le16_to_cpu(igp_info->info_8.usHDMISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_LVDS: + percentage = le16_to_cpu(igp_info->info_8.usLvdsSSPercentage); + rate = le16_to_cpu(igp_info->info_8.usLvdsSSpreadRateIn10Hz); + break; + } + break; default: DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev); break; @@ -1459,6 +1505,12 @@ union asic_ss_info { struct _ATOM_ASIC_INTERNAL_SS_INFO_V3 info_3; }; +union asic_ss_assignment { + struct _ATOM_ASIC_SS_ASSIGNMENT v1; + struct _ATOM_ASIC_SS_ASSIGNMENT_V2 v2; + struct _ATOM_ASIC_SS_ASSIGNMENT_V3 v3; +}; + bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev, struct radeon_atom_ss *ss, int id, u32 clock) @@ -1467,9 +1519,19 @@ bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev, int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info); uint16_t data_offset, size; union asic_ss_info *ss_info; + union asic_ss_assignment *ss_assign; uint8_t frev, crev; int i, num_indices; + if (id == ASIC_INTERNAL_MEMORY_SS) { + if (!(rdev->mode_info.firmware_flags & ATOM_BIOS_INFO_MEMORY_CLOCK_SS_SUPPORT)) + return false; + } + if (id == ASIC_INTERNAL_ENGINE_SS) { + if (!(rdev->mode_info.firmware_flags & ATOM_BIOS_INFO_ENGINE_CLOCK_SS_SUPPORT)) + return false; + } + memset(ss, 0, sizeof(struct radeon_atom_ss)); if (atom_parse_data_header(mode_info->atom_context, index, &size, &frev, &crev, &data_offset)) { @@ -1482,45 +1544,68 @@ bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev, num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / sizeof(ATOM_ASIC_SS_ASSIGNMENT); + ss_assign = (union asic_ss_assignment *)((u8 *)&ss_info->info.asSpreadSpectrum[0]); for (i = 0; i < num_indices; i++) { - if ((ss_info->info.asSpreadSpectrum[i].ucClockIndication == id) && - (clock <= le32_to_cpu(ss_info->info.asSpreadSpectrum[i].ulTargetClockRange))) { + if ((ss_assign->v1.ucClockIndication == id) && + (clock <= le32_to_cpu(ss_assign->v1.ulTargetClockRange))) { ss->percentage = - le16_to_cpu(ss_info->info.asSpreadSpectrum[i].usSpreadSpectrumPercentage); - ss->type = ss_info->info.asSpreadSpectrum[i].ucSpreadSpectrumMode; - ss->rate = le16_to_cpu(ss_info->info.asSpreadSpectrum[i].usSpreadRateInKhz); + le16_to_cpu(ss_assign->v1.usSpreadSpectrumPercentage); + ss->type = ss_assign->v1.ucSpreadSpectrumMode; + ss->rate = le16_to_cpu(ss_assign->v1.usSpreadRateInKhz); + ss->percentage_divider = 100; return true; } + ss_assign = (union asic_ss_assignment *) + ((u8 *)ss_assign + sizeof(ATOM_ASIC_SS_ASSIGNMENT)); } break; case 2: num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / sizeof(ATOM_ASIC_SS_ASSIGNMENT_V2); + ss_assign = (union asic_ss_assignment *)((u8 *)&ss_info->info_2.asSpreadSpectrum[0]); for (i = 0; i < num_indices; i++) { - if ((ss_info->info_2.asSpreadSpectrum[i].ucClockIndication == id) && - (clock <= le32_to_cpu(ss_info->info_2.asSpreadSpectrum[i].ulTargetClockRange))) { + if ((ss_assign->v2.ucClockIndication == id) && + (clock <= le32_to_cpu(ss_assign->v2.ulTargetClockRange))) { ss->percentage = - le16_to_cpu(ss_info->info_2.asSpreadSpectrum[i].usSpreadSpectrumPercentage); - ss->type = ss_info->info_2.asSpreadSpectrum[i].ucSpreadSpectrumMode; - ss->rate = le16_to_cpu(ss_info->info_2.asSpreadSpectrum[i].usSpreadRateIn10Hz); + le16_to_cpu(ss_assign->v2.usSpreadSpectrumPercentage); + ss->type = ss_assign->v2.ucSpreadSpectrumMode; + ss->rate = le16_to_cpu(ss_assign->v2.usSpreadRateIn10Hz); + ss->percentage_divider = 100; + if ((crev == 2) && + ((id == ASIC_INTERNAL_ENGINE_SS) || + (id == ASIC_INTERNAL_MEMORY_SS))) + ss->rate /= 100; return true; } + ss_assign = (union asic_ss_assignment *) + ((u8 *)ss_assign + sizeof(ATOM_ASIC_SS_ASSIGNMENT_V2)); } break; case 3: num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / sizeof(ATOM_ASIC_SS_ASSIGNMENT_V3); + ss_assign = (union asic_ss_assignment *)((u8 *)&ss_info->info_3.asSpreadSpectrum[0]); for (i = 0; i < num_indices; i++) { - if ((ss_info->info_3.asSpreadSpectrum[i].ucClockIndication == id) && - (clock <= le32_to_cpu(ss_info->info_3.asSpreadSpectrum[i].ulTargetClockRange))) { + if ((ss_assign->v3.ucClockIndication == id) && + (clock <= le32_to_cpu(ss_assign->v3.ulTargetClockRange))) { ss->percentage = - le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadSpectrumPercentage); - ss->type = ss_info->info_3.asSpreadSpectrum[i].ucSpreadSpectrumMode; - ss->rate = le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadRateIn10Hz); + le16_to_cpu(ss_assign->v3.usSpreadSpectrumPercentage); + ss->type = ss_assign->v3.ucSpreadSpectrumMode; + ss->rate = le16_to_cpu(ss_assign->v3.usSpreadRateIn10Hz); + if (ss_assign->v3.ucSpreadSpectrumMode & + SS_MODE_V3_PERCENTAGE_DIV_BY_1000_MASK) + ss->percentage_divider = 1000; + else + ss->percentage_divider = 100; + if ((id == ASIC_INTERNAL_ENGINE_SS) || + (id == ASIC_INTERNAL_MEMORY_SS)) + ss->rate /= 100; if (rdev->flags & RADEON_IS_IGP) radeon_atombios_get_igp_ss_overrides(rdev, ss, id); return true; } + ss_assign = (union asic_ss_assignment *) + ((u8 *)ss_assign + sizeof(ATOM_ASIC_SS_ASSIGNMENT_V3)); } break; default: @@ -1755,7 +1840,8 @@ bool radeon_atom_get_tv_timings(struct radeon_device *rdev, int index, if (misc & ATOM_DOUBLE_CLOCK_MODE) mode->flags |= DRM_MODE_FLAG_DBLSCAN; - mode->clock = le16_to_cpu(tv_info->aModeTimings[index].usPixelClock) * 10; + mode->crtc_clock = mode->clock = + le16_to_cpu(tv_info->aModeTimings[index].usPixelClock) * 10; if (index == 1) { /* PAL timings appear to have wrong values for totals */ @@ -1798,7 +1884,8 @@ bool radeon_atom_get_tv_timings(struct radeon_device *rdev, int index, if (misc & ATOM_DOUBLE_CLOCK_MODE) mode->flags |= DRM_MODE_FLAG_DBLSCAN; - mode->clock = le16_to_cpu(dtd_timings->usPixClk) * 10; + mode->crtc_clock = mode->clock = + le16_to_cpu(dtd_timings->usPixClk) * 10; break; } return true; @@ -1909,7 +1996,7 @@ static const char *thermal_controller_names[] = { "adm1032", "adm1030", "max6649", - "lm64", + "lm63", /* lm64 */ "f75375", "asc7xxx", }; @@ -1920,7 +2007,7 @@ static const char *pp_lib_thermal_controller_names[] = { "adm1032", "adm1030", "max6649", - "lm64", + "lm63", /* lm64 */ "f75375", "RV6xx", "RV770", @@ -1933,6 +2020,7 @@ static const char *pp_lib_thermal_controller_names[] = { "Northern Islands", "Southern Islands", "lm96163", + "Sea Islands", }; union power_info { @@ -1950,6 +2038,7 @@ union pplib_clock_info { struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; struct _ATOM_PPLIB_SI_CLOCK_INFO si; + struct _ATOM_PPLIB_CI_CLOCK_INFO ci; }; union pplib_power_state { @@ -2067,7 +2156,7 @@ static int radeon_atombios_parse_power_table_1_3(struct radeon_device *rdev) rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_GPIO; rdev->pm.power_state[state_index].clock_info[0].voltage.gpio = - radeon_lookup_gpio(rdev, + radeon_atombios_lookup_gpio(rdev, power_info->info.asPowerPlayInfo[i].ucVoltageDropIndex); if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH) rdev->pm.power_state[state_index].clock_info[0].voltage.active_high = @@ -2103,7 +2192,7 @@ static int radeon_atombios_parse_power_table_1_3(struct radeon_device *rdev) rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_GPIO; rdev->pm.power_state[state_index].clock_info[0].voltage.gpio = - radeon_lookup_gpio(rdev, + radeon_atombios_lookup_gpio(rdev, power_info->info_2.asPowerPlayInfo[i].ucVoltageDropIndex); if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH) rdev->pm.power_state[state_index].clock_info[0].voltage.active_high = @@ -2139,7 +2228,7 @@ static int radeon_atombios_parse_power_table_1_3(struct radeon_device *rdev) rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_GPIO; rdev->pm.power_state[state_index].clock_info[0].voltage.gpio = - radeon_lookup_gpio(rdev, + radeon_atombios_lookup_gpio(rdev, power_info->info_3.asPowerPlayInfo[i].ucVoltageDropIndex); if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH) rdev->pm.power_state[state_index].clock_info[0].voltage.active_high = @@ -2187,6 +2276,14 @@ static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *r /* add the i2c bus for thermal/fan chip */ if (controller->ucType > 0) { + if (controller->ucFanParameters & ATOM_PP_FANPARAMETERS_NOFAN) + rdev->pm.no_fan = true; + rdev->pm.fan_pulses_per_revolution = + controller->ucFanParameters & ATOM_PP_FANPARAMETERS_TACHOMETER_PULSES_PER_REVOLUTION_MASK; + if (rdev->pm.fan_pulses_per_revolution) { + rdev->pm.fan_min_rpm = controller->ucFanMinRPM; + rdev->pm.fan_max_rpm = controller->ucFanMaxRPM; + } if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV6xx) { DRM_INFO("Internal thermal controller %s fan control\n", (controller->ucFanParameters & @@ -2217,19 +2314,41 @@ static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *r (controller->ucFanParameters & ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); rdev->pm.int_thermal_type = THERMAL_TYPE_SI; - } else if ((controller->ucType == - ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO) || - (controller->ucType == - ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL) || - (controller->ucType == - ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL)) { - DRM_INFO("Special thermal controller config\n"); + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_CISLANDS) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + rdev->pm.int_thermal_type = THERMAL_TYPE_CI; + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_KAVERI) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + rdev->pm.int_thermal_type = THERMAL_TYPE_KV; + } else if (controller->ucType == + ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO) { + DRM_INFO("External GPIO thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + rdev->pm.int_thermal_type = THERMAL_TYPE_EXTERNAL_GPIO; + } else if (controller->ucType == + ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL) { + DRM_INFO("ADT7473 with internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + rdev->pm.int_thermal_type = THERMAL_TYPE_ADT7473_WITH_INTERNAL; + } else if (controller->ucType == + ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL) { + DRM_INFO("EMC2103 with internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + rdev->pm.int_thermal_type = THERMAL_TYPE_EMC2103_WITH_INTERNAL; } else if (controller->ucType < ARRAY_SIZE(pp_lib_thermal_controller_names)) { DRM_INFO("Possible %s thermal controller at 0x%02x %s fan control\n", pp_lib_thermal_controller_names[controller->ucType], controller->ucI2cAddress >> 1, (controller->ucFanParameters & ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + rdev->pm.int_thermal_type = THERMAL_TYPE_EXTERNAL; i2c_bus = radeon_lookup_i2c_gpio(rdev, controller->ucI2cLine); rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus); #ifdef notyet @@ -2251,8 +2370,8 @@ static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *r } } -static void radeon_atombios_get_default_voltages(struct radeon_device *rdev, - u16 *vddc, u16 *vddci) +void radeon_atombios_get_default_voltages(struct radeon_device *rdev, + u16 *vddc, u16 *vddci, u16 *mvdd) { struct radeon_mode_info *mode_info = &rdev->mode_info; int index = GetIndexIntoMasterTable(DATA, FirmwareInfo); @@ -2262,6 +2381,7 @@ static void radeon_atombios_get_default_voltages(struct radeon_device *rdev, *vddc = 0; *vddci = 0; + *mvdd = 0; if (atom_parse_data_header(mode_info->atom_context, index, NULL, &frev, &crev, &data_offset)) { @@ -2269,8 +2389,10 @@ static void radeon_atombios_get_default_voltages(struct radeon_device *rdev, (union firmware_info *)(mode_info->atom_context->bios + data_offset); *vddc = le16_to_cpu(firmware_info->info_14.usBootUpVDDCVoltage); - if ((frev == 2) && (crev >= 2)) + if ((frev == 2) && (crev >= 2)) { *vddci = le16_to_cpu(firmware_info->info_22.usBootUpVDDCIVoltage); + *mvdd = le16_to_cpu(firmware_info->info_22.usBootUpMVDDCVoltage); + } } } @@ -2281,9 +2403,9 @@ static void radeon_atombios_parse_pplib_non_clock_info(struct radeon_device *rde int j; u32 misc = le32_to_cpu(non_clock_info->ulCapsAndSettings); u32 misc2 = le16_to_cpu(non_clock_info->usClassification); - u16 vddc, vddci; + u16 vddc, vddci, mvdd; - radeon_atombios_get_default_voltages(rdev, &vddc, &vddci); + radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd); rdev->pm.power_state[state_index].misc = misc; rdev->pm.power_state[state_index].misc2 = misc2; @@ -2319,14 +2441,20 @@ static void radeon_atombios_parse_pplib_non_clock_info(struct radeon_device *rde rdev->pm.default_power_state_index = state_index; rdev->pm.power_state[state_index].default_clock_mode = &rdev->pm.power_state[state_index].clock_info[mode_index - 1]; - if (ASIC_IS_DCE5(rdev) && !(rdev->flags & RADEON_IS_IGP)) { + if ((rdev->family >= CHIP_BARTS) && !(rdev->flags & RADEON_IS_IGP)) { /* NI chips post without MC ucode, so default clocks are strobe mode only */ rdev->pm.default_sclk = rdev->pm.power_state[state_index].clock_info[0].sclk; rdev->pm.default_mclk = rdev->pm.power_state[state_index].clock_info[0].mclk; rdev->pm.default_vddc = rdev->pm.power_state[state_index].clock_info[0].voltage.voltage; rdev->pm.default_vddci = rdev->pm.power_state[state_index].clock_info[0].voltage.vddci; } else { - /* patch the table values with the default slck/mclk from firmware info */ + u16 max_vddci = 0; + + if (ASIC_IS_DCE4(rdev)) + radeon_atom_get_max_voltage(rdev, + SET_VOLTAGE_TYPE_ASIC_VDDCI, + &max_vddci); + /* patch the table values with the default sclk/mclk from firmware info */ for (j = 0; j < mode_index; j++) { rdev->pm.power_state[state_index].clock_info[j].mclk = rdev->clock.default_mclk; @@ -2335,6 +2463,9 @@ static void radeon_atombios_parse_pplib_non_clock_info(struct radeon_device *rde if (vddc) rdev->pm.power_state[state_index].clock_info[j].voltage.voltage = vddc; + if (max_vddci) + rdev->pm.power_state[state_index].clock_info[j].voltage.vddci = + max_vddci; } } } @@ -2357,7 +2488,16 @@ static bool radeon_atombios_parse_pplib_clock_info(struct radeon_device *rdev, sclk |= clock_info->rs780.ucLowEngineClockHigh << 16; rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk; } - } else if (ASIC_IS_DCE6(rdev)) { + } else if (rdev->family >= CHIP_BONAIRE) { + sclk = le16_to_cpu(clock_info->ci.usEngineClockLow); + sclk |= clock_info->ci.ucEngineClockHigh << 16; + mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow); + mclk |= clock_info->ci.ucMemoryClockHigh << 16; + rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk; + rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk; + rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type = + VOLTAGE_NONE; + } else if (rdev->family >= CHIP_TAHITI) { sclk = le16_to_cpu(clock_info->si.usEngineClockLow); sclk |= clock_info->si.ucEngineClockHigh << 16; mclk = le16_to_cpu(clock_info->si.usMemoryClockLow); @@ -2370,7 +2510,7 @@ static bool radeon_atombios_parse_pplib_clock_info(struct radeon_device *rdev, le16_to_cpu(clock_info->si.usVDDC); rdev->pm.power_state[state_index].clock_info[mode_index].voltage.vddci = le16_to_cpu(clock_info->si.usVDDCI); - } else if (ASIC_IS_DCE4(rdev)) { + } else if (rdev->family >= CHIP_CEDAR) { sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow); sclk |= clock_info->evergreen.ucEngineClockHigh << 16; mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow); @@ -2402,6 +2542,10 @@ static bool radeon_atombios_parse_pplib_clock_info(struct radeon_device *rdev, case ATOM_VIRTUAL_VOLTAGE_ID1: case ATOM_VIRTUAL_VOLTAGE_ID2: case ATOM_VIRTUAL_VOLTAGE_ID3: + case ATOM_VIRTUAL_VOLTAGE_ID4: + case ATOM_VIRTUAL_VOLTAGE_ID5: + case ATOM_VIRTUAL_VOLTAGE_ID6: + case ATOM_VIRTUAL_VOLTAGE_ID7: if (radeon_atom_get_max_vddc(rdev, VOLTAGE_TYPE_VDDC, rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage, &vddc) == 0) @@ -2671,6 +2815,184 @@ void radeon_atombios_get_power_modes(struct radeon_device *rdev) rdev->pm.current_vddc = 0; } +union get_clock_dividers { + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS v1; + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V2 v2; + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V3 v3; + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 v4; + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5 v5; + struct _COMPUTE_GPU_CLOCK_INPUT_PARAMETERS_V1_6 v6_in; + struct _COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 v6_out; +}; + +int radeon_atom_get_clock_dividers(struct radeon_device *rdev, + u8 clock_type, + u32 clock, + bool strobe_mode, + struct atom_clock_dividers *dividers) +{ + union get_clock_dividers args; + int index = GetIndexIntoMasterTable(COMMAND, ComputeMemoryEnginePLL); + u8 frev, crev; + + memset(&args, 0, sizeof(args)); + memset(dividers, 0, sizeof(struct atom_clock_dividers)); + + if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev)) + return -EINVAL; + + switch (crev) { + case 1: + /* r4xx, r5xx */ + args.v1.ucAction = clock_type; + args.v1.ulClock = cpu_to_le32(clock); /* 10 khz */ + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->post_div = args.v1.ucPostDiv; + dividers->fb_div = args.v1.ucFbDiv; + dividers->enable_post_div = true; + break; + case 2: + case 3: + case 5: + /* r6xx, r7xx, evergreen, ni, si */ + if (rdev->family <= CHIP_RV770) { + args.v2.ucAction = clock_type; + args.v2.ulClock = cpu_to_le32(clock); /* 10 khz */ + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->post_div = args.v2.ucPostDiv; + dividers->fb_div = le16_to_cpu(args.v2.usFbDiv); + dividers->ref_div = args.v2.ucAction; + if (rdev->family == CHIP_RV770) { + dividers->enable_post_div = (le32_to_cpu(args.v2.ulClock) & (1 << 24)) ? + true : false; + dividers->vco_mode = (le32_to_cpu(args.v2.ulClock) & (1 << 25)) ? 1 : 0; + } else + dividers->enable_post_div = (dividers->fb_div & 1) ? true : false; + } else { + if (clock_type == COMPUTE_ENGINE_PLL_PARAM) { + args.v3.ulClockParams = cpu_to_le32((clock_type << 24) | clock); + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->post_div = args.v3.ucPostDiv; + dividers->enable_post_div = (args.v3.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_PLL_POST_DIV_EN) ? true : false; + dividers->enable_dithen = (args.v3.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_FRACTION_DISABLE) ? false : true; + dividers->whole_fb_div = le16_to_cpu(args.v3.ulFbDiv.usFbDiv); + dividers->frac_fb_div = le16_to_cpu(args.v3.ulFbDiv.usFbDivFrac); + dividers->ref_div = args.v3.ucRefDiv; + dividers->vco_mode = (args.v3.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_MPLL_VCO_MODE) ? 1 : 0; + } else { + /* for SI we use ComputeMemoryClockParam for memory plls */ + if (rdev->family >= CHIP_TAHITI) + return -EINVAL; + args.v5.ulClockParams = cpu_to_le32((clock_type << 24) | clock); + if (strobe_mode) + args.v5.ucInputFlag = ATOM_PLL_INPUT_FLAG_PLL_STROBE_MODE_EN; + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->post_div = args.v5.ucPostDiv; + dividers->enable_post_div = (args.v5.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_PLL_POST_DIV_EN) ? true : false; + dividers->enable_dithen = (args.v5.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_FRACTION_DISABLE) ? false : true; + dividers->whole_fb_div = le16_to_cpu(args.v5.ulFbDiv.usFbDiv); + dividers->frac_fb_div = le16_to_cpu(args.v5.ulFbDiv.usFbDivFrac); + dividers->ref_div = args.v5.ucRefDiv; + dividers->vco_mode = (args.v5.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_MPLL_VCO_MODE) ? 1 : 0; + } + } + break; + case 4: + /* fusion */ + args.v4.ulClock = cpu_to_le32(clock); /* 10 khz */ + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->post_divider = dividers->post_div = args.v4.ucPostDiv; + dividers->real_clock = le32_to_cpu(args.v4.ulClock); + break; + case 6: + /* CI */ + /* COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, COMPUTE_GPUCLK_INPUT_FLAG_SCLK */ + args.v6_in.ulClock.ulComputeClockFlag = clock_type; + args.v6_in.ulClock.ulClockFreq = cpu_to_le32(clock); /* 10 khz */ + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->whole_fb_div = le16_to_cpu(args.v6_out.ulFbDiv.usFbDiv); + dividers->frac_fb_div = le16_to_cpu(args.v6_out.ulFbDiv.usFbDivFrac); + dividers->ref_div = args.v6_out.ucPllRefDiv; + dividers->post_div = args.v6_out.ucPllPostDiv; + dividers->flags = args.v6_out.ucPllCntlFlag; + dividers->real_clock = le32_to_cpu(args.v6_out.ulClock.ulClock); + dividers->post_divider = args.v6_out.ulClock.ucPostDiv; + break; + default: + return -EINVAL; + } + return 0; +} + +int radeon_atom_get_memory_pll_dividers(struct radeon_device *rdev, + u32 clock, + bool strobe_mode, + struct atom_mpll_param *mpll_param) +{ + COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_1 args; + int index = GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam); + u8 frev, crev; + + memset(&args, 0, sizeof(args)); + memset(mpll_param, 0, sizeof(struct atom_mpll_param)); + + if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev)) + return -EINVAL; + + switch (frev) { + case 2: + switch (crev) { + case 1: + /* SI */ + args.ulClock = cpu_to_le32(clock); /* 10 khz */ + args.ucInputFlag = 0; + if (strobe_mode) + args.ucInputFlag |= MPLL_INPUT_FLAG_STROBE_MODE_EN; + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + mpll_param->clkfrac = le16_to_cpu(args.ulFbDiv.usFbDivFrac); + mpll_param->clkf = le16_to_cpu(args.ulFbDiv.usFbDiv); + mpll_param->post_div = args.ucPostDiv; + mpll_param->dll_speed = args.ucDllSpeed; + mpll_param->bwcntl = args.ucBWCntl; + mpll_param->vco_mode = + (args.ucPllCntlFlag & MPLL_CNTL_FLAG_VCO_MODE_MASK); + mpll_param->yclk_sel = + (args.ucPllCntlFlag & MPLL_CNTL_FLAG_BYPASS_DQ_PLL) ? 1 : 0; + mpll_param->qdr = + (args.ucPllCntlFlag & MPLL_CNTL_FLAG_QDR_ENABLE) ? 1 : 0; + mpll_param->half_rate = + (args.ucPllCntlFlag & MPLL_CNTL_FLAG_AD_HALF_RATE) ? 1 : 0; + break; + default: + return -EINVAL; + } + break; + default: + return -EINVAL; + } + return 0; +} + void radeon_atom_set_clock_gating(struct radeon_device *rdev, int enable) { DYNAMIC_CLOCK_GATING_PS_ALLOCATION args; @@ -2724,6 +3046,48 @@ void radeon_atom_set_memory_clock(struct radeon_device *rdev, atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); } +void radeon_atom_set_engine_dram_timings(struct radeon_device *rdev, + u32 eng_clock, u32 mem_clock) +{ + SET_ENGINE_CLOCK_PS_ALLOCATION args; + int index = GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings); + u32 tmp; + + memset(&args, 0, sizeof(args)); + + tmp = eng_clock & SET_CLOCK_FREQ_MASK; + tmp |= (COMPUTE_ENGINE_PLL_PARAM << 24); + + args.ulTargetEngineClock = cpu_to_le32(tmp); + if (mem_clock) + args.sReserved.ulClock = cpu_to_le32(mem_clock & SET_CLOCK_FREQ_MASK); + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void radeon_atom_update_memory_dll(struct radeon_device *rdev, + u32 mem_clock) +{ + u32 args; + int index = GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings); + + args = cpu_to_le32(mem_clock); /* 10 khz */ + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void radeon_atom_set_ac_timing(struct radeon_device *rdev, + u32 mem_clock) +{ + SET_MEMORY_CLOCK_PS_ALLOCATION args; + int index = GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings); + u32 tmp = mem_clock | (COMPUTE_MEMORY_PLL_PARAM << 24); + + args.ulTargetMemoryClock = cpu_to_le32(tmp); /* 10 khz */ + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); +} + union set_voltage { struct _SET_VOLTAGE_PS_ALLOCATION alloc; struct _SET_VOLTAGE_PARAMETERS v1; @@ -2768,8 +3132,8 @@ void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 v atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); } -static int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type, - u16 voltage_id, u16 *voltage) +int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type, + u16 voltage_id, u16 *voltage) { union set_voltage args; int index = GetIndexIntoMasterTable(COMMAND, SetVoltage); @@ -2807,6 +3171,899 @@ static int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type, return 0; } +int radeon_atom_get_leakage_vddc_based_on_leakage_idx(struct radeon_device *rdev, + u16 *voltage, + u16 leakage_idx) +{ + return radeon_atom_get_max_vddc(rdev, VOLTAGE_TYPE_VDDC, leakage_idx, voltage); +} + +int radeon_atom_get_leakage_id_from_vbios(struct radeon_device *rdev, + u16 *leakage_id) +{ + union set_voltage args; + int index = GetIndexIntoMasterTable(COMMAND, SetVoltage); + u8 frev, crev; + + if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev)) + return -EINVAL; + + switch (crev) { + case 3: + case 4: + args.v3.ucVoltageType = 0; + args.v3.ucVoltageMode = ATOM_GET_LEAKAGE_ID; + args.v3.usVoltageLevel = 0; + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + *leakage_id = le16_to_cpu(args.v3.usVoltageLevel); + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + + return 0; +} + +int radeon_atom_get_leakage_vddc_based_on_leakage_params(struct radeon_device *rdev, + u16 *vddc, u16 *vddci, + u16 virtual_voltage_id, + u16 vbios_voltage_id) +{ + int index = GetIndexIntoMasterTable(DATA, ASIC_ProfilingInfo); + u8 frev, crev; + u16 data_offset, size; + int i, j; + ATOM_ASIC_PROFILING_INFO_V2_1 *profile; + u16 *leakage_bin, *vddc_id_buf, *vddc_buf, *vddci_id_buf, *vddci_buf; + + *vddc = 0; + *vddci = 0; + + if (!atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) + return -EINVAL; + + profile = (ATOM_ASIC_PROFILING_INFO_V2_1 *) + (rdev->mode_info.atom_context->bios + data_offset); + + switch (frev) { + case 1: + return -EINVAL; + case 2: + switch (crev) { + case 1: + if (size < sizeof(ATOM_ASIC_PROFILING_INFO_V2_1)) + return -EINVAL; + leakage_bin = (u16 *) + (rdev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usLeakageBinArrayOffset)); + vddc_id_buf = (u16 *) + (rdev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usElbVDDC_IdArrayOffset)); + vddc_buf = (u16 *) + (rdev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usElbVDDC_LevelArrayOffset)); + vddci_id_buf = (u16 *) + (rdev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usElbVDDCI_IdArrayOffset)); + vddci_buf = (u16 *) + (rdev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usElbVDDCI_LevelArrayOffset)); + + if (profile->ucElbVDDC_Num > 0) { + for (i = 0; i < profile->ucElbVDDC_Num; i++) { + if (vddc_id_buf[i] == virtual_voltage_id) { + for (j = 0; j < profile->ucLeakageBinNum; j++) { + if (vbios_voltage_id <= leakage_bin[j]) { + *vddc = vddc_buf[j * profile->ucElbVDDC_Num + i]; + break; + } + } + break; + } + } + } + if (profile->ucElbVDDCI_Num > 0) { + for (i = 0; i < profile->ucElbVDDCI_Num; i++) { + if (vddci_id_buf[i] == virtual_voltage_id) { + for (j = 0; j < profile->ucLeakageBinNum; j++) { + if (vbios_voltage_id <= leakage_bin[j]) { + *vddci = vddci_buf[j * profile->ucElbVDDCI_Num + i]; + break; + } + } + break; + } + } + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + + return 0; +} + +union get_voltage_info { + struct _GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2 in; + struct _GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 evv_out; +}; + +int radeon_atom_get_voltage_evv(struct radeon_device *rdev, + u16 virtual_voltage_id, + u16 *voltage) +{ + int index = GetIndexIntoMasterTable(COMMAND, GetVoltageInfo); + u32 entry_id; + u32 count = rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; + union get_voltage_info args; + + for (entry_id = 0; entry_id < count; entry_id++) { + if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[entry_id].v == + virtual_voltage_id) + break; + } + + if (entry_id >= count) + return -EINVAL; + + args.in.ucVoltageType = VOLTAGE_TYPE_VDDC; + args.in.ucVoltageMode = ATOM_GET_VOLTAGE_EVV_VOLTAGE; + args.in.usVoltageLevel = cpu_to_le16(virtual_voltage_id); + args.in.ulSCLKFreq = + cpu_to_le32(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[entry_id].clk); + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + *voltage = le16_to_cpu(args.evv_out.usVoltageLevel); + + return 0; +} + +int radeon_atom_get_voltage_gpio_settings(struct radeon_device *rdev, + u16 voltage_level, u8 voltage_type, + u32 *gpio_value, u32 *gpio_mask) +{ + union set_voltage args; + int index = GetIndexIntoMasterTable(COMMAND, SetVoltage); + u8 frev, crev; + + if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev)) + return -EINVAL; + + switch (crev) { + case 1: + return -EINVAL; + case 2: + args.v2.ucVoltageType = voltage_type; + args.v2.ucVoltageMode = SET_ASIC_VOLTAGE_MODE_GET_GPIOMASK; + args.v2.usVoltageLevel = cpu_to_le16(voltage_level); + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + *gpio_mask = le32_to_cpu(*(u32 *)&args.v2); + + args.v2.ucVoltageType = voltage_type; + args.v2.ucVoltageMode = SET_ASIC_VOLTAGE_MODE_GET_GPIOVAL; + args.v2.usVoltageLevel = cpu_to_le16(voltage_level); + + atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args); + + *gpio_value = le32_to_cpu(*(u32 *)&args.v2); + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + + return 0; +} + +union voltage_object_info { + struct _ATOM_VOLTAGE_OBJECT_INFO v1; + struct _ATOM_VOLTAGE_OBJECT_INFO_V2 v2; + struct _ATOM_VOLTAGE_OBJECT_INFO_V3_1 v3; +}; + +union voltage_object { + struct _ATOM_VOLTAGE_OBJECT v1; + struct _ATOM_VOLTAGE_OBJECT_V2 v2; + union _ATOM_VOLTAGE_OBJECT_V3 v3; +}; + +static ATOM_VOLTAGE_OBJECT *atom_lookup_voltage_object_v1(ATOM_VOLTAGE_OBJECT_INFO *v1, + u8 voltage_type) +{ + u32 size = le16_to_cpu(v1->sHeader.usStructureSize); + u32 offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO, asVoltageObj[0]); + u8 *start = (u8 *)v1; + + while (offset < size) { + ATOM_VOLTAGE_OBJECT *vo = (ATOM_VOLTAGE_OBJECT *)(start + offset); + if (vo->ucVoltageType == voltage_type) + return vo; + offset += offsetof(ATOM_VOLTAGE_OBJECT, asFormula.ucVIDAdjustEntries) + + vo->asFormula.ucNumOfVoltageEntries; + } + return NULL; +} + +static ATOM_VOLTAGE_OBJECT_V2 *atom_lookup_voltage_object_v2(ATOM_VOLTAGE_OBJECT_INFO_V2 *v2, + u8 voltage_type) +{ + u32 size = le16_to_cpu(v2->sHeader.usStructureSize); + u32 offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO_V2, asVoltageObj[0]); + u8 *start = (u8*)v2; + + while (offset < size) { + ATOM_VOLTAGE_OBJECT_V2 *vo = (ATOM_VOLTAGE_OBJECT_V2 *)(start + offset); + if (vo->ucVoltageType == voltage_type) + return vo; + offset += offsetof(ATOM_VOLTAGE_OBJECT_V2, asFormula.asVIDAdjustEntries) + + (vo->asFormula.ucNumOfVoltageEntries * sizeof(VOLTAGE_LUT_ENTRY)); + } + return NULL; +} + +static ATOM_VOLTAGE_OBJECT_V3 *atom_lookup_voltage_object_v3(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *v3, + u8 voltage_type, u8 voltage_mode) +{ + u32 size = le16_to_cpu(v3->sHeader.usStructureSize); + u32 offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO_V3_1, asVoltageObj[0]); + u8 *start = (u8*)v3; + + while (offset < size) { + ATOM_VOLTAGE_OBJECT_V3 *vo = (ATOM_VOLTAGE_OBJECT_V3 *)(start + offset); + if ((vo->asGpioVoltageObj.sHeader.ucVoltageType == voltage_type) && + (vo->asGpioVoltageObj.sHeader.ucVoltageMode == voltage_mode)) + return vo; + offset += le16_to_cpu(vo->asGpioVoltageObj.sHeader.usSize); + } + return NULL; +} + +bool +radeon_atom_is_voltage_gpio(struct radeon_device *rdev, + u8 voltage_type, u8 voltage_mode) +{ + int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo); + u8 frev, crev; + u16 data_offset, size; + union voltage_object_info *voltage_info; + union voltage_object *voltage_object = NULL; + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + voltage_info = (union voltage_object_info *) + (rdev->mode_info.atom_context->bios + data_offset); + + switch (frev) { + case 1: + case 2: + switch (crev) { + case 1: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v1(&voltage_info->v1, voltage_type); + if (voltage_object && + (voltage_object->v1.asControl.ucVoltageControlId == VOLTAGE_CONTROLLED_BY_GPIO)) + return true; + break; + case 2: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v2(&voltage_info->v2, voltage_type); + if (voltage_object && + (voltage_object->v2.asControl.ucVoltageControlId == VOLTAGE_CONTROLLED_BY_GPIO)) + return true; + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return false; + } + break; + case 3: + switch (crev) { + case 1: + if (atom_lookup_voltage_object_v3(&voltage_info->v3, + voltage_type, voltage_mode)) + return true; + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return false; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return false; + } + + } + return false; +} + +int radeon_atom_get_svi2_info(struct radeon_device *rdev, + u8 voltage_type, + u8 *svd_gpio_id, u8 *svc_gpio_id) +{ + int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo); + u8 frev, crev; + u16 data_offset, size; + union voltage_object_info *voltage_info; + union voltage_object *voltage_object = NULL; + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + voltage_info = (union voltage_object_info *) + (rdev->mode_info.atom_context->bios + data_offset); + + switch (frev) { + case 3: + switch (crev) { + case 1: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v3(&voltage_info->v3, + voltage_type, + VOLTAGE_OBJ_SVID2); + if (voltage_object) { + *svd_gpio_id = voltage_object->v3.asSVID2Obj.ucSVDGpioId; + *svc_gpio_id = voltage_object->v3.asSVID2Obj.ucSVCGpioId; + } else { + return -EINVAL; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + + } + return 0; +} + +int radeon_atom_get_max_voltage(struct radeon_device *rdev, + u8 voltage_type, u16 *max_voltage) +{ + int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo); + u8 frev, crev; + u16 data_offset, size; + union voltage_object_info *voltage_info; + union voltage_object *voltage_object = NULL; + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + voltage_info = (union voltage_object_info *) + (rdev->mode_info.atom_context->bios + data_offset); + + switch (crev) { + case 1: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v1(&voltage_info->v1, voltage_type); + if (voltage_object) { + ATOM_VOLTAGE_FORMULA *formula = + &voltage_object->v1.asFormula; + if (formula->ucFlag & 1) + *max_voltage = + le16_to_cpu(formula->usVoltageBaseLevel) + + formula->ucNumOfVoltageEntries / 2 * + le16_to_cpu(formula->usVoltageStep); + else + *max_voltage = + le16_to_cpu(formula->usVoltageBaseLevel) + + (formula->ucNumOfVoltageEntries - 1) * + le16_to_cpu(formula->usVoltageStep); + return 0; + } + break; + case 2: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v2(&voltage_info->v2, voltage_type); + if (voltage_object) { + ATOM_VOLTAGE_FORMULA_V2 *formula = + &voltage_object->v2.asFormula; + if (formula->ucNumOfVoltageEntries) { + VOLTAGE_LUT_ENTRY *lut = (VOLTAGE_LUT_ENTRY *) + ((u8 *)&formula->asVIDAdjustEntries[0] + + (sizeof(VOLTAGE_LUT_ENTRY) * (formula->ucNumOfVoltageEntries - 1))); + *max_voltage = + le16_to_cpu(lut->usVoltageValue); + return 0; + } + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + + } + return -EINVAL; +} + +int radeon_atom_get_min_voltage(struct radeon_device *rdev, + u8 voltage_type, u16 *min_voltage) +{ + int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo); + u8 frev, crev; + u16 data_offset, size; + union voltage_object_info *voltage_info; + union voltage_object *voltage_object = NULL; + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + voltage_info = (union voltage_object_info *) + (rdev->mode_info.atom_context->bios + data_offset); + + switch (crev) { + case 1: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v1(&voltage_info->v1, voltage_type); + if (voltage_object) { + ATOM_VOLTAGE_FORMULA *formula = + &voltage_object->v1.asFormula; + *min_voltage = + le16_to_cpu(formula->usVoltageBaseLevel); + return 0; + } + break; + case 2: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v2(&voltage_info->v2, voltage_type); + if (voltage_object) { + ATOM_VOLTAGE_FORMULA_V2 *formula = + &voltage_object->v2.asFormula; + if (formula->ucNumOfVoltageEntries) { + *min_voltage = + le16_to_cpu(formula->asVIDAdjustEntries[ + 0 + ].usVoltageValue); + return 0; + } + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + + } + return -EINVAL; +} + +int radeon_atom_get_voltage_step(struct radeon_device *rdev, + u8 voltage_type, u16 *voltage_step) +{ + int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo); + u8 frev, crev; + u16 data_offset, size; + union voltage_object_info *voltage_info; + union voltage_object *voltage_object = NULL; + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + voltage_info = (union voltage_object_info *) + (rdev->mode_info.atom_context->bios + data_offset); + + switch (crev) { + case 1: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v1(&voltage_info->v1, voltage_type); + if (voltage_object) { + ATOM_VOLTAGE_FORMULA *formula = + &voltage_object->v1.asFormula; + if (formula->ucFlag & 1) + *voltage_step = + (le16_to_cpu(formula->usVoltageStep) + 1) / 2; + else + *voltage_step = + le16_to_cpu(formula->usVoltageStep); + return 0; + } + break; + case 2: + return -EINVAL; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + + } + return -EINVAL; +} + +int radeon_atom_round_to_true_voltage(struct radeon_device *rdev, + u8 voltage_type, + u16 nominal_voltage, + u16 *true_voltage) +{ + u16 min_voltage, max_voltage, voltage_step; + + if (radeon_atom_get_max_voltage(rdev, voltage_type, &max_voltage)) + return -EINVAL; + if (radeon_atom_get_min_voltage(rdev, voltage_type, &min_voltage)) + return -EINVAL; + if (radeon_atom_get_voltage_step(rdev, voltage_type, &voltage_step)) + return -EINVAL; + + if (nominal_voltage <= min_voltage) + *true_voltage = min_voltage; + else if (nominal_voltage >= max_voltage) + *true_voltage = max_voltage; + else + *true_voltage = min_voltage + + ((nominal_voltage - min_voltage) / voltage_step) * + voltage_step; + + return 0; +} + +int radeon_atom_get_voltage_table(struct radeon_device *rdev, + u8 voltage_type, u8 voltage_mode, + struct atom_voltage_table *voltage_table) +{ + int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo); + u8 frev, crev; + u16 data_offset, size; + int i, ret; + union voltage_object_info *voltage_info; + union voltage_object *voltage_object = NULL; + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + voltage_info = (union voltage_object_info *) + (rdev->mode_info.atom_context->bios + data_offset); + + switch (frev) { + case 1: + case 2: + switch (crev) { + case 1: + DRM_ERROR("old table version %d, %d\n", frev, crev); + return -EINVAL; + case 2: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v2(&voltage_info->v2, voltage_type); + if (voltage_object) { + ATOM_VOLTAGE_FORMULA_V2 *formula = + &voltage_object->v2.asFormula; + VOLTAGE_LUT_ENTRY *lut; + if (formula->ucNumOfVoltageEntries > MAX_VOLTAGE_ENTRIES) + return -EINVAL; + lut = &formula->asVIDAdjustEntries[0]; + for (i = 0; i < formula->ucNumOfVoltageEntries; i++) { + voltage_table->entries[i].value = + le16_to_cpu(lut->usVoltageValue); + ret = radeon_atom_get_voltage_gpio_settings(rdev, + voltage_table->entries[i].value, + voltage_type, + &voltage_table->entries[i].smio_low, + &voltage_table->mask_low); + if (ret) + return ret; + lut = (VOLTAGE_LUT_ENTRY *) + ((u8 *)lut + sizeof(VOLTAGE_LUT_ENTRY)); + } + voltage_table->count = formula->ucNumOfVoltageEntries; + return 0; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + break; + case 3: + switch (crev) { + case 1: + voltage_object = (union voltage_object *) + atom_lookup_voltage_object_v3(&voltage_info->v3, + voltage_type, voltage_mode); + if (voltage_object) { + ATOM_GPIO_VOLTAGE_OBJECT_V3 *gpio = + &voltage_object->v3.asGpioVoltageObj; + VOLTAGE_LUT_ENTRY_V2 *lut; + if (gpio->ucGpioEntryNum > MAX_VOLTAGE_ENTRIES) + return -EINVAL; + lut = &gpio->asVolGpioLut[0]; + for (i = 0; i < gpio->ucGpioEntryNum; i++) { + voltage_table->entries[i].value = + le16_to_cpu(lut->usVoltageValue); + voltage_table->entries[i].smio_low = + le32_to_cpu(lut->ulVoltageId); + lut = (VOLTAGE_LUT_ENTRY_V2 *) + ((u8 *)lut + sizeof(VOLTAGE_LUT_ENTRY_V2)); + } + voltage_table->mask_low = le32_to_cpu(gpio->ulGpioMaskVal); + voltage_table->count = gpio->ucGpioEntryNum; + voltage_table->phase_delay = gpio->ucPhaseDelay; + return 0; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + } + return -EINVAL; +} + +union vram_info { + struct _ATOM_VRAM_INFO_V3 v1_3; + struct _ATOM_VRAM_INFO_V4 v1_4; + struct _ATOM_VRAM_INFO_HEADER_V2_1 v2_1; +}; + +int radeon_atom_get_memory_info(struct radeon_device *rdev, + u8 module_index, struct atom_memory_info *mem_info) +{ + int index = GetIndexIntoMasterTable(DATA, VRAM_Info); + u8 frev, crev, i; + u16 data_offset, size; + union vram_info *vram_info; + + memset(mem_info, 0, sizeof(struct atom_memory_info)); + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + vram_info = (union vram_info *) + (rdev->mode_info.atom_context->bios + data_offset); + switch (frev) { + case 1: + switch (crev) { + case 3: + /* r6xx */ + if (module_index < vram_info->v1_3.ucNumOfVRAMModule) { + ATOM_VRAM_MODULE_V3 *vram_module = + (ATOM_VRAM_MODULE_V3 *)vram_info->v1_3.aVramInfo; + + for (i = 0; i < module_index; i++) { + if (le16_to_cpu(vram_module->usSize) == 0) + return -EINVAL; + vram_module = (ATOM_VRAM_MODULE_V3 *) + ((u8 *)vram_module + le16_to_cpu(vram_module->usSize)); + } + mem_info->mem_vendor = vram_module->asMemory.ucMemoryVenderID & 0xf; + mem_info->mem_type = vram_module->asMemory.ucMemoryType & 0xf0; + } else + return -EINVAL; + break; + case 4: + /* r7xx, evergreen */ + if (module_index < vram_info->v1_4.ucNumOfVRAMModule) { + ATOM_VRAM_MODULE_V4 *vram_module = + (ATOM_VRAM_MODULE_V4 *)vram_info->v1_4.aVramInfo; + + for (i = 0; i < module_index; i++) { + if (le16_to_cpu(vram_module->usModuleSize) == 0) + return -EINVAL; + vram_module = (ATOM_VRAM_MODULE_V4 *) + ((u8 *)vram_module + le16_to_cpu(vram_module->usModuleSize)); + } + mem_info->mem_vendor = vram_module->ucMemoryVenderID & 0xf; + mem_info->mem_type = vram_module->ucMemoryType & 0xf0; + } else + return -EINVAL; + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + break; + case 2: + switch (crev) { + case 1: + /* ni */ + if (module_index < vram_info->v2_1.ucNumOfVRAMModule) { + ATOM_VRAM_MODULE_V7 *vram_module = + (ATOM_VRAM_MODULE_V7 *)vram_info->v2_1.aVramInfo; + + for (i = 0; i < module_index; i++) { + if (le16_to_cpu(vram_module->usModuleSize) == 0) + return -EINVAL; + vram_module = (ATOM_VRAM_MODULE_V7 *) + ((u8 *)vram_module + le16_to_cpu(vram_module->usModuleSize)); + } + mem_info->mem_vendor = vram_module->ucMemoryVenderID & 0xf; + mem_info->mem_type = vram_module->ucMemoryType & 0xf0; + } else + return -EINVAL; + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + return 0; + } + return -EINVAL; +} + +int radeon_atom_get_mclk_range_table(struct radeon_device *rdev, + bool gddr5, u8 module_index, + struct atom_memory_clock_range_table *mclk_range_table) +{ + int index = GetIndexIntoMasterTable(DATA, VRAM_Info); + u8 frev, crev, i; + u16 data_offset, size; + union vram_info *vram_info; + u32 mem_timing_size = gddr5 ? + sizeof(ATOM_MEMORY_TIMING_FORMAT_V2) : sizeof(ATOM_MEMORY_TIMING_FORMAT); + + memset(mclk_range_table, 0, sizeof(struct atom_memory_clock_range_table)); + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + vram_info = (union vram_info *) + (rdev->mode_info.atom_context->bios + data_offset); + switch (frev) { + case 1: + switch (crev) { + case 3: + DRM_ERROR("old table version %d, %d\n", frev, crev); + return -EINVAL; + case 4: + /* r7xx, evergreen */ + if (module_index < vram_info->v1_4.ucNumOfVRAMModule) { + ATOM_VRAM_MODULE_V4 *vram_module = + (ATOM_VRAM_MODULE_V4 *)vram_info->v1_4.aVramInfo; + ATOM_MEMORY_TIMING_FORMAT *format; + + for (i = 0; i < module_index; i++) { + if (le16_to_cpu(vram_module->usModuleSize) == 0) + return -EINVAL; + vram_module = (ATOM_VRAM_MODULE_V4 *) + ((u8 *)vram_module + le16_to_cpu(vram_module->usModuleSize)); + } + mclk_range_table->num_entries = (u8) + ((le16_to_cpu(vram_module->usModuleSize) - offsetof(ATOM_VRAM_MODULE_V4, asMemTiming)) / + mem_timing_size); + format = &vram_module->asMemTiming[0]; + for (i = 0; i < mclk_range_table->num_entries; i++) { + mclk_range_table->mclk[i] = le32_to_cpu(format->ulClkRange); + format = (ATOM_MEMORY_TIMING_FORMAT *) + ((u8 *)format + mem_timing_size); + } + } else + return -EINVAL; + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + break; + case 2: + DRM_ERROR("new table version %d, %d\n", frev, crev); + return -EINVAL; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + return 0; + } + return -EINVAL; +} + +#define MEM_ID_MASK 0xff000000 +#define MEM_ID_SHIFT 24 +#define CLOCK_RANGE_MASK 0x00ffffff +#define CLOCK_RANGE_SHIFT 0 +#define LOW_NIBBLE_MASK 0xf +#define DATA_EQU_PREV 0 +#define DATA_FROM_TABLE 4 + +int radeon_atom_init_mc_reg_table(struct radeon_device *rdev, + u8 module_index, + struct atom_mc_reg_table *reg_table) +{ + int index = GetIndexIntoMasterTable(DATA, VRAM_Info); + u8 frev, crev, num_entries, t_mem_id, num_ranges = 0; + u32 i = 0, j; + u16 data_offset, size; + union vram_info *vram_info; + + memset(reg_table, 0, sizeof(struct atom_mc_reg_table)); + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + vram_info = (union vram_info *) + (rdev->mode_info.atom_context->bios + data_offset); + switch (frev) { + case 1: + DRM_ERROR("old table version %d, %d\n", frev, crev); + return -EINVAL; + case 2: + switch (crev) { + case 1: + if (module_index < vram_info->v2_1.ucNumOfVRAMModule) { + ATOM_INIT_REG_BLOCK *reg_block = + (ATOM_INIT_REG_BLOCK *) + ((u8 *)vram_info + le16_to_cpu(vram_info->v2_1.usMemClkPatchTblOffset)); + ATOM_MEMORY_SETTING_DATA_BLOCK *reg_data = + (ATOM_MEMORY_SETTING_DATA_BLOCK *) + ((u8 *)reg_block + (2 * sizeof(u16)) + + le16_to_cpu(reg_block->usRegIndexTblSize)); + ATOM_INIT_REG_INDEX_FORMAT *format = ®_block->asRegIndexBuf[0]; + num_entries = (u8)((le16_to_cpu(reg_block->usRegIndexTblSize)) / + sizeof(ATOM_INIT_REG_INDEX_FORMAT)) - 1; + if (num_entries > VBIOS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + while (i < num_entries) { + if (format->ucPreRegDataLength & ACCESS_PLACEHOLDER) + break; + reg_table->mc_reg_address[i].s1 = + (u16)(le16_to_cpu(format->usRegIndex)); + reg_table->mc_reg_address[i].pre_reg_data = + (u8)(format->ucPreRegDataLength); + i++; + format = (ATOM_INIT_REG_INDEX_FORMAT *) + ((u8 *)format + sizeof(ATOM_INIT_REG_INDEX_FORMAT)); + } + reg_table->last = i; + while ((le32_to_cpu(*(u32 *)reg_data) != END_OF_REG_DATA_BLOCK) && + (num_ranges < VBIOS_MAX_AC_TIMING_ENTRIES)) { + t_mem_id = (u8)((le32_to_cpu(*(u32 *)reg_data) & MEM_ID_MASK) + >> MEM_ID_SHIFT); + if (module_index == t_mem_id) { + reg_table->mc_reg_table_entry[num_ranges].mclk_max = + (u32)((le32_to_cpu(*(u32 *)reg_data) & CLOCK_RANGE_MASK) + >> CLOCK_RANGE_SHIFT); + for (i = 0, j = 1; i < reg_table->last; i++) { + if ((reg_table->mc_reg_address[i].pre_reg_data & LOW_NIBBLE_MASK) == DATA_FROM_TABLE) { + reg_table->mc_reg_table_entry[num_ranges].mc_data[i] = + (u32)le32_to_cpu(*((u32 *)reg_data + j)); + j++; + } else if ((reg_table->mc_reg_address[i].pre_reg_data & LOW_NIBBLE_MASK) == DATA_EQU_PREV) { + reg_table->mc_reg_table_entry[num_ranges].mc_data[i] = + reg_table->mc_reg_table_entry[num_ranges].mc_data[i - 1]; + } + } + num_ranges++; + } + reg_data = (ATOM_MEMORY_SETTING_DATA_BLOCK *) + ((u8 *)reg_data + le16_to_cpu(reg_block->usRegDataBlkSize)); + } + if (le32_to_cpu(*(u32 *)reg_data) != END_OF_REG_DATA_BLOCK) + return -EINVAL; + reg_table->num_entries = num_ranges; + } else + return -EINVAL; + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + return 0; + } + return -EINVAL; +} + void radeon_atom_initialize_bios_scratch_regs(struct drm_device *dev) { struct radeon_device *rdev = dev->dev_private; diff --git a/sys/dev/pci/drm/radeon/radeon_audio.c b/sys/dev/pci/drm/radeon/radeon_audio.c new file mode 100644 index 00000000000..6c9d95c1990 --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_audio.c @@ -0,0 +1,788 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Slava Grigorev + */ + +#include +#include +#include "radeon.h" +#include "atom.h" +#include "radeon_audio.h" + +void r600_audio_enable(struct radeon_device *rdev, struct r600_audio_pin *pin, + u8 enable_mask); +void dce4_audio_enable(struct radeon_device *rdev, struct r600_audio_pin *pin, + u8 enable_mask); +void dce6_audio_enable(struct radeon_device *rdev, struct r600_audio_pin *pin, + u8 enable_mask); +u32 dce6_endpoint_rreg(struct radeon_device *rdev, u32 offset, u32 reg); +void dce6_endpoint_wreg(struct radeon_device *rdev, + u32 offset, u32 reg, u32 v); +void dce3_2_afmt_write_sad_regs(struct drm_encoder *encoder, + struct cea_sad *sads, int sad_count); +void evergreen_hdmi_write_sad_regs(struct drm_encoder *encoder, + struct cea_sad *sads, int sad_count); +void dce6_afmt_write_sad_regs(struct drm_encoder *encoder, + struct cea_sad *sads, int sad_count); +void dce3_2_afmt_hdmi_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count); +void dce3_2_afmt_dp_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count); +void dce4_afmt_hdmi_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count); +void dce4_afmt_dp_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count); +void dce6_afmt_hdmi_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count); +void dce6_afmt_dp_write_speaker_allocation(struct drm_encoder *encoder, + u8 *sadb, int sad_count); +void dce4_afmt_write_latency_fields(struct drm_encoder *encoder, + struct drm_connector *connector, struct drm_display_mode *mode); +void dce6_afmt_write_latency_fields(struct drm_encoder *encoder, + struct drm_connector *connector, struct drm_display_mode *mode); +struct r600_audio_pin* r600_audio_get_pin(struct radeon_device *rdev); +struct r600_audio_pin* dce6_audio_get_pin(struct radeon_device *rdev); +void dce6_afmt_select_pin(struct drm_encoder *encoder); +void r600_hdmi_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock); +void dce3_2_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock); +void dce4_hdmi_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock); +void dce4_dp_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock); +void dce6_hdmi_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock); +void dce6_dp_audio_set_dto(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock); +void r600_set_avi_packet(struct radeon_device *rdev, u32 offset, + unsigned char *buffer, size_t size); +void evergreen_set_avi_packet(struct radeon_device *rdev, u32 offset, + unsigned char *buffer, size_t size); +void r600_hdmi_update_acr(struct drm_encoder *encoder, long offset, + const struct radeon_hdmi_acr *acr); +void dce3_2_hdmi_update_acr(struct drm_encoder *encoder, long offset, + const struct radeon_hdmi_acr *acr); +void evergreen_hdmi_update_acr(struct drm_encoder *encoder, long offset, + const struct radeon_hdmi_acr *acr); +void r600_set_vbi_packet(struct drm_encoder *encoder, u32 offset); +void dce4_set_vbi_packet(struct drm_encoder *encoder, u32 offset); +void dce4_hdmi_set_color_depth(struct drm_encoder *encoder, + u32 offset, int bpc); +void r600_set_audio_packet(struct drm_encoder *encoder, u32 offset); +void dce3_2_set_audio_packet(struct drm_encoder *encoder, u32 offset); +void dce4_set_audio_packet(struct drm_encoder *encoder, u32 offset); +void r600_set_mute(struct drm_encoder *encoder, u32 offset, bool mute); +void dce3_2_set_mute(struct drm_encoder *encoder, u32 offset, bool mute); +void dce4_set_mute(struct drm_encoder *encoder, u32 offset, bool mute); +static void radeon_audio_hdmi_mode_set(struct drm_encoder *encoder, + struct drm_display_mode *mode); +static void radeon_audio_dp_mode_set(struct drm_encoder *encoder, + struct drm_display_mode *mode); +void r600_hdmi_enable(struct drm_encoder *encoder, bool enable); +void evergreen_hdmi_enable(struct drm_encoder *encoder, bool enable); +void evergreen_dp_enable(struct drm_encoder *encoder, bool enable); + +static const u32 pin_offsets[7] = +{ + (0x5e00 - 0x5e00), + (0x5e18 - 0x5e00), + (0x5e30 - 0x5e00), + (0x5e48 - 0x5e00), + (0x5e60 - 0x5e00), + (0x5e78 - 0x5e00), + (0x5e90 - 0x5e00), +}; + +static u32 radeon_audio_rreg(struct radeon_device *rdev, u32 offset, u32 reg) +{ + return RREG32(reg); +} + +static void radeon_audio_wreg(struct radeon_device *rdev, u32 offset, + u32 reg, u32 v) +{ + WREG32(reg, v); +} + +static struct radeon_audio_basic_funcs r600_funcs = { + .endpoint_rreg = radeon_audio_rreg, + .endpoint_wreg = radeon_audio_wreg, + .enable = r600_audio_enable, +}; + +static struct radeon_audio_basic_funcs dce32_funcs = { + .endpoint_rreg = radeon_audio_rreg, + .endpoint_wreg = radeon_audio_wreg, + .enable = r600_audio_enable, +}; + +static struct radeon_audio_basic_funcs dce4_funcs = { + .endpoint_rreg = radeon_audio_rreg, + .endpoint_wreg = radeon_audio_wreg, + .enable = dce4_audio_enable, +}; + +static struct radeon_audio_basic_funcs dce6_funcs = { + .endpoint_rreg = dce6_endpoint_rreg, + .endpoint_wreg = dce6_endpoint_wreg, + .enable = dce6_audio_enable, +}; + +static struct radeon_audio_funcs r600_hdmi_funcs = { + .get_pin = r600_audio_get_pin, + .set_dto = r600_hdmi_audio_set_dto, + .update_acr = r600_hdmi_update_acr, + .set_vbi_packet = r600_set_vbi_packet, + .set_avi_packet = r600_set_avi_packet, + .set_audio_packet = r600_set_audio_packet, + .set_mute = r600_set_mute, + .mode_set = radeon_audio_hdmi_mode_set, + .dpms = r600_hdmi_enable, +}; + +static struct radeon_audio_funcs dce32_hdmi_funcs = { + .get_pin = r600_audio_get_pin, + .write_sad_regs = dce3_2_afmt_write_sad_regs, + .write_speaker_allocation = dce3_2_afmt_hdmi_write_speaker_allocation, + .set_dto = dce3_2_audio_set_dto, + .update_acr = dce3_2_hdmi_update_acr, + .set_vbi_packet = r600_set_vbi_packet, + .set_avi_packet = r600_set_avi_packet, + .set_audio_packet = dce3_2_set_audio_packet, + .set_mute = dce3_2_set_mute, + .mode_set = radeon_audio_hdmi_mode_set, + .dpms = r600_hdmi_enable, +}; + +static struct radeon_audio_funcs dce32_dp_funcs = { + .get_pin = r600_audio_get_pin, + .write_sad_regs = dce3_2_afmt_write_sad_regs, + .write_speaker_allocation = dce3_2_afmt_dp_write_speaker_allocation, + .set_dto = dce3_2_audio_set_dto, + .set_avi_packet = r600_set_avi_packet, + .set_audio_packet = dce3_2_set_audio_packet, +}; + +static struct radeon_audio_funcs dce4_hdmi_funcs = { + .get_pin = r600_audio_get_pin, + .write_sad_regs = evergreen_hdmi_write_sad_regs, + .write_speaker_allocation = dce4_afmt_hdmi_write_speaker_allocation, + .write_latency_fields = dce4_afmt_write_latency_fields, + .set_dto = dce4_hdmi_audio_set_dto, + .update_acr = evergreen_hdmi_update_acr, + .set_vbi_packet = dce4_set_vbi_packet, + .set_color_depth = dce4_hdmi_set_color_depth, + .set_avi_packet = evergreen_set_avi_packet, + .set_audio_packet = dce4_set_audio_packet, + .set_mute = dce4_set_mute, + .mode_set = radeon_audio_hdmi_mode_set, + .dpms = evergreen_hdmi_enable, +}; + +static struct radeon_audio_funcs dce4_dp_funcs = { + .get_pin = r600_audio_get_pin, + .write_sad_regs = evergreen_hdmi_write_sad_regs, + .write_speaker_allocation = dce4_afmt_dp_write_speaker_allocation, + .write_latency_fields = dce4_afmt_write_latency_fields, + .set_dto = dce4_dp_audio_set_dto, + .set_avi_packet = evergreen_set_avi_packet, + .set_audio_packet = dce4_set_audio_packet, + .mode_set = radeon_audio_dp_mode_set, + .dpms = evergreen_dp_enable, +}; + +static struct radeon_audio_funcs dce6_hdmi_funcs = { + .select_pin = dce6_afmt_select_pin, + .get_pin = dce6_audio_get_pin, + .write_sad_regs = dce6_afmt_write_sad_regs, + .write_speaker_allocation = dce6_afmt_hdmi_write_speaker_allocation, + .write_latency_fields = dce6_afmt_write_latency_fields, + .set_dto = dce6_hdmi_audio_set_dto, + .update_acr = evergreen_hdmi_update_acr, + .set_vbi_packet = dce4_set_vbi_packet, + .set_color_depth = dce4_hdmi_set_color_depth, + .set_avi_packet = evergreen_set_avi_packet, + .set_audio_packet = dce4_set_audio_packet, + .set_mute = dce4_set_mute, + .mode_set = radeon_audio_hdmi_mode_set, + .dpms = evergreen_hdmi_enable, +}; + +static struct radeon_audio_funcs dce6_dp_funcs = { + .select_pin = dce6_afmt_select_pin, + .get_pin = dce6_audio_get_pin, + .write_sad_regs = dce6_afmt_write_sad_regs, + .write_speaker_allocation = dce6_afmt_dp_write_speaker_allocation, + .write_latency_fields = dce6_afmt_write_latency_fields, + .set_dto = dce6_dp_audio_set_dto, + .set_avi_packet = evergreen_set_avi_packet, + .set_audio_packet = dce4_set_audio_packet, + .mode_set = radeon_audio_dp_mode_set, + .dpms = evergreen_dp_enable, +}; + +static void radeon_audio_enable(struct radeon_device *rdev, + struct r600_audio_pin *pin, u8 enable_mask) +{ + struct drm_encoder *encoder; + struct radeon_encoder *radeon_encoder; + struct radeon_encoder_atom_dig *dig; + int pin_count = 0; + + if (!pin) + return; + + if (rdev->mode_info.mode_config_initialized) { + list_for_each_entry(encoder, &rdev->ddev->mode_config.encoder_list, head) { + if (radeon_encoder_is_digital(encoder)) { + radeon_encoder = to_radeon_encoder(encoder); + dig = radeon_encoder->enc_priv; + if (dig->pin == pin) + pin_count++; + } + } + + if ((pin_count > 1) && (enable_mask == 0)) + return; + } + + if (rdev->audio.funcs->enable) + rdev->audio.funcs->enable(rdev, pin, enable_mask); +} + +static void radeon_audio_interface_init(struct radeon_device *rdev) +{ + if (ASIC_IS_DCE6(rdev)) { + rdev->audio.funcs = &dce6_funcs; + rdev->audio.hdmi_funcs = &dce6_hdmi_funcs; + rdev->audio.dp_funcs = &dce6_dp_funcs; + } else if (ASIC_IS_DCE4(rdev)) { + rdev->audio.funcs = &dce4_funcs; + rdev->audio.hdmi_funcs = &dce4_hdmi_funcs; + rdev->audio.dp_funcs = &dce4_dp_funcs; + } else if (ASIC_IS_DCE32(rdev)) { + rdev->audio.funcs = &dce32_funcs; + rdev->audio.hdmi_funcs = &dce32_hdmi_funcs; + rdev->audio.dp_funcs = &dce32_dp_funcs; + } else { + rdev->audio.funcs = &r600_funcs; + rdev->audio.hdmi_funcs = &r600_hdmi_funcs; + rdev->audio.dp_funcs = 0; + } +} + +static int radeon_audio_chipset_supported(struct radeon_device *rdev) +{ + return ASIC_IS_DCE2(rdev) && !ASIC_IS_NODCE(rdev); +} + +int radeon_audio_init(struct radeon_device *rdev) +{ + int i; + + if (!radeon_audio || !radeon_audio_chipset_supported(rdev)) + return 0; + + rdev->audio.enabled = true; + + if (ASIC_IS_DCE83(rdev)) /* KB: 2 streams, 3 endpoints */ + rdev->audio.num_pins = 3; + else if (ASIC_IS_DCE81(rdev)) /* KV: 4 streams, 7 endpoints */ + rdev->audio.num_pins = 7; + else if (ASIC_IS_DCE8(rdev)) /* BN/HW: 6 streams, 7 endpoints */ + rdev->audio.num_pins = 7; + else if (ASIC_IS_DCE64(rdev)) /* OL: 2 streams, 2 endpoints */ + rdev->audio.num_pins = 2; + else if (ASIC_IS_DCE61(rdev)) /* TN: 4 streams, 6 endpoints */ + rdev->audio.num_pins = 6; + else if (ASIC_IS_DCE6(rdev)) /* SI: 6 streams, 6 endpoints */ + rdev->audio.num_pins = 6; + else + rdev->audio.num_pins = 1; + + for (i = 0; i < rdev->audio.num_pins; i++) { + rdev->audio.pin[i].channels = -1; + rdev->audio.pin[i].rate = -1; + rdev->audio.pin[i].bits_per_sample = -1; + rdev->audio.pin[i].status_bits = 0; + rdev->audio.pin[i].category_code = 0; + rdev->audio.pin[i].connected = false; + rdev->audio.pin[i].offset = pin_offsets[i]; + rdev->audio.pin[i].id = i; + } + + radeon_audio_interface_init(rdev); + + /* disable audio. it will be set up later */ + for (i = 0; i < rdev->audio.num_pins; i++) + radeon_audio_enable(rdev, &rdev->audio.pin[i], 0); + + return 0; +} + +u32 radeon_audio_endpoint_rreg(struct radeon_device *rdev, u32 offset, u32 reg) +{ + if (rdev->audio.funcs->endpoint_rreg) + return rdev->audio.funcs->endpoint_rreg(rdev, offset, reg); + + return 0; +} + +void radeon_audio_endpoint_wreg(struct radeon_device *rdev, u32 offset, + u32 reg, u32 v) +{ + if (rdev->audio.funcs->endpoint_wreg) + rdev->audio.funcs->endpoint_wreg(rdev, offset, reg, v); +} + +static void radeon_audio_write_sad_regs(struct drm_encoder *encoder) +{ + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct cea_sad *sads; + int sad_count; + + if (!connector) + return; + + sad_count = drm_edid_to_sad(radeon_connector_edid(connector), &sads); + if (sad_count <= 0) { + DRM_ERROR("Couldn't read SADs: %d\n", sad_count); + return; + } + BUG_ON(!sads); + + if (radeon_encoder->audio && radeon_encoder->audio->write_sad_regs) + radeon_encoder->audio->write_sad_regs(encoder, sads, sad_count); + + kfree(sads); +} + +static void radeon_audio_write_speaker_allocation(struct drm_encoder *encoder) +{ + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + u8 *sadb = NULL; + int sad_count; + + if (!connector) + return; + + sad_count = drm_edid_to_speaker_allocation(radeon_connector_edid(connector), + &sadb); + if (sad_count < 0) { + DRM_DEBUG("Couldn't read Speaker Allocation Data Block: %d\n", + sad_count); + sad_count = 0; + } + + if (radeon_encoder->audio && radeon_encoder->audio->write_speaker_allocation) + radeon_encoder->audio->write_speaker_allocation(encoder, sadb, sad_count); + + kfree(sadb); +} + +static void radeon_audio_write_latency_fields(struct drm_encoder *encoder, + struct drm_display_mode *mode) +{ + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + + if (!connector) + return; + + if (radeon_encoder->audio && radeon_encoder->audio->write_latency_fields) + radeon_encoder->audio->write_latency_fields(encoder, connector, mode); +} + +struct r600_audio_pin* radeon_audio_get_pin(struct drm_encoder *encoder) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + + if (radeon_encoder->audio && radeon_encoder->audio->get_pin) + return radeon_encoder->audio->get_pin(rdev); + + return NULL; +} + +static void radeon_audio_select_pin(struct drm_encoder *encoder) +{ + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + + if (radeon_encoder->audio && radeon_encoder->audio->select_pin) + radeon_encoder->audio->select_pin(encoder); +} + +void radeon_audio_detect(struct drm_connector *connector, + struct drm_encoder *encoder, + enum drm_connector_status status) +{ + struct drm_device *dev = connector->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig; + + if (!radeon_audio_chipset_supported(rdev)) + return; + + if (!radeon_encoder_is_digital(encoder)) + return; + + dig = radeon_encoder->enc_priv; + + if (status == connector_status_connected) { + if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) { + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + + if (radeon_dp_getsinktype(radeon_connector) == + CONNECTOR_OBJECT_ID_DISPLAYPORT) + radeon_encoder->audio = rdev->audio.dp_funcs; + else + radeon_encoder->audio = rdev->audio.hdmi_funcs; + } else { + radeon_encoder->audio = rdev->audio.hdmi_funcs; + } + + if (drm_detect_monitor_audio(radeon_connector_edid(connector))) { + if (!dig->pin) + dig->pin = radeon_audio_get_pin(encoder); + radeon_audio_enable(rdev, dig->pin, 0xf); + } else { + radeon_audio_enable(rdev, dig->pin, 0); + dig->pin = NULL; + } + } else { + radeon_audio_enable(rdev, dig->pin, 0); + dig->pin = NULL; + } +} + +void radeon_audio_fini(struct radeon_device *rdev) +{ + int i; + + if (!rdev->audio.enabled) + return; + + for (i = 0; i < rdev->audio.num_pins; i++) + radeon_audio_enable(rdev, &rdev->audio.pin[i], 0); + + rdev->audio.enabled = false; +} + +static void radeon_audio_set_dto(struct drm_encoder *encoder, unsigned int clock) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_crtc *crtc = to_radeon_crtc(encoder->crtc); + + if (radeon_encoder->audio && radeon_encoder->audio->set_dto) + radeon_encoder->audio->set_dto(rdev, crtc, clock); +} + +static int radeon_audio_set_avi_packet(struct drm_encoder *encoder, + struct drm_display_mode *mode) +{ + struct radeon_device *rdev = encoder->dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + u8 buffer[HDMI_INFOFRAME_HEADER_SIZE + HDMI_AVI_INFOFRAME_SIZE]; + struct hdmi_avi_infoframe frame; + int err; + + if (!connector) + return -EINVAL; + + err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode); + if (err < 0) { + DRM_ERROR("failed to setup AVI infoframe: %d\n", err); + return err; + } + + if (radeon_encoder->output_csc != RADEON_OUTPUT_CSC_BYPASS) { + if (drm_rgb_quant_range_selectable(radeon_connector_edid(connector))) { + if (radeon_encoder->output_csc == RADEON_OUTPUT_CSC_TVRGB) + frame.quantization_range = HDMI_QUANTIZATION_RANGE_LIMITED; + else + frame.quantization_range = HDMI_QUANTIZATION_RANGE_FULL; + } else { + frame.quantization_range = HDMI_QUANTIZATION_RANGE_DEFAULT; + } + } + + err = hdmi_avi_infoframe_pack(&frame, buffer, sizeof(buffer)); + if (err < 0) { + DRM_ERROR("failed to pack AVI infoframe: %d\n", err); + return err; + } + + if (dig && dig->afmt && radeon_encoder->audio && + radeon_encoder->audio->set_avi_packet) + radeon_encoder->audio->set_avi_packet(rdev, dig->afmt->offset, + buffer, sizeof(buffer)); + + return 0; +} + +/* + * calculate CTS and N values if they are not found in the table + */ +static void radeon_audio_calc_cts(unsigned int clock, int *CTS, int *N, int freq) +{ + int n, cts; + unsigned long div, mul; + + /* Safe, but overly large values */ + n = 128 * freq; + cts = clock * 1000; + + /* Smallest valid fraction */ + div = gcd(n, cts); + + n /= div; + cts /= div; + + /* + * The optimal N is 128*freq/1000. Calculate the closest larger + * value that doesn't truncate any bits. + */ + mul = ((128*freq/1000) + (n-1))/n; + + n *= mul; + cts *= mul; + + /* Check that we are in spec (not always possible) */ + if (n < (128*freq/1500)) + printk(KERN_WARNING "Calculated ACR N value is too small. You may experience audio problems.\n"); + if (n > (128*freq/300)) + printk(KERN_WARNING "Calculated ACR N value is too large. You may experience audio problems.\n"); + + *N = n; + *CTS = cts; + + DRM_DEBUG("Calculated ACR timing N=%d CTS=%d for frequency %d\n", + *N, *CTS, freq); +} + +static const struct radeon_hdmi_acr* radeon_audio_acr(unsigned int clock) +{ + static struct radeon_hdmi_acr res; + u8 i; + + static const struct radeon_hdmi_acr hdmi_predefined_acr[] = { + /* 32kHz 44.1kHz 48kHz */ + /* Clock N CTS N CTS N CTS */ + { 25175, 4096, 25175, 28224, 125875, 6144, 25175 }, /* 25,20/1.001 MHz */ + { 25200, 4096, 25200, 6272, 28000, 6144, 25200 }, /* 25.20 MHz */ + { 27000, 4096, 27000, 6272, 30000, 6144, 27000 }, /* 27.00 MHz */ + { 27027, 4096, 27027, 6272, 30030, 6144, 27027 }, /* 27.00*1.001 MHz */ + { 54000, 4096, 54000, 6272, 60000, 6144, 54000 }, /* 54.00 MHz */ + { 54054, 4096, 54054, 6272, 60060, 6144, 54054 }, /* 54.00*1.001 MHz */ + { 74176, 4096, 74176, 5733, 75335, 6144, 74176 }, /* 74.25/1.001 MHz */ + { 74250, 4096, 74250, 6272, 82500, 6144, 74250 }, /* 74.25 MHz */ + { 148352, 4096, 148352, 5733, 150670, 6144, 148352 }, /* 148.50/1.001 MHz */ + { 148500, 4096, 148500, 6272, 165000, 6144, 148500 }, /* 148.50 MHz */ + }; + + /* Precalculated values for common clocks */ + for (i = 0; i < ARRAY_SIZE(hdmi_predefined_acr); i++) + if (hdmi_predefined_acr[i].clock == clock) + return &hdmi_predefined_acr[i]; + + /* And odd clocks get manually calculated */ + radeon_audio_calc_cts(clock, &res.cts_32khz, &res.n_32khz, 32000); + radeon_audio_calc_cts(clock, &res.cts_44_1khz, &res.n_44_1khz, 44100); + radeon_audio_calc_cts(clock, &res.cts_48khz, &res.n_48khz, 48000); + + return &res; +} + +/* + * update the N and CTS parameters for a given pixel clock rate + */ +static void radeon_audio_update_acr(struct drm_encoder *encoder, unsigned int clock) +{ + const struct radeon_hdmi_acr *acr = radeon_audio_acr(clock); + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + + if (!dig || !dig->afmt) + return; + + if (radeon_encoder->audio && radeon_encoder->audio->update_acr) + radeon_encoder->audio->update_acr(encoder, dig->afmt->offset, acr); +} + +static void radeon_audio_set_vbi_packet(struct drm_encoder *encoder) +{ + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + + if (!dig || !dig->afmt) + return; + + if (radeon_encoder->audio && radeon_encoder->audio->set_vbi_packet) + radeon_encoder->audio->set_vbi_packet(encoder, dig->afmt->offset); +} + +static void radeon_hdmi_set_color_depth(struct drm_encoder *encoder) +{ + int bpc = 8; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + + if (!dig || !dig->afmt) + return; + + if (encoder->crtc) { + struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); + bpc = radeon_crtc->bpc; + } + + if (radeon_encoder->audio && radeon_encoder->audio->set_color_depth) + radeon_encoder->audio->set_color_depth(encoder, dig->afmt->offset, bpc); +} + +static void radeon_audio_set_audio_packet(struct drm_encoder *encoder) +{ + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + + if (!dig || !dig->afmt) + return; + + if (radeon_encoder->audio && radeon_encoder->audio->set_audio_packet) + radeon_encoder->audio->set_audio_packet(encoder, dig->afmt->offset); +} + +static void radeon_audio_set_mute(struct drm_encoder *encoder, bool mute) +{ + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + + if (!dig || !dig->afmt) + return; + + if (radeon_encoder->audio && radeon_encoder->audio->set_mute) + radeon_encoder->audio->set_mute(encoder, dig->afmt->offset, mute); +} + +/* + * update the info frames with the data from the current display mode + */ +static void radeon_audio_hdmi_mode_set(struct drm_encoder *encoder, + struct drm_display_mode *mode) +{ + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + + if (!dig || !dig->afmt) + return; + + if (!connector) + return; + + if (drm_detect_monitor_audio(radeon_connector_edid(connector))) { + radeon_audio_set_mute(encoder, true); + + radeon_audio_write_speaker_allocation(encoder); + radeon_audio_write_sad_regs(encoder); + radeon_audio_write_latency_fields(encoder, mode); + radeon_audio_set_dto(encoder, mode->clock); + radeon_audio_set_vbi_packet(encoder); + radeon_hdmi_set_color_depth(encoder); + radeon_audio_update_acr(encoder, mode->clock); + radeon_audio_set_audio_packet(encoder); + radeon_audio_select_pin(encoder); + + if (radeon_audio_set_avi_packet(encoder, mode) < 0) + return; + + radeon_audio_set_mute(encoder, false); + } else { + radeon_hdmi_set_color_depth(encoder); + + if (radeon_audio_set_avi_packet(encoder, mode) < 0) + return; + } +} + +static void radeon_audio_dp_mode_set(struct drm_encoder *encoder, + struct drm_display_mode *mode) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv; + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + + if (!dig || !dig->afmt) + return; + + if (!connector) + return; + + if (drm_detect_monitor_audio(radeon_connector_edid(connector))) { + radeon_audio_write_speaker_allocation(encoder); + radeon_audio_write_sad_regs(encoder); + radeon_audio_write_latency_fields(encoder, mode); + radeon_audio_set_dto(encoder, rdev->clock.vco_freq * 10); + radeon_audio_set_audio_packet(encoder); + radeon_audio_select_pin(encoder); + + if (radeon_audio_set_avi_packet(encoder, mode) < 0) + return; + } +} + +void radeon_audio_mode_set(struct drm_encoder *encoder, + struct drm_display_mode *mode) +{ + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + + if (radeon_encoder->audio && radeon_encoder->audio->mode_set) + radeon_encoder->audio->mode_set(encoder, mode); +} + +void radeon_audio_dpms(struct drm_encoder *encoder, int mode) +{ + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + + if (radeon_encoder->audio && radeon_encoder->audio->dpms) + radeon_encoder->audio->dpms(encoder, mode == DRM_MODE_DPMS_ON); +} + +unsigned int radeon_audio_decode_dfs_div(unsigned int div) +{ + if (div >= 8 && div < 64) + return (div - 8) * 25 + 200; + else if (div >= 64 && div < 96) + return (div - 64) * 50 + 1600; + else if (div >= 96 && div < 128) + return (div - 96) * 100 + 3200; + else + return 0; +} diff --git a/sys/dev/pci/drm/radeon/radeon_audio.h b/sys/dev/pci/drm/radeon/radeon_audio.h new file mode 100644 index 00000000000..7379bd3f13f --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_audio.h @@ -0,0 +1,84 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Slava Grigorev + */ + +#ifndef __RADEON_AUDIO_H__ +#define __RADEON_AUDIO_H__ + +#include + +#define RREG32_ENDPOINT(block, reg) \ + radeon_audio_endpoint_rreg(rdev, (block), (reg)) +#define WREG32_ENDPOINT(block, reg, v) \ + radeon_audio_endpoint_wreg(rdev, (block), (reg), (v)) + +struct radeon_audio_basic_funcs +{ + u32 (*endpoint_rreg)(struct radeon_device *rdev, u32 offset, u32 reg); + void (*endpoint_wreg)(struct radeon_device *rdev, + u32 offset, u32 reg, u32 v); + void (*enable)(struct radeon_device *rdev, + struct r600_audio_pin *pin, u8 enable_mask); +}; + +struct radeon_audio_funcs +{ + void (*select_pin)(struct drm_encoder *encoder); + struct r600_audio_pin* (*get_pin)(struct radeon_device *rdev); + void (*write_latency_fields)(struct drm_encoder *encoder, + struct drm_connector *connector, struct drm_display_mode *mode); + void (*write_sad_regs)(struct drm_encoder *encoder, + struct cea_sad *sads, int sad_count); + void (*write_speaker_allocation)(struct drm_encoder *encoder, + u8 *sadb, int sad_count); + void (*set_dto)(struct radeon_device *rdev, + struct radeon_crtc *crtc, unsigned int clock); + void (*update_acr)(struct drm_encoder *encoder, long offset, + const struct radeon_hdmi_acr *acr); + void (*set_vbi_packet)(struct drm_encoder *encoder, u32 offset); + void (*set_color_depth)(struct drm_encoder *encoder, u32 offset, int bpc); + void (*set_avi_packet)(struct radeon_device *rdev, u32 offset, + unsigned char *buffer, size_t size); + void (*set_audio_packet)(struct drm_encoder *encoder, u32 offset); + void (*set_mute)(struct drm_encoder *encoder, u32 offset, bool mute); + void (*mode_set)(struct drm_encoder *encoder, + struct drm_display_mode *mode); + void (*dpms)(struct drm_encoder *encoder, bool mode); +}; + +int radeon_audio_init(struct radeon_device *rdev); +void radeon_audio_detect(struct drm_connector *connector, + struct drm_encoder *encoder, + enum drm_connector_status status); +u32 radeon_audio_endpoint_rreg(struct radeon_device *rdev, + u32 offset, u32 reg); +void radeon_audio_endpoint_wreg(struct radeon_device *rdev, + u32 offset, u32 reg, u32 v); +struct r600_audio_pin *radeon_audio_get_pin(struct drm_encoder *encoder); +void radeon_audio_fini(struct radeon_device *rdev); +void radeon_audio_mode_set(struct drm_encoder *encoder, + struct drm_display_mode *mode); +void radeon_audio_dpms(struct drm_encoder *encoder, int mode); +unsigned int radeon_audio_decode_dfs_div(unsigned int div); + +#endif diff --git a/sys/dev/pci/drm/radeon/radeon_benchmark.c b/sys/dev/pci/drm/radeon/radeon_benchmark.c index 0d561fe5d97..33a574967b3 100644 --- a/sys/dev/pci/drm/radeon/radeon_benchmark.c +++ b/sys/dev/pci/drm/radeon/radeon_benchmark.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_benchmark.c,v 1.7 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2009 Jerome Glisse. * @@ -35,7 +34,8 @@ static int radeon_benchmark_do_move(struct radeon_device *rdev, unsigned size, uint64_t saddr, uint64_t daddr, - int flag, int n) + int flag, int n, + struct reservation_object *resv) { unsigned long start_jiffies; unsigned long end_jiffies; @@ -46,33 +46,29 @@ static int radeon_benchmark_do_move(struct radeon_device *rdev, unsigned size, for (i = 0; i < n; i++) { switch (flag) { case RADEON_BENCHMARK_COPY_DMA: - r = radeon_copy_dma(rdev, saddr, daddr, - size / RADEON_GPU_PAGE_SIZE, - &fence); + fence = radeon_copy_dma(rdev, saddr, daddr, + size / RADEON_GPU_PAGE_SIZE, + resv); break; case RADEON_BENCHMARK_COPY_BLIT: - r = radeon_copy_blit(rdev, saddr, daddr, - size / RADEON_GPU_PAGE_SIZE, - &fence); + fence = radeon_copy_blit(rdev, saddr, daddr, + size / RADEON_GPU_PAGE_SIZE, + resv); break; default: DRM_ERROR("Unknown copy method\n"); - r = -EINVAL; + return -EINVAL; } - if (r) - goto exit_do_move; + if (IS_ERR(fence)) + return PTR_ERR(fence); + r = radeon_fence_wait(fence, false); - if (r) - goto exit_do_move; radeon_fence_unref(&fence); + if (r) + return r; } end_jiffies = jiffies; - r = jiffies_to_msecs(end_jiffies - start_jiffies); - -exit_do_move: - if (fence) - radeon_fence_unref(&fence); - return r; + return jiffies_to_msecs(end_jiffies - start_jiffies); } @@ -100,7 +96,7 @@ static void radeon_benchmark_move(struct radeon_device *rdev, unsigned size, int time; n = RADEON_BENCHMARK_ITERATIONS; - r = radeon_bo_create(rdev, size, PAGE_SIZE, true, sdomain, NULL, &sobj); + r = radeon_bo_create(rdev, size, PAGE_SIZE, true, sdomain, 0, NULL, NULL, &sobj); if (r) { goto out_cleanup; } @@ -112,7 +108,7 @@ static void radeon_benchmark_move(struct radeon_device *rdev, unsigned size, if (r) { goto out_cleanup; } - r = radeon_bo_create(rdev, size, PAGE_SIZE, true, ddomain, NULL, &dobj); + r = radeon_bo_create(rdev, size, PAGE_SIZE, true, ddomain, 0, NULL, NULL, &dobj); if (r) { goto out_cleanup; } @@ -125,12 +121,10 @@ static void radeon_benchmark_move(struct radeon_device *rdev, unsigned size, goto out_cleanup; } - /* r100 doesn't have dma engine so skip the test */ - /* also, VRAM-to-VRAM test doesn't make much sense for DMA */ - /* skip it as well if domains are the same */ - if ((rdev->asic->copy.dma) && (sdomain != ddomain)) { + if (rdev->asic->copy.dma) { time = radeon_benchmark_do_move(rdev, size, saddr, daddr, - RADEON_BENCHMARK_COPY_DMA, n); + RADEON_BENCHMARK_COPY_DMA, n, + dobj->tbo.resv); if (time < 0) goto out_cleanup; if (time > 0) @@ -140,7 +134,8 @@ static void radeon_benchmark_move(struct radeon_device *rdev, unsigned size, if (rdev->asic->copy.blit) { time = radeon_benchmark_do_move(rdev, size, saddr, daddr, - RADEON_BENCHMARK_COPY_BLIT, n); + RADEON_BENCHMARK_COPY_BLIT, n, + dobj->tbo.resv); if (time < 0) goto out_cleanup; if (time > 0) diff --git a/sys/dev/pci/drm/radeon/radeon_bios.c b/sys/dev/pci/drm/radeon/radeon_bios.c index 3fdfdd39916..370322c27b9 100644 --- a/sys/dev/pci/drm/radeon/radeon_bios.c +++ b/sys/dev/pci/drm/radeon/radeon_bios.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_bios.c,v 1.12 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -27,7 +26,6 @@ * Jerome Glisse */ #include -#include #include "radeon_reg.h" #include "radeon.h" #include "atom.h" @@ -35,6 +33,7 @@ #if defined(__amd64__) || defined(__i386__) #include #include +#include "acpi.h" #endif #if defined (__loongson__) @@ -45,61 +44,44 @@ * BIOS. */ -bool radeon_read_platform_bios(struct radeon_device *); - -bool -radeon_read_platform_bios(struct radeon_device *rdev) +/* If you boot an IGP board with a discrete card as the primary, + * the IGP rom is not accessible via the rom bar as the IGP rom is + * part of the system bios. On boot, the system bios puts a + * copy of the igp rom at the start of vram if a discrete card is + * present. + */ +#ifdef __linux__ +static bool igp_read_bios_from_vram(struct radeon_device *rdev) { -#if defined(__amd64__) || defined(__i386__) || defined(__loongson__) uint8_t __iomem *bios; - bus_size_t size = 256 * 1024; /* ??? */ - uint8_t *found = NULL; - int i; - - + resource_size_t vram_base; + resource_size_t size = 256 * 1024; /* ??? */ + if (!(rdev->flags & RADEON_IS_IGP)) if (!radeon_card_posted(rdev)) return false; rdev->bios = NULL; - -#if defined(__loongson__) - if (loongson_videobios == NULL) + vram_base = pci_resource_start(rdev->pdev, 0); + bios = ioremap(vram_base, size); + if (!bios) { return false; - bios = loongson_videobios; -#else - bios = (u8 *)ISA_HOLE_VADDR(0xc0000); -#endif - - for (i = 0; i + 2 < size; i++) { - if (bios[i] == 0x55 && bios[i + 1] == 0xaa) { - found = bios + i; - break; - } - } - if (found == NULL) { - DRM_ERROR("bios size zero or checksum mismatch\n"); + + if (size == 0 || bios[0] != 0x55 || bios[1] != 0xaa) { + iounmap(bios); return false; } - rdev->bios = kmalloc(size, GFP_KERNEL); - if (rdev->bios == NULL) + if (rdev->bios == NULL) { + iounmap(bios); return false; - - memcpy(rdev->bios, found, size); - + } + memcpy_fromio(rdev->bios, bios, size); + iounmap(bios); return true; -#endif - return false; } - -/* If you boot an IGP board with a discrete card as the primary, - * the IGP rom is not accessible via the rom bar as the IGP rom is - * part of the system bios. On boot, the system bios puts a - * copy of the igp rom at the start of vram if a discrete card is - * present. - */ +#else static bool igp_read_bios_from_vram(struct radeon_device *rdev) { uint8_t __iomem *bios; @@ -135,7 +117,38 @@ static bool igp_read_bios_from_vram(struct radeon_device *rdev) bus_space_unmap(bst, bsh, size); return true; } +#endif +#ifdef __linux__ +static bool radeon_read_bios(struct radeon_device *rdev) +{ + uint8_t __iomem *bios, val1, val2; + size_t size; + + rdev->bios = NULL; + /* XXX: some cards may return 0 for rom size? ddx has a workaround */ + bios = pci_map_rom(rdev->pdev, &size); + if (!bios) { + return false; + } + + val1 = readb(&bios[0]); + val2 = readb(&bios[1]); + + if (size == 0 || val1 != 0x55 || val2 != 0xaa) { + pci_unmap_rom(rdev->pdev, bios); + return false; + } + rdev->bios = kzalloc(size, GFP_KERNEL); + if (rdev->bios == NULL) { + pci_unmap_rom(rdev->pdev, bios); + return false; + } + memcpy_fromio(rdev->bios, bios, size); + pci_unmap_rom(rdev->pdev, bios); + return true; +} +#else static bool radeon_read_bios(struct radeon_device *rdev) { uint8_t __iomem *bios; @@ -179,6 +192,78 @@ fail: return false; } +#endif + +#ifdef __linux__ +static bool radeon_read_platform_bios(struct radeon_device *rdev) +{ + uint8_t __iomem *bios; + size_t size; + + rdev->bios = NULL; + + bios = pci_platform_rom(rdev->pdev, &size); + if (!bios) { + return false; + } + + if (size == 0 || bios[0] != 0x55 || bios[1] != 0xaa) { + return false; + } + rdev->bios = kmemdup(bios, size, GFP_KERNEL); + if (rdev->bios == NULL) { + return false; + } + + return true; +} +#else +static bool radeon_read_platform_bios(struct radeon_device *rdev) +{ +#if defined(__amd64__) || defined(__i386__) || defined(__loongson__) + uint8_t __iomem *bios; + bus_size_t size = 256 * 1024; /* ??? */ + uint8_t *found = NULL; + int i; + + if (!(rdev->flags & RADEON_IS_IGP)) + if (!radeon_card_posted(rdev)) + return false; + + rdev->bios = NULL; + +#if defined(__loongson__) + if (loongson_videobios == NULL) + return false; + bios = loongson_videobios; +#else + bios = (u8 *)ISA_HOLE_VADDR(0xc0000); +#endif + + for (i = 0; i + 2 < size; i++) { + if (bios[i] == 0x55 && bios[i + 1] == 0xaa) { + found = bios + i; + break; + } + + } + if (found == NULL) { + DRM_ERROR("bios size zero or checksum mismatch\n"); + return false; + } + + rdev->bios = kmalloc(size, GFP_KERNEL); + if (rdev->bios == NULL) + return false; + + memcpy(rdev->bios, found, size); + + return true; +#endif + return false; +} +#endif + #ifdef CONFIG_ACPI /* ATRM is used to get the BIOS on the discrete cards in * dual-gpu systems. @@ -242,7 +327,7 @@ static bool radeon_atrm_get_bios(struct radeon_device *rdev) return false; while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, pdev)) != NULL) { - dhandle = DEVICE_ACPI_HANDLE(&pdev->dev); + dhandle = ACPI_HANDLE(&pdev->dev); if (!dhandle) continue; @@ -315,24 +400,28 @@ static bool ni_read_disabled_bios(struct radeon_device *rdev) /* enable the rom */ WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS)); - /* Disable VGA mode */ - WREG32(AVIVO_D1VGA_CONTROL, - (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE | - AVIVO_DVGA_CONTROL_TIMING_SELECT))); - WREG32(AVIVO_D2VGA_CONTROL, - (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE | - AVIVO_DVGA_CONTROL_TIMING_SELECT))); - WREG32(AVIVO_VGA_RENDER_CONTROL, - (vga_render_control & ~AVIVO_VGA_VSTATUS_CNTL_MASK)); + if (!ASIC_IS_NODCE(rdev)) { + /* Disable VGA mode */ + WREG32(AVIVO_D1VGA_CONTROL, + (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE | + AVIVO_DVGA_CONTROL_TIMING_SELECT))); + WREG32(AVIVO_D2VGA_CONTROL, + (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE | + AVIVO_DVGA_CONTROL_TIMING_SELECT))); + WREG32(AVIVO_VGA_RENDER_CONTROL, + (vga_render_control & ~AVIVO_VGA_VSTATUS_CNTL_MASK)); + } WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE); r = radeon_read_bios(rdev); /* restore regs */ WREG32(R600_BUS_CNTL, bus_cntl); - WREG32(AVIVO_D1VGA_CONTROL, d1vga_control); - WREG32(AVIVO_D2VGA_CONTROL, d2vga_control); - WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control); + if (!ASIC_IS_NODCE(rdev)) { + WREG32(AVIVO_D1VGA_CONTROL, d1vga_control); + WREG32(AVIVO_D2VGA_CONTROL, d2vga_control); + WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control); + } WREG32(R600_ROM_CNTL, rom_cntl); return r; } @@ -566,7 +655,7 @@ static bool legacy_read_disabled_bios(struct radeon_device *rdev) crtc_ext_cntl = RREG32(RADEON_CRTC_EXT_CNTL); fp2_gen_cntl = 0; - if (rdev->ddev->pci_device == PCI_DEVICE_ID_ATI_RADEON_QY) { + if (rdev->ddev->pdev->device == PCI_DEVICE_ID_ATI_RADEON_QY) { fp2_gen_cntl = RREG32(RADEON_FP2_GEN_CNTL); } @@ -603,7 +692,7 @@ static bool legacy_read_disabled_bios(struct radeon_device *rdev) (RADEON_CRTC_SYNC_TRISTAT | RADEON_CRTC_DISPLAY_DIS))); - if (rdev->ddev->pci_device == PCI_DEVICE_ID_ATI_RADEON_QY) { + if (rdev->ddev->pdev->device == PCI_DEVICE_ID_ATI_RADEON_QY) { WREG32(RADEON_FP2_GEN_CNTL, (fp2_gen_cntl & ~RADEON_FP2_ON)); } @@ -621,7 +710,7 @@ static bool legacy_read_disabled_bios(struct radeon_device *rdev) WREG32(RADEON_CRTC2_GEN_CNTL, crtc2_gen_cntl); } WREG32(RADEON_CRTC_EXT_CNTL, crtc_ext_cntl); - if (rdev->ddev->pci_device == PCI_DEVICE_ID_ATI_RADEON_QY) { + if (rdev->ddev->pdev->device == PCI_DEVICE_ID_ATI_RADEON_QY) { WREG32(RADEON_FP2_GEN_CNTL, fp2_gen_cntl); } return r; @@ -643,10 +732,6 @@ static bool radeon_read_disabled_bios(struct radeon_device *rdev) return legacy_read_disabled_bios(rdev); } -#if defined(__amd64__) || defined(__i386__) -#include "acpi.h" -#endif - #if NACPI > 0 #define CONFIG_ACPI #endif @@ -687,7 +772,7 @@ static bool radeon_acpi_vfct_bios(struct radeon_device *rdev) vhdr->DeviceID != rdev->pdev->device) { DRM_INFO("ACPI VFCT table is not for this card\n"); goto out_unmap; - }; + } if (vfct->VBIOSImageOffset + sizeof(VFCT_IMAGE_HEADER) + vhdr->ImageLength > tbl_size) { DRM_ERROR("ACPI VFCT image truncated\n"); @@ -717,13 +802,12 @@ bool radeon_get_bios(struct radeon_device *rdev) r = radeon_acpi_vfct_bios(rdev); if (r == false) r = igp_read_bios_from_vram(rdev); - if (r == false) - r = radeon_read_platform_bios(rdev); if (r == false) r = radeon_read_bios(rdev); - if (r == false) { + if (r == false) r = radeon_read_disabled_bios(rdev); - } + if (r == false) + r = radeon_read_platform_bios(rdev); if (r == false || rdev->bios == NULL) { DRM_ERROR("Unable to locate a BIOS ROM\n"); rdev->bios = NULL; diff --git a/sys/dev/pci/drm/radeon/radeon_blit_common.h b/sys/dev/pci/drm/radeon/radeon_blit_common.h deleted file mode 100644 index 3a18d4b22ee..00000000000 --- a/sys/dev/pci/drm/radeon/radeon_blit_common.h +++ /dev/null @@ -1,45 +0,0 @@ -/* $OpenBSD: radeon_blit_common.h,v 1.3 2018/04/20 21:12:50 naddy Exp $ */ -/* - * Copyright 2009 Advanced Micro Devices, Inc. - * Copyright 2009 Red Hat Inc. - * Copyright 2012 Alcatel-Lucent, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef __RADEON_BLIT_COMMON_H__ - -#define DI_PT_RECTLIST 0x11 -#define DI_INDEX_SIZE_16_BIT 0x0 -#define DI_SRC_SEL_AUTO_INDEX 0x2 - -#define FMT_8 0x1 -#define FMT_5_6_5 0x8 -#define FMT_8_8_8_8 0x1a -#define COLOR_8 0x1 -#define COLOR_5_6_5 0x8 -#define COLOR_8_8_8_8 0x1a - -#define RECT_UNIT_H 32 -#define RECT_UNIT_W (RADEON_GPU_PAGE_SIZE / 4 / RECT_UNIT_H) - -#define __RADEON_BLIT_COMMON_H__ -#endif diff --git a/sys/dev/pci/drm/radeon/radeon_clocks.c b/sys/dev/pci/drm/radeon/radeon_clocks.c index 5cdee96d28c..a2d1bd915ac 100644 --- a/sys/dev/pci/drm/radeon/radeon_clocks.c +++ b/sys/dev/pci/drm/radeon/radeon_clocks.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_clocks.c,v 1.5 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/radeon_combios.c b/sys/dev/pci/drm/radeon/radeon_combios.c index 3ce6db979e9..749d9ccfe99 100644 --- a/sys/dev/pci/drm/radeon/radeon_combios.c +++ b/sys/dev/pci/drm/radeon/radeon_combios.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_combios.c,v 1.13 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2004 ATI Technologies Inc., Markham, Ontario * Copyright 2007-8 Advanced Micro Devices, Inc. @@ -30,21 +29,13 @@ #include "radeon.h" #include "atom.h" -/* from radeon_encoder.c */ -extern uint32_t -radeon_get_encoder_enum(struct drm_device *dev, uint32_t supported_device, - uint8_t dac); -extern void radeon_link_encoder_connector(struct drm_device *dev); - -/* from radeon_connector.c */ -extern void -radeon_add_legacy_connector(struct drm_device *dev, - uint32_t connector_id, - uint32_t supported_device, - int connector_type, - struct radeon_i2c_bus_rec *i2c_bus, - uint16_t connector_object_id, - struct radeon_hpd *hpd); +#ifdef CONFIG_PPC_PMAC +/* not sure which of these are needed */ +#include +#include +#include +#include +#endif /* CONFIG_PPC_PMAC */ /* from radeon_legacy_encoder.c */ extern void @@ -125,7 +116,7 @@ enum radeon_combios_connector { CONNECTOR_UNSUPPORTED_LEGACY }; -const int legacy_connector_convert[] = { +static const int legacy_connector_convert[] = { DRM_MODE_CONNECTOR_Unknown, DRM_MODE_CONNECTOR_DVID, DRM_MODE_CONNECTOR_VGA, @@ -364,11 +355,13 @@ static uint16_t combios_get_table_offset(struct drm_device *dev, } break; default: + check_offset = 0; break; } size = RBIOS8(rdev->bios_header_start + 0x6); - if (table < COMBIOS_ASIC_INIT_3_TABLE && check_offset < size) + /* check absolute offset tables */ + if (table < COMBIOS_ASIC_INIT_3_TABLE && check_offset && check_offset < size) offset = RBIOS16(rdev->bios_header_start + check_offset); return offset; @@ -421,9 +414,9 @@ radeon_bios_get_hardcoded_edid(struct radeon_device *rdev) #ifdef __clang__ static inline struct radeon_i2c_bus_rec combios_setup_i2c_bus(struct radeon_device *rdev, - enum radeon_combios_ddc ddc, - u32 clk_mask, - u32 data_mask) + enum radeon_combios_ddc ddc, + u32 clk_mask, + u32 data_mask) #else static struct radeon_i2c_bus_rec combios_setup_i2c_bus(struct radeon_device *rdev, enum radeon_combios_ddc ddc, @@ -1269,10 +1262,15 @@ struct radeon_encoder_lvds *radeon_combios_get_lvds_info(struct radeon_encoder if ((RBIOS16(tmp) == lvds->native_mode.hdisplay) && (RBIOS16(tmp + 2) == lvds->native_mode.vdisplay)) { + u32 hss = (RBIOS16(tmp + 21) - RBIOS16(tmp + 19) - 1) * 8; + + if (hss > lvds->native_mode.hdisplay) + hss = (10 - 1) * 8; + lvds->native_mode.htotal = lvds->native_mode.hdisplay + (RBIOS16(tmp + 17) - RBIOS16(tmp + 19)) * 8; lvds->native_mode.hsync_start = lvds->native_mode.hdisplay + - (RBIOS16(tmp + 21) - RBIOS16(tmp + 19) - 1) * 8; + hss; lvds->native_mode.hsync_end = lvds->native_mode.hsync_start + (RBIOS8(tmp + 23) * 8); @@ -3402,6 +3400,21 @@ void radeon_combios_asic_init(struct drm_device *dev) rdev->pdev->subsystem_device == 0x30ae) return; + /* quirk for rs4xx HP Compaq dc5750 Small Form Factor to make it resume + * - it hangs on resume inside the dynclk 1 table. + */ + if (rdev->family == CHIP_RS480 && + rdev->pdev->subsystem_vendor == 0x103c && + rdev->pdev->subsystem_device == 0x280a) + return; + /* quirk for rs4xx Toshiba Sattellite L20-183 latop to make it resume + * - it hangs on resume inside the dynclk 1 table. + */ + if (rdev->family == CHIP_RS400 && + rdev->pdev->subsystem_vendor == 0x1179 && + rdev->pdev->subsystem_device == 0xff31) + return; + /* DYN CLK 1 */ table = combios_get_table_offset(dev, COMBIOS_DYN_CLK_1_TABLE); if (table) diff --git a/sys/dev/pci/drm/radeon/radeon_connectors.c b/sys/dev/pci/drm/radeon/radeon_connectors.c index 28ce501f682..fa852640736 100644 --- a/sys/dev/pci/drm/radeon/radeon_connectors.c +++ b/sys/dev/pci/drm/radeon/radeon_connectors.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_connectors.c,v 1.9 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2007-8 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -28,25 +27,40 @@ #include #include #include +#include #include #include "radeon.h" +#include "radeon_audio.h" #include "atom.h" -extern void -radeon_combios_connected_scratch_regs(struct drm_connector *connector, - struct drm_encoder *encoder, - bool connected); -extern void -radeon_atombios_connected_scratch_regs(struct drm_connector *connector, - struct drm_encoder *encoder, - bool connected); +static int radeon_dp_handle_hpd(struct drm_connector *connector) +{ + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + int ret; + + ret = radeon_dp_mst_check_status(radeon_connector); + if (ret == -EINVAL) + return 1; + return 0; +} void radeon_connector_hotplug(struct drm_connector *connector) { struct drm_device *dev = connector->dev; struct radeon_device *rdev = dev->dev_private; struct radeon_connector *radeon_connector = to_radeon_connector(connector); + if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) { + struct radeon_connector_atom_dig *dig_connector = + radeon_connector->con_priv; + + if (radeon_connector->is_mst_connector) + return; + if (dig_connector->is_mst) { + radeon_dp_handle_hpd(connector); + return; + } + } /* bail if the connector does not have hpd pin, e.g., * VGA, TV, etc. */ @@ -56,6 +70,7 @@ void radeon_connector_hotplug(struct drm_connector *connector) radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd); /* if the connector is already off, don't turn it back on */ + /* FIXME: This access isn't protected by any locks. */ if (connector->dpms != DRM_MODE_DPMS_ON) return; @@ -73,20 +88,18 @@ void radeon_connector_hotplug(struct drm_connector *connector) /* don't do anything if sink is not display port, i.e., * passive dp->(dvi|hdmi) adaptor */ - if (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) { - int saved_dpms = connector->dpms; - /* Only turn off the display if it's physically disconnected */ - if (!radeon_hpd_sense(rdev, radeon_connector->hpd.hpd)) { - drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF); - } else if (radeon_dp_needs_link_train(radeon_connector)) { - /* set it to OFF so that drm_helper_connector_dpms() - * won't return immediately since the current state - * is ON at this point. - */ - connector->dpms = DRM_MODE_DPMS_OFF; - drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); - } - connector->dpms = saved_dpms; + if (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT && + radeon_hpd_sense(rdev, radeon_connector->hpd.hpd) && + radeon_dp_needs_link_train(radeon_connector)) { + /* Don't start link training before we have the DPCD */ + if (!radeon_dp_getdpcd(radeon_connector)) + return; + + /* Turn the connector off and back on immediately, which + * will trigger link training + */ + drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF); + drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); } } } @@ -108,12 +121,13 @@ int radeon_get_monitor_bpc(struct drm_connector *connector) struct radeon_connector *radeon_connector = to_radeon_connector(connector); struct radeon_connector_atom_dig *dig_connector; int bpc = 8; + int mode_clock, max_tmds_clock; switch (connector->connector_type) { case DRM_MODE_CONNECTOR_DVII: case DRM_MODE_CONNECTOR_HDMIB: if (radeon_connector->use_digital) { - if (drm_detect_hdmi_monitor(radeon_connector->edid)) { + if (drm_detect_hdmi_monitor(radeon_connector_edid(connector))) { if (connector->display_info.bpc) bpc = connector->display_info.bpc; } @@ -121,7 +135,7 @@ int radeon_get_monitor_bpc(struct drm_connector *connector) break; case DRM_MODE_CONNECTOR_DVID: case DRM_MODE_CONNECTOR_HDMIA: - if (drm_detect_hdmi_monitor(radeon_connector->edid)) { + if (drm_detect_hdmi_monitor(radeon_connector_edid(connector))) { if (connector->display_info.bpc) bpc = connector->display_info.bpc; } @@ -130,7 +144,7 @@ int radeon_get_monitor_bpc(struct drm_connector *connector) dig_connector = radeon_connector->con_priv; if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP) || - drm_detect_hdmi_monitor(radeon_connector->edid)) { + drm_detect_hdmi_monitor(radeon_connector_edid(connector))) { if (connector->display_info.bpc) bpc = connector->display_info.bpc; } @@ -153,6 +167,73 @@ int radeon_get_monitor_bpc(struct drm_connector *connector) } break; } + + if (drm_detect_hdmi_monitor(radeon_connector_edid(connector))) { + /* hdmi deep color only implemented on DCE4+ */ + if ((bpc > 8) && !ASIC_IS_DCE4(rdev)) { + DRM_DEBUG("%s: HDMI deep color %d bpc unsupported. Using 8 bpc.\n", + connector->name, bpc); + bpc = 8; + } + + /* + * Pre DCE-8 hw can't handle > 12 bpc, and more than 12 bpc doesn't make + * much sense without support for > 12 bpc framebuffers. RGB 4:4:4 at + * 12 bpc is always supported on hdmi deep color sinks, as this is + * required by the HDMI-1.3 spec. Clamp to a safe 12 bpc maximum. + */ + if (bpc > 12) { + DRM_DEBUG("%s: HDMI deep color %d bpc unsupported. Using 12 bpc.\n", + connector->name, bpc); + bpc = 12; + } + + /* Any defined maximum tmds clock limit we must not exceed? */ + if (connector->max_tmds_clock > 0) { + /* mode_clock is clock in kHz for mode to be modeset on this connector */ + mode_clock = radeon_connector->pixelclock_for_modeset; + + /* Maximum allowable input clock in kHz */ + max_tmds_clock = connector->max_tmds_clock * 1000; + + DRM_DEBUG("%s: hdmi mode dotclock %d kHz, max tmds input clock %d kHz.\n", + connector->name, mode_clock, max_tmds_clock); + + /* Check if bpc is within clock limit. Try to degrade gracefully otherwise */ + if ((bpc == 12) && (mode_clock * 3/2 > max_tmds_clock)) { + if ((connector->display_info.edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_30) && + (mode_clock * 5/4 <= max_tmds_clock)) + bpc = 10; + else + bpc = 8; + + DRM_DEBUG("%s: HDMI deep color 12 bpc exceeds max tmds clock. Using %d bpc.\n", + connector->name, bpc); + } + + if ((bpc == 10) && (mode_clock * 5/4 > max_tmds_clock)) { + bpc = 8; + DRM_DEBUG("%s: HDMI deep color 10 bpc exceeds max tmds clock. Using %d bpc.\n", + connector->name, bpc); + } + } + else if (bpc > 8) { + /* max_tmds_clock missing, but hdmi spec mandates it for deep color. */ + DRM_DEBUG("%s: Required max tmds clock for HDMI deep color missing. Using 8 bpc.\n", + connector->name); + bpc = 8; + } + } + + if ((radeon_deep_color == 0) && (bpc > 8)) { + DRM_DEBUG("%s: Deep color disabled. Set radeon module param deep_color=1 to enable.\n", + connector->name); + bpc = 8; + } + + DRM_DEBUG("%s: Display bpc=%d, returned bpc=%d\n", + connector->name, connector->display_info.bpc, bpc); + return bpc; } @@ -164,7 +245,6 @@ radeon_connector_update_scratch_regs(struct drm_connector *connector, enum drm_c struct drm_encoder *best_encoder = NULL; struct drm_encoder *encoder = NULL; const struct drm_connector_helper_funcs *connector_funcs = connector->helper_private; - struct drm_mode_object *obj; bool connected; int i; @@ -174,14 +254,11 @@ radeon_connector_update_scratch_regs(struct drm_connector *connector, enum drm_c if (connector->encoder_ids[i] == 0) break; - obj = drm_mode_object_find(connector->dev, - connector->encoder_ids[i], - DRM_MODE_OBJECT_ENCODER); - if (!obj) + encoder = drm_encoder_find(connector->dev, + connector->encoder_ids[i]); + if (!encoder) continue; - encoder = obj_to_encoder(obj); - if ((encoder == best_encoder) && (status == connector_status_connected)) connected = true; else @@ -197,7 +274,6 @@ radeon_connector_update_scratch_regs(struct drm_connector *connector, enum drm_c static struct drm_encoder *radeon_find_encoder(struct drm_connector *connector, int encoder_type) { - struct drm_mode_object *obj; struct drm_encoder *encoder; int i; @@ -205,34 +281,140 @@ static struct drm_encoder *radeon_find_encoder(struct drm_connector *connector, if (connector->encoder_ids[i] == 0) break; - obj = drm_mode_object_find(connector->dev, connector->encoder_ids[i], DRM_MODE_OBJECT_ENCODER); - if (!obj) + encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]); + if (!encoder) continue; - encoder = obj_to_encoder(obj); if (encoder->encoder_type == encoder_type) return encoder; } return NULL; } +struct edid *radeon_connector_edid(struct drm_connector *connector) +{ + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + struct drm_property_blob *edid_blob = connector->edid_blob_ptr; + + if (radeon_connector->edid) { + return radeon_connector->edid; + } else if (edid_blob) { + struct edid *edid = kmemdup(edid_blob->data, edid_blob->length, GFP_KERNEL); + if (edid) + radeon_connector->edid = edid; + } + return radeon_connector->edid; +} + +static void radeon_connector_get_edid(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + + if (radeon_connector->edid) + return; + + /* on hw with routers, select right port */ + if (radeon_connector->router.ddc_valid) + radeon_router_select_ddc_port(radeon_connector); + + if ((radeon_connector_encoder_get_dp_bridge_encoder_id(connector) != + ENCODER_OBJECT_ID_NONE) && + radeon_connector->ddc_bus->has_aux) { + radeon_connector->edid = drm_get_edid(connector, + &radeon_connector->ddc_bus->aux.ddc); + } else if ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) || + (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) { + struct radeon_connector_atom_dig *dig = radeon_connector->con_priv; + + if ((dig->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT || + dig->dp_sink_type == CONNECTOR_OBJECT_ID_eDP) && + radeon_connector->ddc_bus->has_aux) + radeon_connector->edid = drm_get_edid(&radeon_connector->base, + &radeon_connector->ddc_bus->aux.ddc); + else if (radeon_connector->ddc_bus) + radeon_connector->edid = drm_get_edid(&radeon_connector->base, + &radeon_connector->ddc_bus->adapter); + } else if (radeon_connector->ddc_bus) { + radeon_connector->edid = drm_get_edid(&radeon_connector->base, + &radeon_connector->ddc_bus->adapter); + } + + if (!radeon_connector->edid) { + /* don't fetch the edid from the vbios if ddc fails and runpm is + * enabled so we report disconnected. + */ + if ((rdev->flags & RADEON_IS_PX) && (radeon_runtime_pm != 0)) + return; + + if (rdev->is_atom_bios) { + /* some laptops provide a hardcoded edid in rom for LCDs */ + if (((connector->connector_type == DRM_MODE_CONNECTOR_LVDS) || + (connector->connector_type == DRM_MODE_CONNECTOR_eDP))) + radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev); + } else { + /* some servers provide a hardcoded edid in rom for KVMs */ + radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev); + } + } +} + +static void radeon_connector_free_edid(struct drm_connector *connector) +{ + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + + if (radeon_connector->edid) { + kfree(radeon_connector->edid); + radeon_connector->edid = NULL; + } +} + +static int radeon_ddc_get_modes(struct drm_connector *connector) +{ + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + int ret; + + if (radeon_connector->edid) { + drm_mode_connector_update_edid_property(connector, radeon_connector->edid); + ret = drm_add_edid_modes(connector, radeon_connector->edid); + drm_edid_to_eld(connector, radeon_connector->edid); + return ret; + } + drm_mode_connector_update_edid_property(connector, NULL); + return 0; +} + static struct drm_encoder *radeon_best_single_encoder(struct drm_connector *connector) { int enc_id = connector->encoder_ids[0]; - struct drm_mode_object *obj; - struct drm_encoder *encoder; - /* pick the encoder ids */ - if (enc_id) { - obj = drm_mode_object_find(connector->dev, enc_id, DRM_MODE_OBJECT_ENCODER); - if (!obj) - return NULL; - encoder = obj_to_encoder(obj); - return encoder; - } + if (enc_id) + return drm_encoder_find(connector->dev, enc_id); return NULL; } +static void radeon_get_native_mode(struct drm_connector *connector) +{ + struct drm_encoder *encoder = radeon_best_single_encoder(connector); + struct radeon_encoder *radeon_encoder; + + if (encoder == NULL) + return; + + radeon_encoder = to_radeon_encoder(encoder); + + if (!list_empty(&connector->probed_modes)) { + struct drm_display_mode *preferred_mode = + list_first_entry(&connector->probed_modes, + struct drm_display_mode, head); + + radeon_encoder->native_mode = *preferred_mode; + } else { + radeon_encoder->native_mode.clock = 0; + } +} + /* * radeon_connector_analog_encoder_conflict_solve * - search for other connectors sharing this encoder @@ -268,13 +450,17 @@ radeon_connector_analog_encoder_conflict_solve(struct drm_connector *connector, continue; if (priority == true) { - DRM_DEBUG_KMS("1: conflicting encoders switching off %s\n", conflict->name); - DRM_DEBUG_KMS("in favor of %s\n", connector->name); + DRM_DEBUG_KMS("1: conflicting encoders switching off %s\n", + conflict->name); + DRM_DEBUG_KMS("in favor of %s\n", + connector->name); conflict->status = connector_status_disconnected; radeon_connector_update_scratch_regs(conflict, connector_status_disconnected); } else { - DRM_DEBUG_KMS("2: conflicting encoders switching off %s\n", connector->name); - DRM_DEBUG_KMS("in favor of %s\n", conflict->name); + DRM_DEBUG_KMS("2: conflicting encoders switching off %s\n", + connector->name); + DRM_DEBUG_KMS("in favor of %s\n", + conflict->name); current_status = connector_status_disconnected; } break; @@ -397,6 +583,36 @@ static int radeon_connector_set_property(struct drm_connector *connector, struct } } + if (property == rdev->mode_info.audio_property) { + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + /* need to find digital encoder on connector */ + encoder = radeon_find_encoder(connector, DRM_MODE_ENCODER_TMDS); + if (!encoder) + return 0; + + radeon_encoder = to_radeon_encoder(encoder); + + if (radeon_connector->audio != val) { + radeon_connector->audio = val; + radeon_property_change_mode(&radeon_encoder->base); + } + } + + if (property == rdev->mode_info.dither_property) { + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + /* need to find digital encoder on connector */ + encoder = radeon_find_encoder(connector, DRM_MODE_ENCODER_TMDS); + if (!encoder) + return 0; + + radeon_encoder = to_radeon_encoder(encoder); + + if (radeon_connector->dither != val) { + radeon_connector->dither = val; + radeon_property_change_mode(&radeon_encoder->base); + } + } + if (property == rdev->mode_info.underscan_property) { /* need to find digital encoder on connector */ encoder = radeon_find_encoder(connector, DRM_MODE_ENCODER_TMDS); @@ -499,6 +715,59 @@ static int radeon_connector_set_property(struct drm_connector *connector, struct radeon_property_change_mode(&radeon_encoder->base); } + if (property == dev->mode_config.scaling_mode_property) { + enum radeon_rmx_type rmx_type; + + if (connector->encoder) + radeon_encoder = to_radeon_encoder(connector->encoder); + else { + const struct drm_connector_helper_funcs *connector_funcs = connector->helper_private; + radeon_encoder = to_radeon_encoder(connector_funcs->best_encoder(connector)); + } + + switch (val) { + default: + case DRM_MODE_SCALE_NONE: rmx_type = RMX_OFF; break; + case DRM_MODE_SCALE_CENTER: rmx_type = RMX_CENTER; break; + case DRM_MODE_SCALE_ASPECT: rmx_type = RMX_ASPECT; break; + case DRM_MODE_SCALE_FULLSCREEN: rmx_type = RMX_FULL; break; + } + if (radeon_encoder->rmx_type == rmx_type) + return 0; + + if ((rmx_type != DRM_MODE_SCALE_NONE) && + (radeon_encoder->native_mode.clock == 0)) + return 0; + + radeon_encoder->rmx_type = rmx_type; + + radeon_property_change_mode(&radeon_encoder->base); + } + + if (property == rdev->mode_info.output_csc_property) { + if (connector->encoder) + radeon_encoder = to_radeon_encoder(connector->encoder); + else { + const struct drm_connector_helper_funcs *connector_funcs = connector->helper_private; + radeon_encoder = to_radeon_encoder(connector_funcs->best_encoder(connector)); + } + + if (radeon_encoder->output_csc == val) + return 0; + + radeon_encoder->output_csc = val; + + if (connector->encoder->crtc) { + struct drm_crtc *crtc = connector->encoder->crtc; + const struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private; + struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); + + radeon_crtc->output_csc = radeon_encoder->output_csc; + + (*crtc_funcs->load_lut)(crtc); + } + } + return 0; } @@ -539,22 +808,20 @@ static void radeon_fixup_lvds_native_mode(struct drm_encoder *encoder, static int radeon_lvds_get_modes(struct drm_connector *connector) { - struct radeon_connector *radeon_connector = to_radeon_connector(connector); struct drm_encoder *encoder; int ret = 0; struct drm_display_mode *mode; - if (radeon_connector->ddc_bus) { - ret = radeon_ddc_get_modes(radeon_connector); - if (ret > 0) { - encoder = radeon_best_single_encoder(connector); - if (encoder) { - radeon_fixup_lvds_native_mode(encoder, connector); - /* add scaled modes */ - radeon_add_common_modes(encoder, connector); - } - return ret; + radeon_connector_get_edid(connector); + ret = radeon_ddc_get_modes(connector); + if (ret > 0) { + encoder = radeon_best_single_encoder(connector); + if (encoder) { + radeon_fixup_lvds_native_mode(encoder, connector); + /* add scaled modes */ + radeon_add_common_modes(encoder, connector); } + return ret; } encoder = radeon_best_single_encoder(connector); @@ -609,9 +876,16 @@ static int radeon_lvds_mode_valid(struct drm_connector *connector, static enum drm_connector_status radeon_lvds_detect(struct drm_connector *connector, bool force) { + struct drm_device *dev = connector->dev; + struct radeon_device *rdev = dev->dev_private; struct radeon_connector *radeon_connector = to_radeon_connector(connector); struct drm_encoder *encoder = radeon_best_single_encoder(connector); enum drm_connector_status ret = connector_status_disconnected; + int r; + + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; if (encoder) { struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); @@ -620,23 +894,22 @@ radeon_lvds_detect(struct drm_connector *connector, bool force) /* check if panel is valid */ if (native_mode->hdisplay >= 320 && native_mode->vdisplay >= 240) ret = connector_status_connected; - + /* don't fetch the edid from the vbios if ddc fails and runpm is + * enabled so we report disconnected. + */ + if ((rdev->flags & RADEON_IS_PX) && (radeon_runtime_pm != 0)) + ret = connector_status_disconnected; } /* check for edid as well */ + radeon_connector_get_edid(connector); if (radeon_connector->edid) ret = connector_status_connected; - else { - if (radeon_connector->ddc_bus) { - radeon_connector->edid = drm_get_edid(&radeon_connector->base, - &radeon_connector->ddc_bus->adapter); - if (radeon_connector->edid) - ret = connector_status_connected; - } - } /* check acpi lid status ??? */ radeon_connector_update_scratch_regs(connector, ret); + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); return ret; } @@ -644,8 +917,7 @@ static void radeon_connector_destroy(struct drm_connector *connector) { struct radeon_connector *radeon_connector = to_radeon_connector(connector); - if (radeon_connector->edid) - kfree(radeon_connector->edid); + radeon_connector_free_edid(connector); kfree(radeon_connector->con_priv); drm_connector_unregister(connector); drm_connector_cleanup(connector); @@ -704,10 +976,12 @@ static const struct drm_connector_funcs radeon_lvds_connector_funcs = { static int radeon_vga_get_modes(struct drm_connector *connector) { - struct radeon_connector *radeon_connector = to_radeon_connector(connector); int ret; - ret = radeon_ddc_get_modes(radeon_connector); + radeon_connector_get_edid(connector); + ret = radeon_ddc_get_modes(connector); + + radeon_get_native_mode(connector); return ret; } @@ -736,6 +1010,11 @@ radeon_vga_detect(struct drm_connector *connector, bool force) const struct drm_encoder_helper_funcs *encoder_funcs; bool dret = false; enum drm_connector_status ret = connector_status_disconnected; + int r; + + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; encoder = radeon_best_single_encoder(connector); if (!encoder) @@ -745,28 +1024,26 @@ radeon_vga_detect(struct drm_connector *connector, bool force) dret = radeon_ddc_probe(radeon_connector, false); if (dret) { radeon_connector->detected_by_load = false; - if (radeon_connector->edid) { - kfree(radeon_connector->edid); - radeon_connector->edid = NULL; - } - radeon_connector->edid = drm_get_edid(&radeon_connector->base, &radeon_connector->ddc_bus->adapter); + radeon_connector_free_edid(connector); + radeon_connector_get_edid(connector); if (!radeon_connector->edid) { DRM_ERROR("%s: probed a monitor but no|invalid EDID\n", connector->name); ret = connector_status_connected; } else { - radeon_connector->use_digital = !!(radeon_connector->edid->input & DRM_EDID_INPUT_DIGITAL); + radeon_connector->use_digital = + !!(radeon_connector->edid->input & DRM_EDID_INPUT_DIGITAL); /* some oems have boards with separate digital and analog connectors * with a shared ddc line (often vga + hdmi) */ if (radeon_connector->use_digital && radeon_connector->shared_ddc) { - kfree(radeon_connector->edid); - radeon_connector->edid = NULL; + radeon_connector_free_edid(connector); ret = connector_status_disconnected; - } else + } else { ret = connector_status_connected; + } } } else { @@ -776,9 +1053,8 @@ radeon_vga_detect(struct drm_connector *connector, bool force) * detected a monitor via load. */ if (radeon_connector->detected_by_load) - return connector->status; - else - return ret; + ret = connector->status; + goto out; } if (radeon_connector->dac_load_detect && encoder) { @@ -803,6 +1079,11 @@ radeon_vga_detect(struct drm_connector *connector, bool force) } radeon_connector_update_scratch_regs(connector, ret); + +out: + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); + return ret; } @@ -859,10 +1140,15 @@ radeon_tv_detect(struct drm_connector *connector, bool force) const struct drm_encoder_helper_funcs *encoder_funcs; struct radeon_connector *radeon_connector = to_radeon_connector(connector); enum drm_connector_status ret = connector_status_disconnected; + int r; if (!radeon_connector->dac_load_detect) return ret; + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; + encoder = radeon_best_single_encoder(connector); if (!encoder) ret = connector_status_disconnected; @@ -873,6 +1159,8 @@ radeon_tv_detect(struct drm_connector *connector, bool force) if (ret == connector_status_connected) ret = radeon_connector_analog_encoder_conflict_solve(connector, encoder, ret, false); radeon_connector_update_scratch_regs(connector, ret); + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); return ret; } @@ -890,15 +1178,6 @@ static const struct drm_connector_funcs radeon_tv_connector_funcs = { .set_property = radeon_connector_set_property, }; -static int radeon_dvi_get_modes(struct drm_connector *connector) -{ - struct radeon_connector *radeon_connector = to_radeon_connector(connector); - int ret; - - ret = radeon_ddc_get_modes(radeon_connector); - return ret; -} - static bool radeon_check_hpd_status_unchanged(struct drm_connector *connector) { struct drm_device *dev = connector->dev; @@ -939,50 +1218,73 @@ radeon_dvi_detect(struct drm_connector *connector, bool force) struct radeon_connector *radeon_connector = to_radeon_connector(connector); struct drm_encoder *encoder = NULL; const struct drm_encoder_helper_funcs *encoder_funcs; - struct drm_mode_object *obj; - int i; + int i, r; enum drm_connector_status ret = connector_status_disconnected; bool dret = false, broken_edid = false; - if (!force && radeon_check_hpd_status_unchanged(connector)) - return connector->status; + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; - if (radeon_connector->ddc_bus) + if (radeon_connector->detected_hpd_without_ddc) { + force = true; + radeon_connector->detected_hpd_without_ddc = false; + } + + if (!force && radeon_check_hpd_status_unchanged(connector)) { + ret = connector->status; + goto exit; + } + + if (radeon_connector->ddc_bus) { dret = radeon_ddc_probe(radeon_connector, false); + + /* Sometimes the pins required for the DDC probe on DVI + * connectors don't make contact at the same time that the ones + * for HPD do. If the DDC probe fails even though we had an HPD + * signal, try again later */ + if (!dret && !force && + connector->status != connector_status_connected) { + DRM_DEBUG_KMS("hpd detected without ddc, retrying in 1 second\n"); + radeon_connector->detected_hpd_without_ddc = true; + schedule_delayed_work(&rdev->hotplug_work, + msecs_to_jiffies(1000)); + goto exit; + } + } if (dret) { radeon_connector->detected_by_load = false; - if (radeon_connector->edid) { - kfree(radeon_connector->edid); - radeon_connector->edid = NULL; - } - radeon_connector->edid = drm_get_edid(&radeon_connector->base, &radeon_connector->ddc_bus->adapter); + radeon_connector_free_edid(connector); + radeon_connector_get_edid(connector); if (!radeon_connector->edid) { DRM_ERROR("%s: probed a monitor but no|invalid EDID\n", connector->name); /* rs690 seems to have a problem with connectors not existing and always * return a block of 0's. If we see this just stop polling on this output */ - if ((rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) && radeon_connector->base.null_edid_counter) { + if ((rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) && + radeon_connector->base.null_edid_counter) { ret = connector_status_disconnected; - DRM_ERROR("%s: detected RS690 floating bus bug, stopping ddc detect\n", connector->name); + DRM_ERROR("%s: detected RS690 floating bus bug, stopping ddc detect\n", + connector->name); radeon_connector->ddc_bus = NULL; } else { ret = connector_status_connected; broken_edid = true; /* defer use_digital to later */ } } else { - radeon_connector->use_digital = !!(radeon_connector->edid->input & DRM_EDID_INPUT_DIGITAL); + radeon_connector->use_digital = + !!(radeon_connector->edid->input & DRM_EDID_INPUT_DIGITAL); /* some oems have boards with separate digital and analog connectors * with a shared ddc line (often vga + hdmi) */ if ((!radeon_connector->use_digital) && radeon_connector->shared_ddc) { - kfree(radeon_connector->edid); - radeon_connector->edid = NULL; + radeon_connector_free_edid(connector); ret = connector_status_disconnected; - } else + } else { ret = connector_status_connected; - + } /* This gets complicated. We have boards with VGA + HDMI with a * shared DDC line and we have boards with DVI-D + HDMI with a shared * DDC line. The latter is more complex because with DVI<->HDMI adapters @@ -1002,8 +1304,7 @@ radeon_dvi_detect(struct drm_connector *connector, bool force) if (list_connector->connector_type != DRM_MODE_CONNECTOR_VGA) { /* hpd is our only option in this case */ if (!radeon_hpd_sense(rdev, radeon_connector->hpd.hpd)) { - kfree(radeon_connector->edid); - radeon_connector->edid = NULL; + radeon_connector_free_edid(connector); ret = connector_status_disconnected; } } @@ -1037,14 +1338,11 @@ radeon_dvi_detect(struct drm_connector *connector, bool force) if (connector->encoder_ids[i] == 0) break; - obj = drm_mode_object_find(connector->dev, - connector->encoder_ids[i], - DRM_MODE_OBJECT_ENCODER); - if (!obj) + encoder = drm_encoder_find(connector->dev, + connector->encoder_ids[i]); + if (!encoder) continue; - encoder = obj_to_encoder(obj); - if (encoder->encoder_type != DRM_MODE_ENCODER_DAC && encoder->encoder_type != DRM_MODE_ENCODER_TVDAC) continue; @@ -1096,6 +1394,22 @@ out: /* updated in get modes as well since we need to know if it's analog or digital */ radeon_connector_update_scratch_regs(connector, ret); + + if ((radeon_audio != 0) && radeon_connector->use_digital) { + const struct drm_connector_helper_funcs *connector_funcs = + connector->helper_private; + + encoder = connector_funcs->best_encoder(connector); + if (encoder && (encoder->encoder_type == DRM_MODE_ENCODER_TMDS)) { + radeon_connector_get_edid(connector); + radeon_audio_detect(connector, encoder, ret); + } + } + +exit: + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); + return ret; } @@ -1104,19 +1418,16 @@ static struct drm_encoder *radeon_dvi_encoder(struct drm_connector *connector) { int enc_id = connector->encoder_ids[0]; struct radeon_connector *radeon_connector = to_radeon_connector(connector); - struct drm_mode_object *obj; struct drm_encoder *encoder; int i; for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) { if (connector->encoder_ids[i] == 0) break; - obj = drm_mode_object_find(connector->dev, connector->encoder_ids[i], DRM_MODE_OBJECT_ENCODER); - if (!obj) + encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]); + if (!encoder) continue; - encoder = obj_to_encoder(obj); - if (radeon_connector->use_digital == true) { if (encoder->encoder_type == DRM_MODE_ENCODER_TMDS) return encoder; @@ -1131,13 +1442,8 @@ static struct drm_encoder *radeon_dvi_encoder(struct drm_connector *connector) /* then check use digitial */ /* pick the first one */ - if (enc_id) { - obj = drm_mode_object_find(connector->dev, enc_id, DRM_MODE_OBJECT_ENCODER); - if (!obj) - return NULL; - encoder = obj_to_encoder(obj); - return encoder; - } + if (enc_id) + return drm_encoder_find(connector->dev, enc_id); return NULL; } @@ -1170,17 +1476,15 @@ static int radeon_dvi_mode_valid(struct drm_connector *connector, (radeon_connector->connector_object_id == CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D) || (radeon_connector->connector_object_id == CONNECTOR_OBJECT_ID_HDMI_TYPE_B)) return MODE_OK; - else if (radeon_connector->connector_object_id == CONNECTOR_OBJECT_ID_HDMI_TYPE_A) { - if (ASIC_IS_DCE6(rdev)) { - /* HDMI 1.3+ supports max clock of 340 Mhz */ - if (mode->clock > 340000) - return MODE_CLOCK_HIGH; - else - return MODE_OK; - } else + else if (ASIC_IS_DCE6(rdev) && drm_detect_hdmi_monitor(radeon_connector_edid(connector))) { + /* HDMI 1.3+ supports max clock of 340 Mhz */ + if (mode->clock > 340000) return MODE_CLOCK_HIGH; - } else + else + return MODE_OK; + } else { return MODE_CLOCK_HIGH; + } } /* check against the max pixel clock */ @@ -1191,7 +1495,7 @@ static int radeon_dvi_mode_valid(struct drm_connector *connector, } static const struct drm_connector_helper_funcs radeon_dvi_connector_helper_funcs = { - .get_modes = radeon_dvi_get_modes, + .get_modes = radeon_vga_get_modes, .mode_valid = radeon_dvi_mode_valid, .best_encoder = radeon_dvi_encoder, }; @@ -1205,21 +1509,6 @@ static const struct drm_connector_funcs radeon_dvi_connector_funcs = { .force = radeon_dvi_force, }; -static void radeon_dp_connector_destroy(struct drm_connector *connector) -{ - struct radeon_connector *radeon_connector = to_radeon_connector(connector); - struct radeon_connector_atom_dig *radeon_dig_connector = radeon_connector->con_priv; - - if (radeon_connector->edid) - kfree(radeon_connector->edid); - if (radeon_dig_connector->dp_i2c_bus) - radeon_i2c_destroy(radeon_dig_connector->dp_i2c_bus); - kfree(radeon_connector->con_priv); - drm_connector_unregister(connector); - drm_connector_cleanup(connector); - kfree(connector); -} - static int radeon_dp_get_modes(struct drm_connector *connector) { struct radeon_connector *radeon_connector = to_radeon_connector(connector); @@ -1235,7 +1524,8 @@ static int radeon_dp_get_modes(struct drm_connector *connector) if (!radeon_dig_connector->edp_on) atombios_set_edp_panel_power(connector, ATOM_TRANSMITTER_ACTION_POWER_ON); - ret = radeon_ddc_get_modes(radeon_connector); + radeon_connector_get_edid(connector); + ret = radeon_ddc_get_modes(connector); if (!radeon_dig_connector->edp_on) atombios_set_edp_panel_power(connector, ATOM_TRANSMITTER_ACTION_POWER_OFF); @@ -1246,7 +1536,8 @@ static int radeon_dp_get_modes(struct drm_connector *connector) if (encoder) radeon_atom_ext_encoder_setup_ddc(encoder); } - ret = radeon_ddc_get_modes(radeon_connector); + radeon_connector_get_edid(connector); + ret = radeon_ddc_get_modes(connector); } if (ret > 0) { @@ -1279,7 +1570,10 @@ static int radeon_dp_get_modes(struct drm_connector *connector) if (encoder) radeon_atom_ext_encoder_setup_ddc(encoder); } - ret = radeon_ddc_get_modes(radeon_connector); + radeon_connector_get_edid(connector); + ret = radeon_ddc_get_modes(connector); + + radeon_get_native_mode(connector); } return ret; @@ -1287,7 +1581,6 @@ static int radeon_dp_get_modes(struct drm_connector *connector) u16 radeon_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *connector) { - struct drm_mode_object *obj; struct drm_encoder *encoder; struct radeon_encoder *radeon_encoder; int i; @@ -1296,11 +1589,10 @@ u16 radeon_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *conn if (connector->encoder_ids[i] == 0) break; - obj = drm_mode_object_find(connector->dev, connector->encoder_ids[i], DRM_MODE_OBJECT_ENCODER); - if (!obj) + encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]); + if (!encoder) continue; - encoder = obj_to_encoder(obj); radeon_encoder = to_radeon_encoder(encoder); switch (radeon_encoder->encoder_id) { @@ -1315,9 +1607,8 @@ u16 radeon_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *conn return ENCODER_OBJECT_ID_NONE; } -bool radeon_connector_encoder_is_hbr2(struct drm_connector *connector) +static bool radeon_connector_encoder_is_hbr2(struct drm_connector *connector) { - struct drm_mode_object *obj; struct drm_encoder *encoder; struct radeon_encoder *radeon_encoder; int i; @@ -1327,11 +1618,10 @@ bool radeon_connector_encoder_is_hbr2(struct drm_connector *connector) if (connector->encoder_ids[i] == 0) break; - obj = drm_mode_object_find(connector->dev, connector->encoder_ids[i], DRM_MODE_OBJECT_ENCODER); - if (!obj) + encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]); + if (!encoder) continue; - encoder = obj_to_encoder(obj); radeon_encoder = to_radeon_encoder(encoder); if (radeon_encoder->caps & ATOM_ENCODER_CAP_RECORD_HBR2) found = true; @@ -1363,15 +1653,22 @@ radeon_dp_detect(struct drm_connector *connector, bool force) enum drm_connector_status ret = connector_status_disconnected; struct radeon_connector_atom_dig *radeon_dig_connector = radeon_connector->con_priv; struct drm_encoder *encoder = radeon_best_single_encoder(connector); + int r; - if (!force && radeon_check_hpd_status_unchanged(connector)) - return connector->status; + if (radeon_dig_connector->is_mst) + return connector_status_disconnected; - if (radeon_connector->edid) { - kfree(radeon_connector->edid); - radeon_connector->edid = NULL; + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; + + if (!force && radeon_check_hpd_status_unchanged(connector)) { + ret = connector->status; + goto out; } + radeon_connector_free_edid(connector); + if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) || (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) { if (encoder) { @@ -1381,6 +1678,11 @@ radeon_dp_detect(struct drm_connector *connector, bool force) /* check if panel is valid */ if (native_mode->hdisplay >= 320 && native_mode->vdisplay >= 240) ret = connector_status_connected; + /* don't fetch the edid from the vbios if ddc fails and runpm is + * enabled so we report disconnected. + */ + if ((rdev->flags & RADEON_IS_PX) && (radeon_runtime_pm != 0)) + ret = connector_status_disconnected; } /* eDP is always DP */ radeon_dig_connector->dp_sink_type = CONNECTOR_OBJECT_ID_DISPLAYPORT; @@ -1414,14 +1716,23 @@ radeon_dp_detect(struct drm_connector *connector, bool force) radeon_dig_connector->dp_sink_type = radeon_dp_getsinktype(radeon_connector); if (radeon_hpd_sense(rdev, radeon_connector->hpd.hpd)) { ret = connector_status_connected; - if (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) + if (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) { radeon_dp_getdpcd(radeon_connector); + r = radeon_dp_mst_probe(radeon_connector); + if (r == 1) + ret = connector_status_disconnected; + } } else { if (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) { - if (radeon_dp_getdpcd(radeon_connector)) - ret = connector_status_connected; + if (radeon_dp_getdpcd(radeon_connector)) { + r = radeon_dp_mst_probe(radeon_connector); + if (r == 1) + ret = connector_status_disconnected; + else + ret = connector_status_connected; + } } else { - /* try non-aux ddc (DP to DVI/HMDI/etc. adapter) */ + /* try non-aux ddc (DP to DVI/HDMI/etc. adapter) */ if (radeon_ddc_probe(radeon_connector, false)) ret = connector_status_connected; } @@ -1429,12 +1740,24 @@ radeon_dp_detect(struct drm_connector *connector, bool force) } radeon_connector_update_scratch_regs(connector, ret); + + if ((radeon_audio != 0) && encoder) { + radeon_connector_get_edid(connector); + radeon_audio_detect(connector, encoder, ret); + } + +out: + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); + return ret; } static int radeon_dp_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { + struct drm_device *dev = connector->dev; + struct radeon_device *rdev = dev->dev_private; struct radeon_connector *radeon_connector = to_radeon_connector(connector); struct radeon_connector_atom_dig *radeon_dig_connector = radeon_connector->con_priv; @@ -1465,14 +1788,23 @@ static int radeon_dp_mode_valid(struct drm_connector *connector, return MODE_PANEL; } } - return MODE_OK; } else { if ((radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || - (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) + (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) { return radeon_dp_mode_valid_helper(connector, mode); - else - return MODE_OK; + } else { + if (ASIC_IS_DCE6(rdev) && drm_detect_hdmi_monitor(radeon_connector_edid(connector))) { + /* HDMI 1.3+ supports max clock of 340 Mhz */ + if (mode->clock > 340000) + return MODE_CLOCK_HIGH; + } else { + if (mode->clock > 165000) + return MODE_CLOCK_HIGH; + } + } } + + return MODE_OK; } static const struct drm_connector_helper_funcs radeon_dp_connector_helper_funcs = { @@ -1486,7 +1818,7 @@ static const struct drm_connector_funcs radeon_dp_connector_funcs = { .detect = radeon_dp_detect, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = radeon_connector_set_property, - .destroy = radeon_dp_connector_destroy, + .destroy = radeon_connector_destroy, .force = radeon_dvi_force, }; @@ -1495,7 +1827,7 @@ static const struct drm_connector_funcs radeon_edp_connector_funcs = { .detect = radeon_dp_detect, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = radeon_lvds_set_property, - .destroy = radeon_dp_connector_destroy, + .destroy = radeon_connector_destroy, .force = radeon_dvi_force, }; @@ -1504,7 +1836,7 @@ static const struct drm_connector_funcs radeon_lvds_bridge_connector_funcs = { .detect = radeon_dp_detect, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = radeon_lvds_set_property, - .destroy = radeon_dp_connector_destroy, + .destroy = radeon_connector_destroy, .force = radeon_dvi_force, }; @@ -1621,6 +1953,13 @@ radeon_add_atom_connector(struct drm_device *dev, drm_object_attach_property(&radeon_connector->base.base, rdev->mode_info.load_detect_property, 1); + drm_object_attach_property(&radeon_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + if (ASIC_IS_DCE5(rdev)) + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.output_csc_property, + RADEON_OUTPUT_CSC_BYPASS); break; case DRM_MODE_CONNECTOR_DVII: case DRM_MODE_CONNECTOR_DVID: @@ -1640,6 +1979,26 @@ radeon_add_atom_connector(struct drm_device *dev, drm_object_attach_property(&radeon_connector->base.base, rdev->mode_info.underscan_vborder_property, 0); + + drm_object_attach_property(&radeon_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.dither_property, + RADEON_FMT_DITHER_DISABLE); + + if (radeon_audio != 0) { + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.audio_property, + RADEON_AUDIO_AUTO); + radeon_connector->audio = RADEON_AUDIO_AUTO; + } + if (ASIC_IS_DCE5(rdev)) + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.output_csc_property, + RADEON_OUTPUT_CSC_BYPASS); + subpixel_order = SubPixelHorizontalRGB; connector->interlace_allowed = true; if (connector_type == DRM_MODE_CONNECTOR_HDMIB) @@ -1681,9 +2040,16 @@ radeon_add_atom_connector(struct drm_device *dev, drm_object_attach_property(&radeon_connector->base.base, rdev->mode_info.load_detect_property, 1); + if (ASIC_IS_AVIVO(rdev)) + drm_object_attach_property(&radeon_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + if (ASIC_IS_DCE5(rdev)) + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.output_csc_property, + RADEON_OUTPUT_CSC_BYPASS); /* no HPD on analog connectors */ radeon_connector->hpd.hpd = RADEON_HPD_NONE; - connector->polled = DRM_CONNECTOR_POLL_CONNECT; connector->interlace_allowed = true; connector->doublescan_allowed = true; break; @@ -1699,6 +2065,14 @@ radeon_add_atom_connector(struct drm_device *dev, drm_object_attach_property(&radeon_connector->base.base, rdev->mode_info.load_detect_property, 1); + if (ASIC_IS_AVIVO(rdev)) + drm_object_attach_property(&radeon_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + if (ASIC_IS_DCE5(rdev)) + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.output_csc_property, + RADEON_OUTPUT_CSC_BYPASS); /* no HPD on analog connectors */ radeon_connector->hpd.hpd = RADEON_HPD_NONE; connector->interlace_allowed = true; @@ -1732,6 +2106,18 @@ radeon_add_atom_connector(struct drm_device *dev, drm_object_attach_property(&radeon_connector->base.base, rdev->mode_info.underscan_vborder_property, 0); + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.dither_property, + RADEON_FMT_DITHER_DISABLE); + drm_object_attach_property(&radeon_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + } + if (ASIC_IS_DCE2(rdev) && (radeon_audio != 0)) { + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.audio_property, + RADEON_AUDIO_AUTO); + radeon_connector->audio = RADEON_AUDIO_AUTO; } if (connector_type == DRM_MODE_CONNECTOR_DVII) { radeon_connector->dac_load_detect = true; @@ -1739,6 +2125,10 @@ radeon_add_atom_connector(struct drm_device *dev, rdev->mode_info.load_detect_property, 1); } + if (ASIC_IS_DCE5(rdev)) + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.output_csc_property, + RADEON_OUTPUT_CSC_BYPASS); connector->interlace_allowed = true; if (connector_type == DRM_MODE_CONNECTOR_DVII) connector->doublescan_allowed = true; @@ -1772,7 +2162,23 @@ radeon_add_atom_connector(struct drm_device *dev, drm_object_attach_property(&radeon_connector->base.base, rdev->mode_info.underscan_vborder_property, 0); + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.dither_property, + RADEON_FMT_DITHER_DISABLE); + drm_object_attach_property(&radeon_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + } + if (ASIC_IS_DCE2(rdev) && (radeon_audio != 0)) { + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.audio_property, + RADEON_AUDIO_AUTO); + radeon_connector->audio = RADEON_AUDIO_AUTO; } + if (ASIC_IS_DCE5(rdev)) + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.output_csc_property, + RADEON_OUTPUT_CSC_BYPASS); subpixel_order = SubPixelHorizontalRGB; connector->interlace_allowed = true; if (connector_type == DRM_MODE_CONNECTOR_HDMIB) @@ -1809,7 +2215,23 @@ radeon_add_atom_connector(struct drm_device *dev, drm_object_attach_property(&radeon_connector->base.base, rdev->mode_info.underscan_vborder_property, 0); + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.dither_property, + RADEON_FMT_DITHER_DISABLE); + drm_object_attach_property(&radeon_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + } + if (ASIC_IS_DCE2(rdev) && (radeon_audio != 0)) { + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.audio_property, + RADEON_AUDIO_AUTO); + radeon_connector->audio = RADEON_AUDIO_AUTO; } + if (ASIC_IS_DCE5(rdev)) + drm_object_attach_property(&radeon_connector->base.base, + rdev->mode_info.output_csc_property, + RADEON_OUTPUT_CSC_BYPASS); connector->interlace_allowed = true; /* in theory with a DP to VGA converter... */ connector->doublescan_allowed = false; @@ -1877,8 +2299,10 @@ radeon_add_atom_connector(struct drm_device *dev, } if (radeon_connector->hpd.hpd == RADEON_HPD_NONE) { - if (i2c_bus->valid) - connector->polled = DRM_CONNECTOR_POLL_CONNECT; + if (i2c_bus->valid) { + connector->polled = DRM_CONNECTOR_POLL_CONNECT | + DRM_CONNECTOR_POLL_DISCONNECT; + } } else connector->polled = DRM_CONNECTOR_POLL_HPD; @@ -1954,7 +2378,6 @@ radeon_add_legacy_connector(struct drm_device *dev, 1); /* no HPD on analog connectors */ radeon_connector->hpd.hpd = RADEON_HPD_NONE; - connector->polled = DRM_CONNECTOR_POLL_CONNECT; connector->interlace_allowed = true; connector->doublescan_allowed = true; break; @@ -2039,10 +2462,37 @@ radeon_add_legacy_connector(struct drm_device *dev, } if (radeon_connector->hpd.hpd == RADEON_HPD_NONE) { - if (i2c_bus->valid) - connector->polled = DRM_CONNECTOR_POLL_CONNECT; + if (i2c_bus->valid) { + connector->polled = DRM_CONNECTOR_POLL_CONNECT | + DRM_CONNECTOR_POLL_DISCONNECT; + } } else connector->polled = DRM_CONNECTOR_POLL_HPD; + connector->display_info.subpixel_order = subpixel_order; drm_connector_register(connector); } + +void radeon_setup_mst_connector(struct drm_device *dev) +{ + struct radeon_device *rdev = dev->dev_private; + struct drm_connector *connector; + struct radeon_connector *radeon_connector; + + if (!ASIC_IS_DCE5(rdev)) + return; + + if (radeon_mst == 0) + return; + + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + int ret; + + radeon_connector = to_radeon_connector(connector); + + if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort) + continue; + + ret = radeon_dp_mst_init(radeon_connector); + } +} diff --git a/sys/dev/pci/drm/radeon/radeon_cs.c b/sys/dev/pci/drm/radeon/radeon_cs.c index 8a784fe813c..6f35c794d11 100644 --- a/sys/dev/pci/drm/radeon/radeon_cs.c +++ b/sys/dev/pci/drm/radeon/radeon_cs.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_cs.c,v 1.6 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Jerome Glisse. * All Rights Reserved. @@ -29,68 +28,166 @@ #include #include "radeon_reg.h" #include "radeon.h" +#include "radeon_trace.h" -void r100_cs_dump_packet(struct radeon_cs_parser *p, - struct radeon_cs_packet *pkt); +#define RADEON_CS_MAX_PRIORITY 32u +#define RADEON_CS_NUM_BUCKETS (RADEON_CS_MAX_PRIORITY + 1) + +/* This is based on the bucket sort with O(n) time complexity. + * An item with priority "i" is added to bucket[i]. The lists are then + * concatenated in descending order. + */ +struct radeon_cs_buckets { + struct list_head bucket[RADEON_CS_NUM_BUCKETS]; +}; + +static void radeon_cs_buckets_init(struct radeon_cs_buckets *b) +{ + unsigned i; + + for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++) + INIT_LIST_HEAD(&b->bucket[i]); +} + +static void radeon_cs_buckets_add(struct radeon_cs_buckets *b, + struct list_head *item, unsigned priority) +{ + /* Since buffers which appear sooner in the relocation list are + * likely to be used more often than buffers which appear later + * in the list, the sort mustn't change the ordering of buffers + * with the same priority, i.e. it must be stable. + */ + list_add_tail(item, &b->bucket[min(priority, RADEON_CS_MAX_PRIORITY)]); +} + +static void radeon_cs_buckets_get_list(struct radeon_cs_buckets *b, + struct list_head *out_list) +{ + unsigned i; + + /* Connect the sorted buckets in the output list. */ + for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++) { + list_splice(&b->bucket[i], out_list); + } +} static int radeon_cs_parser_relocs(struct radeon_cs_parser *p) { struct drm_device *ddev = p->rdev->ddev; struct radeon_cs_chunk *chunk; - unsigned i, j; - bool duplicate; + struct radeon_cs_buckets buckets; + unsigned i; + bool need_mmap_lock = false; + int r; - if (p->chunk_relocs_idx == -1) { + if (p->chunk_relocs == NULL) { return 0; } - chunk = &p->chunks[p->chunk_relocs_idx]; + chunk = p->chunk_relocs; p->dma_reloc_idx = 0; /* FIXME: we assume that each relocs use 4 dwords */ p->nrelocs = chunk->length_dw / 4; - p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL); - if (p->relocs_ptr == NULL) { - return -ENOMEM; - } - p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL); + p->relocs = drm_calloc_large(p->nrelocs, sizeof(struct radeon_bo_list)); if (p->relocs == NULL) { return -ENOMEM; } + + radeon_cs_buckets_init(&buckets); + for (i = 0; i < p->nrelocs; i++) { struct drm_radeon_cs_reloc *r; + struct drm_gem_object *gobj; + unsigned priority; - duplicate = false; r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4]; - for (j = 0; j < i; j++) { - if (r->handle == p->relocs[j].handle) { - p->relocs_ptr[i] = &p->relocs[j]; - duplicate = true; - break; + gobj = drm_gem_object_lookup(ddev, p->filp, r->handle); + if (gobj == NULL) { + DRM_ERROR("gem object lookup failed 0x%x\n", + r->handle); + return -ENOENT; + } + p->relocs[i].robj = gem_to_radeon_bo(gobj); + + /* The userspace buffer priorities are from 0 to 15. A higher + * number means the buffer is more important. + * Also, the buffers used for write have a higher priority than + * the buffers used for read only, which doubles the range + * to 0 to 31. 32 is reserved for the kernel driver. + */ + priority = (r->flags & RADEON_RELOC_PRIO_MASK) * 2 + + !!r->write_domain; + + /* the first reloc of an UVD job is the msg and that must be in + VRAM, also but everything into VRAM on AGP cards and older + IGP chips to avoid image corruptions */ + if (p->ring == R600_RING_TYPE_UVD_INDEX && + (i == 0 || (p->rdev->flags & RADEON_IS_AGP) || + p->rdev->family == CHIP_RS780 || + p->rdev->family == CHIP_RS880)) { + + /* TODO: is this still needed for NI+ ? */ + p->relocs[i].prefered_domains = + RADEON_GEM_DOMAIN_VRAM; + + p->relocs[i].allowed_domains = + RADEON_GEM_DOMAIN_VRAM; + + /* prioritize this over any other relocation */ + priority = RADEON_CS_MAX_PRIORITY; + } else { + uint32_t domain = r->write_domain ? + r->write_domain : r->read_domains; + + if (domain & RADEON_GEM_DOMAIN_CPU) { + DRM_ERROR("RADEON_GEM_DOMAIN_CPU is not valid " + "for command submission\n"); + return -EINVAL; } + + p->relocs[i].prefered_domains = domain; + if (domain == RADEON_GEM_DOMAIN_VRAM) + domain |= RADEON_GEM_DOMAIN_GTT; + p->relocs[i].allowed_domains = domain; } - if (!duplicate) { - p->relocs[i].gobj = drm_gem_object_lookup(ddev, - p->filp, - r->handle); - if (p->relocs[i].gobj == NULL) { - DRM_ERROR("gem object lookup failed 0x%x\n", - r->handle); - return -ENOENT; + + if (radeon_ttm_tt_has_userptr(p->relocs[i].robj->tbo.ttm)) { + uint32_t domain = p->relocs[i].prefered_domains; + if (!(domain & RADEON_GEM_DOMAIN_GTT)) { + DRM_ERROR("Only RADEON_GEM_DOMAIN_GTT is " + "allowed for userptr BOs\n"); + return -EINVAL; } - p->relocs_ptr[i] = &p->relocs[i]; - p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj); - p->relocs[i].lobj.bo = p->relocs[i].robj; - p->relocs[i].lobj.wdomain = r->write_domain; - p->relocs[i].lobj.rdomain = r->read_domains; - p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo; - p->relocs[i].handle = r->handle; - p->relocs[i].flags = r->flags; - radeon_bo_list_add_object(&p->relocs[i].lobj, - &p->validated); + need_mmap_lock = true; + domain = RADEON_GEM_DOMAIN_GTT; + p->relocs[i].prefered_domains = domain; + p->relocs[i].allowed_domains = domain; + } - } else - p->relocs[i].handle = 0; + p->relocs[i].tv.bo = &p->relocs[i].robj->tbo; + p->relocs[i].tv.shared = !r->write_domain; + + radeon_cs_buckets_add(&buckets, &p->relocs[i].tv.head, + priority); } - return radeon_bo_list_validate(&p->validated); + + radeon_cs_buckets_get_list(&buckets, &p->validated); + + if (p->cs_flags & RADEON_CS_USE_VM) + p->vm_bos = radeon_vm_get_bos(p->rdev, p->ib.vm, + &p->validated); +#ifdef notyet + if (need_mmap_lock) + down_read(¤t->mm->mmap_sem); +#endif + + r = radeon_bo_list_validate(p->rdev, &p->ticket, &p->validated, p->ring); + +#ifdef notyet + if (need_mmap_lock) + up_read(¤t->mm->mmap_sem); +#endif + + return r; } static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority) @@ -119,38 +216,38 @@ static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority p->ring = R600_RING_TYPE_DMA_INDEX; else p->ring = CAYMAN_RING_TYPE_DMA1_INDEX; - } else if (p->rdev->family >= CHIP_R600) { + } else if (p->rdev->family >= CHIP_RV770) { p->ring = R600_RING_TYPE_DMA_INDEX; } else { return -EINVAL; } break; + case RADEON_CS_RING_UVD: + p->ring = R600_RING_TYPE_UVD_INDEX; + break; + case RADEON_CS_RING_VCE: + /* TODO: only use the low priority ring for now */ + p->ring = TN_RING_TYPE_VCE1_INDEX; + break; } return 0; } -static void radeon_cs_sync_to(struct radeon_cs_parser *p, - struct radeon_fence *fence) +static int radeon_cs_sync_rings(struct radeon_cs_parser *p) { - struct radeon_fence *other; - - if (!fence) - return; - - other = p->ib.sync_to[fence->ring]; - p->ib.sync_to[fence->ring] = radeon_fence_later(fence, other); -} - -static void radeon_cs_sync_rings(struct radeon_cs_parser *p) -{ - int i; + struct radeon_bo_list *reloc; + int r; - for (i = 0; i < p->nrelocs; i++) { - if (!p->relocs[i].robj) - continue; + list_for_each_entry(reloc, &p->validated, tv.head) { + struct reservation_object *resv; - radeon_cs_sync_to(p, p->relocs[i].robj->tbo.sync_obj); + resv = reloc->robj->tbo.resv; + r = radeon_sync_resv(p->rdev, &p->ib.sync, resv, + reloc->tv.shared); + if (r) + return r; } + return 0; } /* XXX: note that this is called from the legacy UMS CS ioctl as well */ @@ -162,26 +259,26 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data) u32 ring = RADEON_CS_RING_GFX; s32 priority = 0; + INIT_LIST_HEAD(&p->validated); + if (!cs->num_chunks) { return 0; } + /* get chunks */ - INIT_LIST_HEAD(&p->validated); p->idx = 0; p->ib.sa_bo = NULL; - p->ib.semaphore = NULL; p->const_ib.sa_bo = NULL; - p->const_ib.semaphore = NULL; - p->chunk_ib_idx = -1; - p->chunk_relocs_idx = -1; - p->chunk_flags_idx = -1; - p->chunk_const_ib_idx = -1; + p->chunk_ib = NULL; + p->chunk_relocs = NULL; + p->chunk_flags = NULL; + p->chunk_const_ib = NULL; p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL); if (p->chunks_array == NULL) { return -ENOMEM; } chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks); - if (DRM_COPY_FROM_USER(p->chunks_array, chunk_array_ptr, + if (copy_from_user(p->chunks_array, chunk_array_ptr, sizeof(uint64_t)*cs->num_chunks)) { return -EFAULT; } @@ -197,58 +294,58 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data) uint32_t __user *cdata; chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i]; - if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr, + if (copy_from_user(&user_chunk, chunk_ptr, sizeof(struct drm_radeon_cs_chunk))) { return -EFAULT; } p->chunks[i].length_dw = user_chunk.length_dw; - p->chunks[i].kdata = NULL; - p->chunks[i].chunk_id = user_chunk.chunk_id; - - if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) { - p->chunk_relocs_idx = i; + if (user_chunk.chunk_id == RADEON_CHUNK_ID_RELOCS) { + p->chunk_relocs = &p->chunks[i]; } - if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) { - p->chunk_ib_idx = i; + if (user_chunk.chunk_id == RADEON_CHUNK_ID_IB) { + p->chunk_ib = &p->chunks[i]; /* zero length IB isn't useful */ if (p->chunks[i].length_dw == 0) return -EINVAL; } - if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) { - p->chunk_const_ib_idx = i; + if (user_chunk.chunk_id == RADEON_CHUNK_ID_CONST_IB) { + p->chunk_const_ib = &p->chunks[i]; /* zero length CONST IB isn't useful */ if (p->chunks[i].length_dw == 0) return -EINVAL; } - if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) { - p->chunk_flags_idx = i; + if (user_chunk.chunk_id == RADEON_CHUNK_ID_FLAGS) { + p->chunk_flags = &p->chunks[i]; /* zero length flags aren't useful */ if (p->chunks[i].length_dw == 0) return -EINVAL; } - p->chunks[i].length_dw = user_chunk.length_dw; - p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data; - - cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data; - if ((p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) || - (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS)) { - size = p->chunks[i].length_dw * sizeof(uint32_t); - p->chunks[i].kdata = kmalloc(size, GFP_KERNEL); - if (p->chunks[i].kdata == NULL) { - return -ENOMEM; - } - if (DRM_COPY_FROM_USER(p->chunks[i].kdata, - p->chunks[i].user_ptr, size)) { - return -EFAULT; - } - if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) { - p->cs_flags = p->chunks[i].kdata[0]; - if (p->chunks[i].length_dw > 1) - ring = p->chunks[i].kdata[1]; - if (p->chunks[i].length_dw > 2) - priority = (s32)p->chunks[i].kdata[2]; - } + size = p->chunks[i].length_dw; + cdata = (void __user *)(unsigned long)user_chunk.chunk_data; + p->chunks[i].user_ptr = cdata; + if (user_chunk.chunk_id == RADEON_CHUNK_ID_CONST_IB) + continue; + + if (user_chunk.chunk_id == RADEON_CHUNK_ID_IB) { + if (!p->rdev || !(p->rdev->flags & RADEON_IS_AGP)) + continue; + } + + p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t)); + size *= sizeof(uint32_t); + if (p->chunks[i].kdata == NULL) { + return -ENOMEM; + } + if (copy_from_user(p->chunks[i].kdata, cdata, size)) { + return -EFAULT; + } + if (user_chunk.chunk_id == RADEON_CHUNK_ID_FLAGS) { + p->cs_flags = p->chunks[i].kdata[0]; + if (p->chunks[i].length_dw > 1) + ring = p->chunks[i].kdata[1]; + if (p->chunks[i].length_dw > 2) + priority = (s32)p->chunks[i].kdata[2]; } } @@ -260,48 +357,37 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data) return -EINVAL; } - /* we only support VM on SI+ */ - if ((p->rdev->family >= CHIP_TAHITI) && - ((p->cs_flags & RADEON_CS_USE_VM) == 0)) { - DRM_ERROR("VM required on SI+!\n"); - return -EINVAL; - } - if (radeon_cs_get_ring(p, ring, priority)) return -EINVAL; - } - /* deal with non-vm */ - if ((p->chunk_ib_idx != -1) && - ((p->cs_flags & RADEON_CS_USE_VM) == 0) && - (p->chunks[p->chunk_ib_idx].chunk_id == RADEON_CHUNK_ID_IB)) { - if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) { - DRM_ERROR("cs IB too big: %d\n", - p->chunks[p->chunk_ib_idx].length_dw); - return -EINVAL; - } - if (p->rdev && (p->rdev->flags & RADEON_IS_AGP)) { - p->chunks[p->chunk_ib_idx].kpage[0] = kmalloc(PAGE_SIZE, GFP_KERNEL); - p->chunks[p->chunk_ib_idx].kpage[1] = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (p->chunks[p->chunk_ib_idx].kpage[0] == NULL || - p->chunks[p->chunk_ib_idx].kpage[1] == NULL) { - kfree(p->chunks[p->chunk_ib_idx].kpage[0]); - kfree(p->chunks[p->chunk_ib_idx].kpage[1]); - p->chunks[p->chunk_ib_idx].kpage[0] = NULL; - p->chunks[p->chunk_ib_idx].kpage[1] = NULL; - return -ENOMEM; + /* we only support VM on some SI+ rings */ + if ((p->cs_flags & RADEON_CS_USE_VM) == 0) { + if (p->rdev->asic->ring[p->ring]->cs_parse == NULL) { + DRM_ERROR("Ring %d requires VM!\n", p->ring); + return -EINVAL; + } + } else { + if (p->rdev->asic->ring[p->ring]->ib_parse == NULL) { + DRM_ERROR("VM not supported on ring %d!\n", + p->ring); + return -EINVAL; } } - p->chunks[p->chunk_ib_idx].kpage_idx[0] = -1; - p->chunks[p->chunk_ib_idx].kpage_idx[1] = -1; - p->chunks[p->chunk_ib_idx].last_copied_page = -1; - p->chunks[p->chunk_ib_idx].last_page_index = - ((p->chunks[p->chunk_ib_idx].length_dw * 4) - 1) / PAGE_SIZE; } return 0; } +static int cmp_size_smaller_first(void *priv, struct list_head *a, + struct list_head *b) +{ + struct radeon_bo_list *la = list_entry(a, struct radeon_bo_list, tv.head); + struct radeon_bo_list *lb = list_entry(b, struct radeon_bo_list, tv.head); + + /* Sort A before B if A is smaller. */ + return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages; +} + /** * cs_parser_fini() - clean parser states * @parser: parser structure holding parsing context. @@ -310,33 +396,45 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data) * If error is set than unvalidate buffer, otherwise just free memory * used by parsing context. **/ -static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error) +static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bool backoff) { unsigned i; if (!error) { - ttm_eu_fence_buffer_objects(&parser->validated, - parser->ib.fence); - } else { - ttm_eu_backoff_reservation(&parser->validated); + /* Sort the buffer list from the smallest to largest buffer, + * which affects the order of buffers in the LRU list. + * This assures that the smallest buffers are added first + * to the LRU list, so they are likely to be later evicted + * first, instead of large buffers whose eviction is more + * expensive. + * + * This slightly lowers the number of bytes moved by TTM + * per frame under memory pressure. + */ + list_sort(NULL, &parser->validated, cmp_size_smaller_first); + + ttm_eu_fence_buffer_objects(&parser->ticket, + &parser->validated, + &parser->ib.fence->base); + } else if (backoff) { + ttm_eu_backoff_reservation(&parser->ticket, + &parser->validated); } if (parser->relocs != NULL) { for (i = 0; i < parser->nrelocs; i++) { - if (parser->relocs[i].gobj) - drm_gem_object_unreference_unlocked(parser->relocs[i].gobj); + struct radeon_bo *bo = parser->relocs[i].robj; + if (bo == NULL) + continue; + + drm_gem_object_unreference_unlocked(&bo->gem_base); } } kfree(parser->track); - kfree(parser->relocs); - kfree(parser->relocs_ptr); - for (i = 0; i < parser->nchunks; i++) { - kfree(parser->chunks[i].kdata); - if ((parser->rdev->flags & RADEON_IS_AGP)) { - kfree(parser->chunks[i].kpage[0]); - kfree(parser->chunks[i].kpage[1]); - } - } + drm_free_large(parser->relocs); + drm_free_large(parser->vm_bos); + for (i = 0; i < parser->nchunks; i++) + drm_free_large(parser->chunks[i].kdata); kfree(parser->chunks); kfree(parser->chunks_array); radeon_ib_free(parser->rdev, &parser->ib); @@ -346,157 +444,134 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error) static int radeon_cs_ib_chunk(struct radeon_device *rdev, struct radeon_cs_parser *parser) { - struct radeon_cs_chunk *ib_chunk; int r; - if (parser->chunk_ib_idx == -1) + if (parser->chunk_ib == NULL) return 0; if (parser->cs_flags & RADEON_CS_USE_VM) return 0; - ib_chunk = &parser->chunks[parser->chunk_ib_idx]; - /* Copy the packet into the IB, the parser will read from the - * input memory (cached) and write to the IB (which can be - * uncached). - */ - r = radeon_ib_get(rdev, parser->ring, &parser->ib, - NULL, ib_chunk->length_dw * 4); - if (r) { - DRM_ERROR("Failed to get ib !\n"); - return r; - } - parser->ib.length_dw = ib_chunk->length_dw; r = radeon_cs_parse(rdev, parser->ring, parser); if (r || parser->parser_error) { DRM_ERROR("Invalid command stream !\n"); return r; } - r = radeon_cs_finish_pages(parser); + + r = radeon_cs_sync_rings(parser); if (r) { - DRM_ERROR("Invalid command stream !\n"); + if (r != -ERESTARTSYS) + DRM_ERROR("Failed to sync rings: %i\n", r); return r; } - radeon_cs_sync_rings(parser); - r = radeon_ib_schedule(rdev, &parser->ib, NULL); + + if (parser->ring == R600_RING_TYPE_UVD_INDEX) + radeon_uvd_note_usage(rdev); + else if ((parser->ring == TN_RING_TYPE_VCE1_INDEX) || + (parser->ring == TN_RING_TYPE_VCE2_INDEX)) + radeon_vce_note_usage(rdev); + + r = radeon_ib_schedule(rdev, &parser->ib, NULL, true); if (r) { DRM_ERROR("Failed to schedule IB !\n"); } return r; } -static int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser, +static int radeon_bo_vm_update_pte(struct radeon_cs_parser *p, struct radeon_vm *vm) { - struct radeon_device *rdev = parser->rdev; - struct radeon_bo_list *lobj; - struct radeon_bo *bo; - int r; + struct radeon_device *rdev = p->rdev; + struct radeon_bo_va *bo_va; + int i, r; - r = radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo, &rdev->ring_tmp_bo.bo->tbo.mem); - if (r) { + r = radeon_vm_update_page_directory(rdev, vm); + if (r) return r; + + r = radeon_vm_clear_freed(rdev, vm); + if (r) + return r; + + if (vm->ib_bo_va == NULL) { + DRM_ERROR("Tmp BO not in VM!\n"); + return -EINVAL; } - list_for_each_entry(lobj, &parser->validated, tv.head) { - bo = lobj->bo; - r = radeon_vm_bo_update_pte(parser->rdev, vm, bo, &bo->tbo.mem); - if (r) { - return r; + + r = radeon_vm_bo_update(rdev, vm->ib_bo_va, + &rdev->ring_tmp_bo.bo->tbo.mem); + if (r) + return r; + + for (i = 0; i < p->nrelocs; i++) { + struct radeon_bo *bo; + + bo = p->relocs[i].robj; + bo_va = radeon_vm_bo_find(vm, bo); + if (bo_va == NULL) { + dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm); + return -EINVAL; } + + r = radeon_vm_bo_update(rdev, bo_va, &bo->tbo.mem); + if (r) + return r; + + radeon_sync_fence(&p->ib.sync, bo_va->last_pt_update); } - return 0; + + return radeon_vm_clear_invalids(rdev, vm); } static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, struct radeon_cs_parser *parser) { - struct radeon_cs_chunk *ib_chunk; struct radeon_fpriv *fpriv = parser->filp->driver_priv; struct radeon_vm *vm = &fpriv->vm; int r; - if (parser->chunk_ib_idx == -1) + if (parser->chunk_ib == NULL) return 0; if ((parser->cs_flags & RADEON_CS_USE_VM) == 0) return 0; - if ((rdev->family >= CHIP_TAHITI) && - (parser->chunk_const_ib_idx != -1)) { - ib_chunk = &parser->chunks[parser->chunk_const_ib_idx]; - if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { - DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw); - return -EINVAL; - } - r = radeon_ib_get(rdev, parser->ring, &parser->const_ib, - vm, ib_chunk->length_dw * 4); - if (r) { - DRM_ERROR("Failed to get const ib !\n"); - return r; - } - parser->const_ib.is_const_ib = true; - parser->const_ib.length_dw = ib_chunk->length_dw; - /* Copy the packet into the IB */ - if (DRM_COPY_FROM_USER(parser->const_ib.ptr, ib_chunk->user_ptr, - ib_chunk->length_dw * 4)) { - return -EFAULT; - } + if (parser->const_ib.length_dw) { r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib); if (r) { return r; } } - ib_chunk = &parser->chunks[parser->chunk_ib_idx]; - if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { - DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw); - return -EINVAL; - } - r = radeon_ib_get(rdev, parser->ring, &parser->ib, - vm, ib_chunk->length_dw * 4); - if (r) { - DRM_ERROR("Failed to get ib !\n"); - return r; - } - parser->ib.length_dw = ib_chunk->length_dw; - /* Copy the packet into the IB */ - if (DRM_COPY_FROM_USER(parser->ib.ptr, ib_chunk->user_ptr, - ib_chunk->length_dw * 4)) { - return -EFAULT; - } r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib); if (r) { return r; } - mutex_lock(&rdev->vm_manager.lock); + if (parser->ring == R600_RING_TYPE_UVD_INDEX) + radeon_uvd_note_usage(rdev); + mutex_lock(&vm->mutex); - r = radeon_vm_alloc_pt(rdev, vm); + r = radeon_bo_vm_update_pte(parser, vm); if (r) { goto out; } - r = radeon_bo_vm_update_pte(parser, vm); + + r = radeon_cs_sync_rings(parser); if (r) { + if (r != -ERESTARTSYS) + DRM_ERROR("Failed to sync rings: %i\n", r); goto out; } - radeon_cs_sync_rings(parser); - radeon_cs_sync_to(parser, vm->fence); - radeon_cs_sync_to(parser, radeon_vm_grab_id(rdev, vm, parser->ring)); if ((rdev->family >= CHIP_TAHITI) && - (parser->chunk_const_ib_idx != -1)) { - r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib); + (parser->chunk_const_ib != NULL)) { + r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib, true); } else { - r = radeon_ib_schedule(rdev, &parser->ib, NULL); - } - - if (!r) { - radeon_vm_fence(rdev, vm, parser->ib.fence); + r = radeon_ib_schedule(rdev, &parser->ib, NULL, true); } out: - radeon_vm_add_to_lru(rdev, vm); mutex_unlock(&vm->mutex); - mutex_unlock(&rdev->vm_manager.lock); return r; } @@ -510,6 +585,62 @@ static int radeon_cs_handle_lockup(struct radeon_device *rdev, int r) return r; } +static int radeon_cs_ib_fill(struct radeon_device *rdev, struct radeon_cs_parser *parser) +{ + struct radeon_cs_chunk *ib_chunk; + struct radeon_vm *vm = NULL; + int r; + + if (parser->chunk_ib == NULL) + return 0; + + if (parser->cs_flags & RADEON_CS_USE_VM) { + struct radeon_fpriv *fpriv = parser->filp->driver_priv; + vm = &fpriv->vm; + + if ((rdev->family >= CHIP_TAHITI) && + (parser->chunk_const_ib != NULL)) { + ib_chunk = parser->chunk_const_ib; + if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { + DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw); + return -EINVAL; + } + r = radeon_ib_get(rdev, parser->ring, &parser->const_ib, + vm, ib_chunk->length_dw * 4); + if (r) { + DRM_ERROR("Failed to get const ib !\n"); + return r; + } + parser->const_ib.is_const_ib = true; + parser->const_ib.length_dw = ib_chunk->length_dw; + if (copy_from_user(parser->const_ib.ptr, + ib_chunk->user_ptr, + ib_chunk->length_dw * 4)) + return -EFAULT; + } + + ib_chunk = parser->chunk_ib; + if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { + DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw); + return -EINVAL; + } + } + ib_chunk = parser->chunk_ib; + + r = radeon_ib_get(rdev, parser->ring, &parser->ib, + vm, ib_chunk->length_dw * 4); + if (r) { + DRM_ERROR("Failed to get ib !\n"); + return r; + } + parser->ib.length_dw = ib_chunk->length_dw; + if (ib_chunk->kdata) + memcpy(parser->ib.ptr, ib_chunk->kdata, ib_chunk->length_dw * 4); + else if (copy_from_user(parser->ib.ptr, ib_chunk->user_ptr, ib_chunk->length_dw * 4)) + return -EFAULT; + return 0; +} + int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { struct radeon_device *rdev = dev->dev_private; @@ -521,31 +652,44 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) up_read(&rdev->exclusive_lock); return -EBUSY; } + if (rdev->in_reset) { + up_read(&rdev->exclusive_lock); + r = radeon_gpu_reset(rdev); + if (!r) + r = -EAGAIN; + return r; + } /* initialize parser */ memset(&parser, 0, sizeof(struct radeon_cs_parser)); parser.filp = filp; parser.rdev = rdev; -#ifdef notyet parser.dev = rdev->dev; -#endif parser.family = rdev->family; r = radeon_cs_parser_init(&parser, data); if (r) { DRM_ERROR("Failed to initialize parser !\n"); - radeon_cs_parser_fini(&parser, r); + radeon_cs_parser_fini(&parser, r, false); up_read(&rdev->exclusive_lock); r = radeon_cs_handle_lockup(rdev, r); return r; } - r = radeon_cs_parser_relocs(&parser); - if (r) { - if (r != -ERESTARTSYS) + + r = radeon_cs_ib_fill(rdev, &parser); + if (!r) { + r = radeon_cs_parser_relocs(&parser); + if (r && r != -ERESTARTSYS) DRM_ERROR("Failed to parse relocation %d!\n", r); - radeon_cs_parser_fini(&parser, r); + } + + if (r) { + radeon_cs_parser_fini(&parser, r, false); up_read(&rdev->exclusive_lock); r = radeon_cs_handle_lockup(rdev, r); return r; } + + trace_radeon_cs(&parser); + r = radeon_cs_ib_chunk(rdev, &parser); if (r) { goto out; @@ -555,99 +699,169 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) goto out; } out: - radeon_cs_parser_fini(&parser, r); + radeon_cs_parser_fini(&parser, r, true); up_read(&rdev->exclusive_lock); r = radeon_cs_handle_lockup(rdev, r); return r; } -int radeon_cs_finish_pages(struct radeon_cs_parser *p) +/** + * radeon_cs_packet_parse() - parse cp packet and point ib index to next packet + * @parser: parser structure holding parsing context. + * @pkt: where to store packet information + * + * Assume that chunk_ib_index is properly set. Will return -EINVAL + * if packet is bigger than remaining ib size. or if packets is unknown. + **/ +int radeon_cs_packet_parse(struct radeon_cs_parser *p, + struct radeon_cs_packet *pkt, + unsigned idx) { - struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx]; - int i; - int size = PAGE_SIZE; - - for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) { - if (i == ibc->last_page_index) { - size = (ibc->length_dw * 4) % PAGE_SIZE; - if (size == 0) - size = PAGE_SIZE; - } - - if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)), - ibc->user_ptr + (i * PAGE_SIZE), - size)) - return -EFAULT; + struct radeon_cs_chunk *ib_chunk = p->chunk_ib; + struct radeon_device *rdev = p->rdev; + uint32_t header; + int ret = 0, i; + + if (idx >= ib_chunk->length_dw) { + DRM_ERROR("Can not parse packet at %d after CS end %d !\n", + idx, ib_chunk->length_dw); + return -EINVAL; } - return 0; -} - -static int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx) -{ - int new_page; - struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx]; - int i; - int size = PAGE_SIZE; - bool copy1 = (p->rdev && (p->rdev->flags & RADEON_IS_AGP)) ? - false : true; - - for (i = ibc->last_copied_page + 1; i < pg_idx; i++) { - if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)), - ibc->user_ptr + (i * PAGE_SIZE), - PAGE_SIZE)) { - p->parser_error = -EFAULT; - return 0; - } + header = radeon_get_ib_value(p, idx); + pkt->idx = idx; + pkt->type = RADEON_CP_PACKET_GET_TYPE(header); + pkt->count = RADEON_CP_PACKET_GET_COUNT(header); + pkt->one_reg_wr = 0; + switch (pkt->type) { + case RADEON_PACKET_TYPE0: + if (rdev->family < CHIP_R600) { + pkt->reg = R100_CP_PACKET0_GET_REG(header); + pkt->one_reg_wr = + RADEON_CP_PACKET0_GET_ONE_REG_WR(header); + } else + pkt->reg = R600_CP_PACKET0_GET_REG(header); + break; + case RADEON_PACKET_TYPE3: + pkt->opcode = RADEON_CP_PACKET3_GET_OPCODE(header); + break; + case RADEON_PACKET_TYPE2: + pkt->count = -1; + break; + default: + DRM_ERROR("Unknown packet type %d at %d !\n", pkt->type, idx); + ret = -EINVAL; + goto dump_ib; } - - if (pg_idx == ibc->last_page_index) { - size = (ibc->length_dw * 4) % PAGE_SIZE; - if (size == 0) - size = PAGE_SIZE; + if ((pkt->count + 1 + pkt->idx) >= ib_chunk->length_dw) { + DRM_ERROR("Packet (%d:%d:%d) end after CS buffer (%d) !\n", + pkt->idx, pkt->type, pkt->count, ib_chunk->length_dw); + ret = -EINVAL; + goto dump_ib; } + return 0; - new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1; - if (copy1) - ibc->kpage[new_page] = p->ib.ptr + (pg_idx * (PAGE_SIZE / 4)); - - if (DRM_COPY_FROM_USER(ibc->kpage[new_page], - ibc->user_ptr + (pg_idx * PAGE_SIZE), - size)) { - p->parser_error = -EFAULT; - return 0; +dump_ib: + for (i = 0; i < ib_chunk->length_dw; i++) { + if (i == idx) + printk("\t0x%08x <---\n", radeon_get_ib_value(p, i)); + else + printk("\t0x%08x\n", radeon_get_ib_value(p, i)); } + return ret; +} - /* copy to IB for non single case */ - if (!copy1) - memcpy((void *)(p->ib.ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size); +/** + * radeon_cs_packet_next_is_pkt3_nop() - test if the next packet is P3 NOP + * @p: structure holding the parser context. + * + * Check if the next packet is NOP relocation packet3. + **/ +bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p) +{ + struct radeon_cs_packet p3reloc; + int r; + + r = radeon_cs_packet_parse(p, &p3reloc, p->idx); + if (r) + return false; + if (p3reloc.type != RADEON_PACKET_TYPE3) + return false; + if (p3reloc.opcode != RADEON_PACKET3_NOP) + return false; + return true; +} - ibc->last_copied_page = pg_idx; - ibc->kpage_idx[new_page] = pg_idx; +/** + * radeon_cs_dump_packet() - dump raw packet context + * @p: structure holding the parser context. + * @pkt: structure holding the packet. + * + * Used mostly for debugging and error reporting. + **/ +void radeon_cs_dump_packet(struct radeon_cs_parser *p, + struct radeon_cs_packet *pkt) +{ + volatile uint32_t *ib; + unsigned i; + unsigned idx; - return new_page; + ib = p->ib.ptr; + idx = pkt->idx; + for (i = 0; i <= (pkt->count + 1); i++, idx++) + DRM_INFO("ib[%d]=0x%08X\n", idx, ib[idx]); } -u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx) +/** + * radeon_cs_packet_next_reloc() - parse next (should be reloc) packet + * @parser: parser structure holding parsing context. + * @data: pointer to relocation data + * @offset_start: starting offset + * @offset_mask: offset mask (to align start offset on) + * @reloc: reloc informations + * + * Check if next packet is relocation packet3, do bo validation and compute + * GPU offset using the provided start. + **/ +int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p, + struct radeon_bo_list **cs_reloc, + int nomm) { - struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx]; - u32 pg_idx, pg_offset; - u32 idx_value = 0; - int new_page; - - pg_idx = (idx * 4) / PAGE_SIZE; - pg_offset = (idx * 4) % PAGE_SIZE; - - if (ibc->kpage_idx[0] == pg_idx) - return ibc->kpage[0][pg_offset/4]; - if (ibc->kpage_idx[1] == pg_idx) - return ibc->kpage[1][pg_offset/4]; - - new_page = radeon_cs_update_pages(p, pg_idx); - if (new_page < 0) { - p->parser_error = new_page; - return 0; - } + struct radeon_cs_chunk *relocs_chunk; + struct radeon_cs_packet p3reloc; + unsigned idx; + int r; - idx_value = ibc->kpage[new_page][pg_offset/4]; - return idx_value; + if (p->chunk_relocs == NULL) { + DRM_ERROR("No relocation chunk !\n"); + return -EINVAL; + } + *cs_reloc = NULL; + relocs_chunk = p->chunk_relocs; + r = radeon_cs_packet_parse(p, &p3reloc, p->idx); + if (r) + return r; + p->idx += p3reloc.count + 2; + if (p3reloc.type != RADEON_PACKET_TYPE3 || + p3reloc.opcode != RADEON_PACKET3_NOP) { + DRM_ERROR("No packet3 for relocation for packet at %d.\n", + p3reloc.idx); + radeon_cs_dump_packet(p, &p3reloc); + return -EINVAL; + } + idx = radeon_get_ib_value(p, p3reloc.idx + 1); + if (idx >= relocs_chunk->length_dw) { + DRM_ERROR("Relocs at %d after relocations chunk end %d !\n", + idx, relocs_chunk->length_dw); + radeon_cs_dump_packet(p, &p3reloc); + return -EINVAL; + } + /* FIXME: we assume reloc size is 4 dwords */ + if (nomm) { + *cs_reloc = p->relocs; + (*cs_reloc)->gpu_offset = + (u64)relocs_chunk->kdata[idx + 3] << 32; + (*cs_reloc)->gpu_offset |= relocs_chunk->kdata[idx + 0]; + } else + *cs_reloc = &p->relocs[(idx / 4)]; + return 0; } diff --git a/sys/dev/pci/drm/radeon/radeon_cursor.c b/sys/dev/pci/drm/radeon/radeon_cursor.c index 88a75e3b259..2f4e73496d0 100644 --- a/sys/dev/pci/drm/radeon/radeon_cursor.c +++ b/sys/dev/pci/drm/radeon/radeon_cursor.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_cursor.c,v 1.5 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2007-8 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -28,9 +27,6 @@ #include #include "radeon.h" -#define CURSOR_WIDTH 64 -#define CURSOR_HEIGHT 64 - static void radeon_lock_cursor(struct drm_crtc *crtc, bool lock) { struct radeon_device *rdev = crtc->dev->dev_private; @@ -94,16 +90,38 @@ static void radeon_show_cursor(struct drm_crtc *crtc) struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct radeon_device *rdev = crtc->dev->dev_private; + if (radeon_crtc->cursor_out_of_bounds) + return; + if (ASIC_IS_DCE4(rdev)) { + WREG32(EVERGREEN_CUR_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, + upper_32_bits(radeon_crtc->cursor_addr)); + WREG32(EVERGREEN_CUR_SURFACE_ADDRESS + radeon_crtc->crtc_offset, + lower_32_bits(radeon_crtc->cursor_addr)); WREG32(RADEON_MM_INDEX, EVERGREEN_CUR_CONTROL + radeon_crtc->crtc_offset); WREG32(RADEON_MM_DATA, EVERGREEN_CURSOR_EN | EVERGREEN_CURSOR_MODE(EVERGREEN_CURSOR_24_8_PRE_MULT) | EVERGREEN_CURSOR_URGENT_CONTROL(EVERGREEN_CURSOR_URGENT_1_2)); } else if (ASIC_IS_AVIVO(rdev)) { + if (rdev->family >= CHIP_RV770) { + if (radeon_crtc->crtc_id) + WREG32(R700_D2CUR_SURFACE_ADDRESS_HIGH, + upper_32_bits(radeon_crtc->cursor_addr)); + else + WREG32(R700_D1CUR_SURFACE_ADDRESS_HIGH, + upper_32_bits(radeon_crtc->cursor_addr)); + } + + WREG32(AVIVO_D1CUR_SURFACE_ADDRESS + radeon_crtc->crtc_offset, + lower_32_bits(radeon_crtc->cursor_addr)); WREG32(RADEON_MM_INDEX, AVIVO_D1CUR_CONTROL + radeon_crtc->crtc_offset); WREG32(RADEON_MM_DATA, AVIVO_D1CURSOR_EN | (AVIVO_D1CURSOR_MODE_24BPP << AVIVO_D1CURSOR_MODE_SHIFT)); } else { + /* offset is from DISP(2)_BASE_ADDRESS */ + WREG32(RADEON_CUR_OFFSET + radeon_crtc->crtc_offset, + radeon_crtc->cursor_addr - radeon_crtc->legacy_display_base_addr); + switch (radeon_crtc->crtc_id) { case 0: WREG32(RADEON_MM_INDEX, RADEON_CRTC_GEN_CNTL); @@ -121,44 +139,150 @@ static void radeon_show_cursor(struct drm_crtc *crtc) } } -static void radeon_set_cursor(struct drm_crtc *crtc, struct drm_gem_object *obj, - uint64_t gpu_addr) +static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct radeon_device *rdev = crtc->dev->dev_private; + int xorigin = 0, yorigin = 0; + int w = radeon_crtc->cursor_width; + + radeon_crtc->cursor_x = x; + radeon_crtc->cursor_y = y; + + if (ASIC_IS_AVIVO(rdev)) { + /* avivo cursor are offset into the total surface */ + x += crtc->x; + y += crtc->y; + } + + if (x < 0) + xorigin = min(-x, radeon_crtc->max_cursor_width - 1); + if (y < 0) + yorigin = min(-y, radeon_crtc->max_cursor_height - 1); + + if (!ASIC_IS_AVIVO(rdev)) { + x += crtc->x; + y += crtc->y; + } + DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y); + + /* fixed on DCE6 and newer */ + if (ASIC_IS_AVIVO(rdev) && !ASIC_IS_DCE6(rdev)) { + int i = 0; + struct drm_crtc *crtc_p; + + /* + * avivo cursor image can't end on 128 pixel boundary or + * go past the end of the frame if both crtcs are enabled + * + * NOTE: It is safe to access crtc->enabled of other crtcs + * without holding either the mode_config lock or the other + * crtc's lock as long as write access to this flag _always_ + * grabs all locks. + */ + list_for_each_entry(crtc_p, &crtc->dev->mode_config.crtc_list, head) { + if (crtc_p->enabled) + i++; + } + if (i > 1) { + int cursor_end, frame_end; + + cursor_end = x + w; + frame_end = crtc->x + crtc->mode.crtc_hdisplay; + if (cursor_end >= frame_end) { + w = w - (cursor_end - frame_end); + if (!(frame_end & 0x7f)) + w--; + } else if (cursor_end <= 0) { + goto out_of_bounds; + } else if (!(cursor_end & 0x7f)) { + w--; + } + if (w <= 0) { + goto out_of_bounds; + } + } + } + + if (x <= (crtc->x - w) || y <= (crtc->y - radeon_crtc->cursor_height) || + x >= (crtc->x + crtc->mode.hdisplay) || + y >= (crtc->y + crtc->mode.vdisplay)) + goto out_of_bounds; + + x += xorigin; + y += yorigin; if (ASIC_IS_DCE4(rdev)) { - WREG32(EVERGREEN_CUR_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, - upper_32_bits(gpu_addr)); - WREG32(EVERGREEN_CUR_SURFACE_ADDRESS + radeon_crtc->crtc_offset, - gpu_addr & 0xffffffff); + WREG32(EVERGREEN_CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); + WREG32(EVERGREEN_CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); + WREG32(EVERGREEN_CUR_SIZE + radeon_crtc->crtc_offset, + ((w - 1) << 16) | (radeon_crtc->cursor_height - 1)); } else if (ASIC_IS_AVIVO(rdev)) { - if (rdev->family >= CHIP_RV770) { - if (radeon_crtc->crtc_id) - WREG32(R700_D2CUR_SURFACE_ADDRESS_HIGH, upper_32_bits(gpu_addr)); - else - WREG32(R700_D1CUR_SURFACE_ADDRESS_HIGH, upper_32_bits(gpu_addr)); - } - WREG32(AVIVO_D1CUR_SURFACE_ADDRESS + radeon_crtc->crtc_offset, - gpu_addr & 0xffffffff); + WREG32(AVIVO_D1CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); + WREG32(AVIVO_D1CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); + WREG32(AVIVO_D1CUR_SIZE + radeon_crtc->crtc_offset, + ((w - 1) << 16) | (radeon_crtc->cursor_height - 1)); } else { - radeon_crtc->legacy_cursor_offset = gpu_addr - radeon_crtc->legacy_display_base_addr; + x -= crtc->x; + y -= crtc->y; + + if (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN) + y *= 2; + + WREG32(RADEON_CUR_HORZ_VERT_OFF + radeon_crtc->crtc_offset, + (RADEON_CUR_LOCK + | (xorigin << 16) + | yorigin)); + WREG32(RADEON_CUR_HORZ_VERT_POSN + radeon_crtc->crtc_offset, + (RADEON_CUR_LOCK + | (x << 16) + | y)); /* offset is from DISP(2)_BASE_ADDRESS */ - WREG32(RADEON_CUR_OFFSET + radeon_crtc->crtc_offset, radeon_crtc->legacy_cursor_offset); + WREG32(RADEON_CUR_OFFSET + radeon_crtc->crtc_offset, + radeon_crtc->cursor_addr - radeon_crtc->legacy_display_base_addr + + yorigin * 256); + } + + if (radeon_crtc->cursor_out_of_bounds) { + radeon_crtc->cursor_out_of_bounds = false; + if (radeon_crtc->cursor_bo) + radeon_show_cursor(crtc); } + + return 0; + + out_of_bounds: + if (!radeon_crtc->cursor_out_of_bounds) { + radeon_hide_cursor(crtc); + radeon_crtc->cursor_out_of_bounds = true; + } + return 0; } -int radeon_crtc_cursor_set(struct drm_crtc *crtc, - struct drm_file *file_priv, - uint32_t handle, - uint32_t width, - uint32_t height) +int radeon_crtc_cursor_move(struct drm_crtc *crtc, + int x, int y) +{ + int ret; + + radeon_lock_cursor(crtc, true); + ret = radeon_cursor_move_locked(crtc, x, y); + radeon_lock_cursor(crtc, false); + + return ret; +} + +int radeon_crtc_cursor_set2(struct drm_crtc *crtc, + struct drm_file *file_priv, + uint32_t handle, + uint32_t width, + uint32_t height, + int32_t hot_x, + int32_t hot_y) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct radeon_device *rdev = crtc->dev->dev_private; struct drm_gem_object *obj; struct radeon_bo *robj; - uint64_t gpu_addr; int ret; if (!handle) { @@ -168,7 +292,8 @@ int radeon_crtc_cursor_set(struct drm_crtc *crtc, goto unpin; } - if ((width > CURSOR_WIDTH) || (height > CURSOR_HEIGHT)) { + if ((width > radeon_crtc->max_cursor_width) || + (height > radeon_crtc->max_cursor_height)) { DRM_ERROR("bad cursor width or height %d x %d\n", width, height); return -EINVAL; } @@ -181,27 +306,47 @@ int radeon_crtc_cursor_set(struct drm_crtc *crtc, robj = gem_to_radeon_bo(obj); ret = radeon_bo_reserve(robj, false); - if (unlikely(ret != 0)) - goto fail; + if (ret != 0) { + drm_gem_object_unreference_unlocked(obj); + return ret; + } /* Only 27 bit offset for legacy cursor */ ret = radeon_bo_pin_restricted(robj, RADEON_GEM_DOMAIN_VRAM, ASIC_IS_AVIVO(rdev) ? 0 : 1 << 27, - &gpu_addr); + &radeon_crtc->cursor_addr); radeon_bo_unreserve(robj); - if (ret) - goto fail; - - radeon_crtc->cursor_width = width; - radeon_crtc->cursor_height = height; + if (ret) { + DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret); + drm_gem_object_unreference_unlocked(obj); + return ret; + } radeon_lock_cursor(crtc, true); - radeon_set_cursor(crtc, obj, gpu_addr); + + if (width != radeon_crtc->cursor_width || + height != radeon_crtc->cursor_height || + hot_x != radeon_crtc->cursor_hot_x || + hot_y != radeon_crtc->cursor_hot_y) { + int x, y; + + x = radeon_crtc->cursor_x + radeon_crtc->cursor_hot_x - hot_x; + y = radeon_crtc->cursor_y + radeon_crtc->cursor_hot_y - hot_y; + + radeon_crtc->cursor_width = width; + radeon_crtc->cursor_height = height; + radeon_crtc->cursor_hot_x = hot_x; + radeon_crtc->cursor_hot_y = hot_y; + + radeon_cursor_move_locked(crtc, x, y); + } + radeon_show_cursor(crtc); + radeon_lock_cursor(crtc, false); unpin: if (radeon_crtc->cursor_bo) { - robj = gem_to_radeon_bo(radeon_crtc->cursor_bo); + struct radeon_bo *robj = gem_to_radeon_bo(radeon_crtc->cursor_bo); ret = radeon_bo_reserve(robj, false); if (likely(ret == 0)) { radeon_bo_unpin(robj); @@ -212,100 +357,28 @@ unpin: radeon_crtc->cursor_bo = obj; return 0; -fail: - drm_gem_object_unreference_unlocked(obj); - - return ret; } -int radeon_crtc_cursor_move(struct drm_crtc *crtc, - int x, int y) +/** + * radeon_cursor_reset - Re-set the current cursor, if any. + * + * @crtc: drm crtc + * + * If the CRTC passed in currently has a cursor assigned, this function + * makes sure it's visible. + */ +void radeon_cursor_reset(struct drm_crtc *crtc) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); - struct radeon_device *rdev = crtc->dev->dev_private; - int xorigin = 0, yorigin = 0; - int w = radeon_crtc->cursor_width; - - if (ASIC_IS_AVIVO(rdev)) { - /* avivo cursor are offset into the total surface */ - x += crtc->x; - y += crtc->y; - } - DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y); - - if (x < 0) { - xorigin = min(-x, CURSOR_WIDTH - 1); - x = 0; - } - if (y < 0) { - yorigin = min(-y, CURSOR_HEIGHT - 1); - y = 0; - } - /* fixed on DCE6 and newer */ - if (ASIC_IS_AVIVO(rdev) && !ASIC_IS_DCE6(rdev)) { - int i = 0; - struct drm_crtc *crtc_p; - - /* avivo cursor image can't end on 128 pixel boundary or - * go past the end of the frame if both crtcs are enabled - */ - list_for_each_entry(crtc_p, &crtc->dev->mode_config.crtc_list, head) { - if (crtc_p->enabled) - i++; - } - if (i > 1) { - int cursor_end, frame_end; + if (radeon_crtc->cursor_bo) { + radeon_lock_cursor(crtc, true); - cursor_end = x - xorigin + w; - frame_end = crtc->x + crtc->mode.crtc_hdisplay; - if (cursor_end >= frame_end) { - w = w - (cursor_end - frame_end); - if (!(frame_end & 0x7f)) - w--; - } else { - if (!(cursor_end & 0x7f)) - w--; - } - if (w <= 0) { - w = 1; - cursor_end = x - xorigin + w; - if (!(cursor_end & 0x7f)) { - x--; - WARN_ON_ONCE(x < 0); - } - } - } - } + radeon_cursor_move_locked(crtc, radeon_crtc->cursor_x, + radeon_crtc->cursor_y); - radeon_lock_cursor(crtc, true); - if (ASIC_IS_DCE4(rdev)) { - WREG32(EVERGREEN_CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); - WREG32(EVERGREEN_CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); - WREG32(EVERGREEN_CUR_SIZE + radeon_crtc->crtc_offset, - ((w - 1) << 16) | (radeon_crtc->cursor_height - 1)); - } else if (ASIC_IS_AVIVO(rdev)) { - WREG32(AVIVO_D1CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); - WREG32(AVIVO_D1CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); - WREG32(AVIVO_D1CUR_SIZE + radeon_crtc->crtc_offset, - ((w - 1) << 16) | (radeon_crtc->cursor_height - 1)); - } else { - if (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN) - y *= 2; + radeon_show_cursor(crtc); - WREG32(RADEON_CUR_HORZ_VERT_OFF + radeon_crtc->crtc_offset, - (RADEON_CUR_LOCK - | (xorigin << 16) - | yorigin)); - WREG32(RADEON_CUR_HORZ_VERT_POSN + radeon_crtc->crtc_offset, - (RADEON_CUR_LOCK - | (x << 16) - | y)); - /* offset is from DISP(2)_BASE_ADDRESS */ - WREG32(RADEON_CUR_OFFSET + radeon_crtc->crtc_offset, (radeon_crtc->legacy_cursor_offset + - (yorigin * 256))); + radeon_lock_cursor(crtc, false); } - radeon_lock_cursor(crtc, false); - - return 0; } diff --git a/sys/dev/pci/drm/radeon/radeon_device.c b/sys/dev/pci/drm/radeon/radeon_device.c index b9d1e72d15f..ec631f66cd9 100644 --- a/sys/dev/pci/drm/radeon/radeon_device.c +++ b/sys/dev/pci/drm/radeon/radeon_device.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_device.c,v 1.16 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -89,9 +88,119 @@ static const char radeon_family_name[][16] = { "TAHITI", "PITCAIRN", "VERDE", + "OLAND", + "HAINAN", + "BONAIRE", + "KAVERI", + "KABINI", + "HAWAII", + "MULLINS", "LAST", }; +#define RADEON_PX_QUIRK_DISABLE_PX (1 << 0) +#define RADEON_PX_QUIRK_LONG_WAKEUP (1 << 1) + +struct radeon_px_quirk { + u32 chip_vendor; + u32 chip_device; + u32 subsys_vendor; + u32 subsys_device; + u32 px_quirk_flags; +}; + +static struct radeon_px_quirk radeon_px_quirk_list[] = { + /* Acer aspire 5560g (CPU: AMD A4-3305M; GPU: AMD Radeon HD 6480g + 7470m) + * https://bugzilla.kernel.org/show_bug.cgi?id=74551 + */ + { PCI_VENDOR_ID_ATI, 0x6760, 0x1025, 0x0672, RADEON_PX_QUIRK_DISABLE_PX }, + /* Asus K73TA laptop with AMD A6-3400M APU and Radeon 6550 GPU + * https://bugzilla.kernel.org/show_bug.cgi?id=51381 + */ + { PCI_VENDOR_ID_ATI, 0x6741, 0x1043, 0x108c, RADEON_PX_QUIRK_DISABLE_PX }, + /* Asus K53TK laptop with AMD A6-3420M APU and Radeon 7670m GPU + * https://bugzilla.kernel.org/show_bug.cgi?id=51381 + */ + { PCI_VENDOR_ID_ATI, 0x6840, 0x1043, 0x2122, RADEON_PX_QUIRK_DISABLE_PX }, + /* Asus K53TK laptop with AMD A6-3420M APU and Radeon 7670m GPU + * https://bugs.freedesktop.org/show_bug.cgi?id=101491 + */ + { PCI_VENDOR_ID_ATI, 0x6741, 0x1043, 0x2122, RADEON_PX_QUIRK_DISABLE_PX }, + /* macbook pro 8.2 */ + { PCI_VENDOR_ID_ATI, 0x6741, PCI_VENDOR_ID_APPLE, 0x00e2, RADEON_PX_QUIRK_LONG_WAKEUP }, + { 0, 0, 0, 0, 0 }, +}; + +bool radeon_is_px(struct drm_device *dev) +{ + struct radeon_device *rdev = dev->dev_private; + + if (rdev->flags & RADEON_IS_PX) + return true; + return false; +} + +static void radeon_device_handle_px_quirks(struct radeon_device *rdev) +{ + struct radeon_px_quirk *p = radeon_px_quirk_list; + + /* Apply PX quirks */ + while (p && p->chip_device != 0) { + if (rdev->pdev->vendor == p->chip_vendor && + rdev->pdev->device == p->chip_device && + rdev->pdev->subsystem_vendor == p->subsys_vendor && + rdev->pdev->subsystem_device == p->subsys_device) { + rdev->px_quirk_flags = p->px_quirk_flags; + break; + } + ++p; + } + + if (rdev->px_quirk_flags & RADEON_PX_QUIRK_DISABLE_PX) + rdev->flags &= ~RADEON_IS_PX; +} + +/** + * radeon_program_register_sequence - program an array of registers. + * + * @rdev: radeon_device pointer + * @registers: pointer to the register array + * @array_size: size of the register array + * + * Programs an array or registers with and and or masks. + * This is a helper for setting golden registers. + */ +void radeon_program_register_sequence(struct radeon_device *rdev, + const u32 *registers, + const u32 array_size) +{ + u32 tmp, reg, and_mask, or_mask; + int i; + + if (array_size % 3) + return; + + for (i = 0; i < array_size; i +=3) { + reg = registers[i + 0]; + and_mask = registers[i + 1]; + or_mask = registers[i + 2]; + + if (and_mask == 0xffffffff) { + tmp = or_mask; + } else { + tmp = RREG32(reg); + tmp &= ~and_mask; + tmp |= or_mask; + } + WREG32(reg, tmp); + } +} + +void radeon_pci_config_reset(struct radeon_device *rdev) +{ + pci_write_config_dword(rdev->pdev, 0x7c, RADEON_ASIC_RESET_DATA); +} + /** * radeon_surface_init - Clear GPU surface registers. * @@ -186,6 +295,129 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg) } } +/* + * GPU doorbell aperture helpers function. + */ +/** + * radeon_doorbell_init - Init doorbell driver information. + * + * @rdev: radeon_device pointer + * + * Init doorbell driver information (CIK) + * Returns 0 on success, error on failure. + */ +static int radeon_doorbell_init(struct radeon_device *rdev) +{ + /* doorbell bar mapping */ +#ifdef __linux__ + rdev->doorbell.base = pci_resource_start(rdev->pdev, 2); + rdev->doorbell.size = pci_resource_len(rdev->pdev, 2); +#endif + + rdev->doorbell.num_doorbells = min_t(u32, rdev->doorbell.size / sizeof(u32), RADEON_MAX_DOORBELLS); + if (rdev->doorbell.num_doorbells == 0) + return -EINVAL; + +#ifdef __linux__ + rdev->doorbell.ptr = ioremap(rdev->doorbell.base, rdev->doorbell.num_doorbells * sizeof(u32)); + if (rdev->doorbell.ptr == NULL) { + return -ENOMEM; + } +#endif + DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)rdev->doorbell.base); + DRM_INFO("doorbell mmio size: %u\n", (unsigned)rdev->doorbell.size); + + memset(&rdev->doorbell.used, 0, sizeof(rdev->doorbell.used)); + + return 0; +} + +/** + * radeon_doorbell_fini - Tear down doorbell driver information. + * + * @rdev: radeon_device pointer + * + * Tear down doorbell driver information (CIK) + */ +static void radeon_doorbell_fini(struct radeon_device *rdev) +{ +#ifdef __linux__ + iounmap(rdev->doorbell.ptr); + rdev->doorbell.ptr = NULL; +#else + if (rdev->doorbell.size > 0) + bus_space_unmap(rdev->memt, rdev->doorbell.bsh, + rdev->doorbell.size); + rdev->doorbell.size = 0; +#endif +} + +/** + * radeon_doorbell_get - Allocate a doorbell entry + * + * @rdev: radeon_device pointer + * @doorbell: doorbell index + * + * Allocate a doorbell for use by the driver (all asics). + * Returns 0 on success or -EINVAL on failure. + */ +int radeon_doorbell_get(struct radeon_device *rdev, u32 *doorbell) +{ + unsigned long offset = find_first_zero_bit(rdev->doorbell.used, rdev->doorbell.num_doorbells); + if (offset < rdev->doorbell.num_doorbells) { + __set_bit(offset, rdev->doorbell.used); + *doorbell = offset; + return 0; + } else { + return -EINVAL; + } +} + +/** + * radeon_doorbell_free - Free a doorbell entry + * + * @rdev: radeon_device pointer + * @doorbell: doorbell index + * + * Free a doorbell allocated for use by the driver (all asics) + */ +void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell) +{ + if (doorbell < rdev->doorbell.num_doorbells) + __clear_bit(doorbell, rdev->doorbell.used); +} + +/** + * radeon_doorbell_get_kfd_info - Report doorbell configuration required to + * setup KFD + * + * @rdev: radeon_device pointer + * @aperture_base: output returning doorbell aperture base physical address + * @aperture_size: output returning doorbell aperture size in bytes + * @start_offset: output returning # of doorbell bytes reserved for radeon. + * + * Radeon and the KFD share the doorbell aperture. Radeon sets it up, + * takes doorbells required for its own rings and reports the setup to KFD. + * Radeon reserved doorbells are at the start of the doorbell aperture. + */ +void radeon_doorbell_get_kfd_info(struct radeon_device *rdev, + phys_addr_t *aperture_base, + size_t *aperture_size, + size_t *start_offset) +{ + /* The first num_doorbells are used by radeon. + * KFD takes whatever's left in the aperture. */ + if (rdev->doorbell.size > rdev->doorbell.num_doorbells * sizeof(u32)) { + *aperture_base = rdev->doorbell.base; + *aperture_size = rdev->doorbell.size; + *start_offset = rdev->doorbell.num_doorbells * sizeof(u32); + } else { + *aperture_base = 0; + *aperture_size = 0; + *start_offset = 0; + } +} + /* * radeon_wb_*() * Writeback is the the method by which the the GPU updates special pages @@ -202,16 +434,6 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg) */ void radeon_wb_disable(struct radeon_device *rdev) { - int r; - - if (rdev->wb.wb_obj) { - r = radeon_bo_reserve(rdev->wb.wb_obj, false); - if (unlikely(r != 0)) - return; - radeon_bo_kunmap(rdev->wb.wb_obj); - radeon_bo_unpin(rdev->wb.wb_obj); - radeon_bo_unreserve(rdev->wb.wb_obj); - } rdev->wb.enabled = false; } @@ -227,6 +449,11 @@ void radeon_wb_fini(struct radeon_device *rdev) { radeon_wb_disable(rdev); if (rdev->wb.wb_obj) { + if (!radeon_bo_reserve(rdev->wb.wb_obj, false)) { + radeon_bo_kunmap(rdev->wb.wb_obj); + radeon_bo_unpin(rdev->wb.wb_obj); + radeon_bo_unreserve(rdev->wb.wb_obj); + } radeon_bo_unref(&rdev->wb.wb_obj); rdev->wb.wb = NULL; rdev->wb.wb_obj = NULL; @@ -248,31 +475,32 @@ int radeon_wb_init(struct radeon_device *rdev) if (rdev->wb.wb_obj == NULL) { r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_GTT, NULL, &rdev->wb.wb_obj); + RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL, + &rdev->wb.wb_obj); if (r) { dev_warn(rdev->dev, "(%d) create WB bo failed\n", r); return r; } - } - r = radeon_bo_reserve(rdev->wb.wb_obj, false); - if (unlikely(r != 0)) { - radeon_wb_fini(rdev); - return r; - } - r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT, - &rdev->wb.gpu_addr); - if (r) { + r = radeon_bo_reserve(rdev->wb.wb_obj, false); + if (unlikely(r != 0)) { + radeon_wb_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT, + &rdev->wb.gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->wb.wb_obj); + dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r); + radeon_wb_fini(rdev); + return r; + } + r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb); radeon_bo_unreserve(rdev->wb.wb_obj); - dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r); - radeon_wb_fini(rdev); - return r; - } - r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb); - radeon_bo_unreserve(rdev->wb.wb_obj); - if (r) { - dev_warn(rdev->dev, "(%d) map WB bo failed\n", r); - radeon_wb_fini(rdev); - return r; + if (r) { + dev_warn(rdev->dev, "(%d) map WB bo failed\n", r); + radeon_wb_fini(rdev); + return r; + } } /* clear wb memory */ @@ -303,9 +531,7 @@ int radeon_wb_init(struct radeon_device *rdev) rdev->wb.use_event = true; } -#ifdef DRMDEBUG dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis"); -#endif return 0; } @@ -356,7 +582,7 @@ void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 uint64_t limit = (uint64_t)radeon_vram_limit << 20; mc->vram_start = base; - if (mc->mc_vram_size > (0xFFFFFFFF - base + 1)) { + if (mc->mc_vram_size > (rdev->mc.mc_mask - base + 1)) { dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n"); mc->real_vram_size = mc->aper_size; mc->mc_vram_size = mc->aper_size; @@ -391,7 +617,7 @@ void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc) { u64 size_af, size_bf; - size_af = ((0xFFFFFFFF - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align; + size_af = ((rdev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align; size_bf = mc->vram_start & ~mc->gtt_base_align; if (size_bf > size_af) { if (mc->gtt_size > size_bf) { @@ -414,6 +640,23 @@ void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc) /* * GPU helpers function. */ + +/** + * radeon_device_is_virtual - check if we are running is a virtual environment + * + * Check if the asic has been passed through to a VM (all asics). + * Used at driver startup. + * Returns true if virtual or false if not. + */ +static bool radeon_device_is_virtual(void) +{ +#ifdef CONFIG_X86 + return boot_cpu_has(X86_FEATURE_HYPERVISOR); +#else + return false; +#endif +} + /** * radeon_card_posted - check if the hw has already been initialized * @@ -427,12 +670,22 @@ bool radeon_card_posted(struct radeon_device *rdev) { uint32_t reg; + /* for pass through, always force asic_init for CI */ + if (rdev->family >= CHIP_BONAIRE && + radeon_device_is_virtual()) + return false; + + /* required for EFI mode on macbook2,1 which uses an r5xx asic */ #ifdef notyet if (efi_enabled(EFI_BOOT) && - rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) + (rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) && + (rdev->family < CHIP_R600)) return false; #endif + if (ASIC_IS_NODCE(rdev)) + goto check_memsize; + /* first check CRTCs */ if (ASIC_IS_DCE4(rdev)) { reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) | @@ -461,6 +714,7 @@ bool radeon_card_posted(struct radeon_device *rdev) } } +check_memsize: /* then check MEM_SIZE, in case the crtcs are off */ if (rdev->family >= CHIP_R600) reg = RREG32(R600_CONFIG_MEMSIZE); @@ -548,6 +802,8 @@ int radeon_dummy_page_init(struct radeon_device *rdev) if (!rdev->dummy_page.dmah) return -ENOMEM; rdev->dummy_page.addr = (bus_addr_t)rdev->dummy_page.dmah->map->dm_segs[0].ds_addr; + rdev->dummy_page.entry = radeon_gart_get_page_entry(rdev->dummy_page.addr, + RADEON_GART_PAGE_DUMMY); return 0; } @@ -751,7 +1007,13 @@ int radeon_atombios_init(struct radeon_device *rdev) atom_card_info->pll_write = cail_pll_write; rdev->mode_info.atom_context = atom_parse(atom_card_info, rdev->bios); + if (!rdev->mode_info.atom_context) { + radeon_atombios_fini(rdev); + return -ENOMEM; + } + rw_init(&rdev->mode_info.atom_context->mutex, "atomcon"); + rw_init(&rdev->mode_info.atom_context->scratch_mutex, "atomscr"); radeon_atom_initialize_bios_scratch_regs(rdev->ddev); atom_allocate_fb_scratch(rdev->mode_info.atom_context); return 0; @@ -770,9 +1032,11 @@ void radeon_atombios_fini(struct radeon_device *rdev) { if (rdev->mode_info.atom_context) { kfree(rdev->mode_info.atom_context->scratch); - kfree(rdev->mode_info.atom_context); } + kfree(rdev->mode_info.atom_context); + rdev->mode_info.atom_context = NULL; kfree(rdev->mode_info.atom_card_info); + rdev->mode_info.atom_card_info = NULL; } /* COMBIOS */ @@ -845,6 +1109,22 @@ static bool radeon_check_pot_argument(int arg) return (arg & (arg - 1)) == 0; } +/** + * Determine a sensible default GART size according to ASIC family. + * + * @family ASIC family name + */ +static int radeon_gart_size_auto(enum radeon_family family) +{ + /* default to a larger gart size on newer asics */ + if (family >= CHIP_TAHITI) + return 2048; + else if (family >= CHIP_RV770) + return 1024; + else + return 512; +} + /** * radeon_check_arguments - validate module params * @@ -862,16 +1142,18 @@ static void radeon_check_arguments(struct radeon_device *rdev) radeon_vram_limit = 0; } + if (radeon_gart_size == -1) { + radeon_gart_size = radeon_gart_size_auto(rdev->family); + } /* gtt size must be power of two and greater or equal to 32M */ if (radeon_gart_size < 32) { - dev_warn(rdev->dev, "gart size (%d) too small forcing to 512M\n", + dev_warn(rdev->dev, "gart size (%d) too small\n", radeon_gart_size); - radeon_gart_size = 512; - + radeon_gart_size = radeon_gart_size_auto(rdev->family); } else if (!radeon_check_pot_argument(radeon_gart_size)) { dev_warn(rdev->dev, "gart size (%d) must be a power of 2\n", radeon_gart_size); - radeon_gart_size = 512; + radeon_gart_size = radeon_gart_size_auto(rdev->family); } rdev->mc.gtt_size = (uint64_t)radeon_gart_size << 20; @@ -890,41 +1172,75 @@ static void radeon_check_arguments(struct radeon_device *rdev) radeon_agpmode = 0; break; } -} -#ifdef notyet -/** - * radeon_switcheroo_quirk_long_wakeup - return true if longer d3 delay is - * needed for waking up. - * - * @pdev: pci dev pointer - */ -static bool radeon_switcheroo_quirk_long_wakeup(struct pci_dev *pdev) -{ + if (!radeon_check_pot_argument(radeon_vm_size)) { + dev_warn(rdev->dev, "VM size (%d) must be a power of 2\n", + radeon_vm_size); + radeon_vm_size = 4; + } - /* 6600m in a macbook pro */ - if (pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE && - pdev->subsystem_device == 0x00e2) { - printk(KERN_INFO "radeon: quirking longer d3 wakeup delay\n"); - return true; + if (radeon_vm_size < 1) { + dev_warn(rdev->dev, "VM size (%d) to small, min is 1GB\n", + radeon_vm_size); + radeon_vm_size = 4; } - return false; + /* + * Max GPUVM size for Cayman, SI and CI are 40 bits. + */ + if (radeon_vm_size > 1024) { + dev_warn(rdev->dev, "VM size (%d) too large, max is 1TB\n", + radeon_vm_size); + radeon_vm_size = 4; + } + + /* defines number of bits in page table versus page directory, + * a page is 4KB so we have 12 bits offset, minimum 9 bits in the + * page table and the remaining bits are in the page directory */ + if (radeon_vm_block_size == -1) { + + /* Total bits covered by PD + PTs */ + unsigned bits = ilog2(radeon_vm_size) + 18; + + /* Make sure the PD is 4K in size up to 8GB address space. + Above that split equal between PD and PTs */ + if (radeon_vm_size <= 8) + radeon_vm_block_size = bits - 9; + else + radeon_vm_block_size = (bits + 3) / 2; + + } else if (radeon_vm_block_size < 9) { + dev_warn(rdev->dev, "VM page table size (%d) too small\n", + radeon_vm_block_size); + radeon_vm_block_size = 9; + } + + if (radeon_vm_block_size > 24 || + (radeon_vm_size * 1024) < (1ull << radeon_vm_block_size)) { + dev_warn(rdev->dev, "VM page table size (%d) too large\n", + radeon_vm_block_size); + radeon_vm_block_size = 9; + } } /** * radeon_switcheroo_set_state - set switcheroo state * * @pdev: pci dev pointer - * @state: vga switcheroo state + * @state: vga_switcheroo state * * Callback for the switcheroo driver. Suspends or resumes the * the asics before or after it is powered up using ACPI methods. */ +#ifdef notyet static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) { struct drm_device *dev = pci_get_drvdata(pdev); - pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; + struct radeon_device *rdev = dev->dev_private; + + if (radeon_is_px(dev) && state == VGA_SWITCHEROO_OFF) + return; + if (state == VGA_SWITCHEROO_ON) { unsigned d3_delay = dev->pdev->d3_delay; @@ -932,10 +1248,10 @@ static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switchero /* don't suspend or resume card normally */ dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - if (d3_delay < 20 && radeon_switcheroo_quirk_long_wakeup(pdev)) + if (d3_delay < 20 && (rdev->px_quirk_flags & RADEON_PX_QUIRK_LONG_WAKEUP)) dev->pdev->d3_delay = 20; - radeon_resume_kms(dev); + radeon_resume_kms(dev, true, true); dev->pdev->d3_delay = d3_delay; @@ -945,7 +1261,7 @@ static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switchero printk(KERN_INFO "radeon: switched off\n"); drm_kms_helper_poll_disable(dev); dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - radeon_suspend_kms(dev, pmm); + radeon_suspend_kms(dev, true, true); dev->switch_power_state = DRM_SWITCH_POWER_OFF; } } @@ -962,12 +1278,13 @@ static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switchero static bool radeon_switcheroo_can_switch(struct pci_dev *pdev) { struct drm_device *dev = pci_get_drvdata(pdev); - bool can_switch; - spin_lock(&dev->count_lock); - can_switch = (dev->open_count == 0); - spin_unlock(&dev->count_lock); - return can_switch; + /* + * FIXME: open_count is protected by drm_global_mutex but that would lead to + * locking inversion with the driver load path. And the access here is + * completely racy anyway. So don't bother with locking for now. + */ + return dev->open_count == 0; } static const struct vga_switcheroo_client_ops radeon_switcheroo_ops = { @@ -990,26 +1307,30 @@ static const struct vga_switcheroo_client_ops radeon_switcheroo_ops = { * Called at driver startup. */ int radeon_device_init(struct radeon_device *rdev, - struct drm_device *ddev) + struct drm_device *ddev, + struct pci_dev *pdev, + uint32_t flags) { -#ifdef DRMDEBUG - struct pci_dev *pdev = ddev->pdev; -#endif int r, i; int dma_bits; + bool runtime = false; rdev->shutdown = false; - rdev->family = rdev->flags & RADEON_FAMILY_MASK; + rdev->ddev = ddev; + rdev->pdev = pdev; + rdev->flags = flags; + rdev->family = flags & RADEON_FAMILY_MASK; rdev->is_atom_bios = false; rdev->usec_timeout = RADEON_MAX_USEC_TIMEOUT; - rdev->mc.gtt_size = radeon_gart_size * 1024 * 1024; + rdev->mc.gtt_size = 512 * 1024 * 1024; rdev->accel_working = false; /* set up ring ids */ for (i = 0; i < RADEON_NUM_RINGS; i++) { rdev->ring[i].idx = i; } + rdev->fence_context = fence_context_alloc(RADEON_NUM_RINGS); - DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X).\n", + printf("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X).\n", radeon_family_name[rdev->family], pdev->vendor, pdev->device, pdev->subsystem_vendor, pdev->subsystem_device); @@ -1021,26 +1342,27 @@ int radeon_device_init(struct radeon_device *rdev, rw_init(&rdev->gem.mutex, "gem"); rw_init(&rdev->pm.mutex, "pm"); rw_init(&rdev->gpu_clock_mutex, "gpuclk"); + rw_init(&rdev->srbm_mutex, "srbm"); + rw_init(&rdev->grbm_idx_mutex, "grbm"); rw_init(&rdev->pm.mclk_lock, "mclk"); rw_init(&rdev->exclusive_lock, "rdnexc"); init_waitqueue_head(&rdev->irq.vblank_queue); + rw_init(&rdev->mn_lock, "mnlk"); + hash_init(rdev->mn_hash); r = radeon_gem_init(rdev); if (r) return r; - /* initialize vm here */ - rw_init(&rdev->vm_manager.lock, "vmmgr"); + + radeon_check_arguments(rdev); /* Adjust VM size here. - * Currently set to 4GB ((1 << 20) 4k pages). - * Max GPUVM size for cayman and SI is 40 bits. + * Max GPUVM size for cayman+ is 40 bits. */ - rdev->vm_manager.max_pfn = 1 << 20; - INIT_LIST_HEAD(&rdev->vm_manager.lru_vm); + rdev->vm_manager.max_pfn = radeon_vm_size << 18; /* Set asic functions */ r = radeon_asic_init(rdev); if (r) return r; - radeon_check_arguments(rdev); /* all of the newer IGP chips have an internal gart * However some rs4xx report as AGP, so remove that here. @@ -1054,6 +1376,17 @@ int radeon_device_init(struct radeon_device *rdev, radeon_agp_disable(rdev); } + /* Set the internal MC address mask + * This is the max address of the GPU's + * internal address space. + */ + if (rdev->family >= CHIP_CAYMAN) + rdev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */ + else if (rdev->family >= CHIP_CEDAR) + rdev->mc.mc_mask = 0xfffffffffULL; /* 36 bit MC */ + else + rdev->mc.mc_mask = 0xffffffffULL; /* 32 bit MC */ + /* set DMA mask + need_dma32 flags. * PCIE - can handle 40-bits. * IGP - can handle 40-bits @@ -1080,22 +1413,44 @@ int radeon_device_init(struct radeon_device *rdev, pci_set_consistent_dma_mask(rdev->pdev, DMA_BIT_MASK(32)); printk(KERN_WARNING "radeon: No coherent DMA available.\n"); } +#endif /* Registers mapping */ /* TODO: block userspace mapping of io register */ -#endif mtx_init(&rdev->mmio_idx_lock, IPL_TTY); -#ifdef notyet - rdev->rmmio_base = pci_resource_start(rdev->pdev, 2); - rdev->rmmio_size = pci_resource_len(rdev->pdev, 2); + mtx_init(&rdev->smc_idx_lock, IPL_TTY); + mtx_init(&rdev->pll_idx_lock, IPL_TTY); + mtx_init(&rdev->mc_idx_lock, IPL_TTY); + mtx_init(&rdev->pcie_idx_lock, IPL_TTY); + mtx_init(&rdev->pciep_idx_lock, IPL_TTY); + mtx_init(&rdev->pif_idx_lock, IPL_TTY); + mtx_init(&rdev->cg_idx_lock, IPL_TTY); + mtx_init(&rdev->uvd_idx_lock, IPL_TTY); + mtx_init(&rdev->rcu_idx_lock, IPL_TTY); + mtx_init(&rdev->didt_idx_lock, IPL_TTY); + mtx_init(&rdev->end_idx_lock, IPL_TTY); +#ifdef __linux__ + if (rdev->family >= CHIP_BONAIRE) { + rdev->rmmio_base = pci_resource_start(rdev->pdev, 5); + rdev->rmmio_size = pci_resource_len(rdev->pdev, 5); + } else { + rdev->rmmio_base = pci_resource_start(rdev->pdev, 2); + rdev->rmmio_size = pci_resource_len(rdev->pdev, 2); + } rdev->rmmio = ioremap(rdev->rmmio_base, rdev->rmmio_size); if (rdev->rmmio == NULL) { return -ENOMEM; } +#endif DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)rdev->rmmio_base); DRM_INFO("register mmio size: %u\n", (unsigned)rdev->rmmio_size); + /* doorbell bar mapping */ + if (rdev->family >= CHIP_BONAIRE) + radeon_doorbell_init(rdev); + /* io port mapping */ +#ifdef linux for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { if (pci_resource_flags(rdev->pdev, i) & IORESOURCE_IO) { rdev->rio_mem_size = pci_resource_len(rdev->pdev, i); @@ -1105,21 +1460,39 @@ int radeon_device_init(struct radeon_device *rdev, } if (rdev->rio_mem == NULL) DRM_ERROR("Unable to find PCI I/O BAR\n"); +#endif + + if (rdev->flags & RADEON_IS_PX) + radeon_device_handle_px_quirks(rdev); /* if we have > 1 VGA cards, then disable the radeon VGA resources */ /* this will fail for cards that aren't VGA class devices, just * ignore it */ +#ifdef notyet vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode); - vga_switcheroo_register_client(rdev->pdev, &radeon_switcheroo_ops); +#endif + + if (rdev->flags & RADEON_IS_PX) + runtime = true; +#ifdef notyet + vga_switcheroo_register_client(rdev->pdev, &radeon_switcheroo_ops, runtime); + if (runtime) + vga_switcheroo_init_domain_pm_ops(rdev->dev, &rdev->vga_pm_domain); #endif r = radeon_init(rdev); if (r) - return r; + goto failed; - r = radeon_ib_ring_tests(rdev); - if (r) - DRM_ERROR("ib ring test failed (%d).\n", r); + r = radeon_gem_debugfs_init(rdev); + if (r) { + DRM_ERROR("registering gem debugfs failed (%d).\n", r); + } + + r = radeon_mst_debugfs_init(rdev); + if (r) { + DRM_ERROR("registering mst debugfs failed (%d).\n", r); + } if (rdev->flags & RADEON_IS_AGP && !rdev->accel_working) { /* Acceleration not working on AGP card try again @@ -1130,8 +1503,28 @@ int radeon_device_init(struct radeon_device *rdev, radeon_agp_disable(rdev); r = radeon_init(rdev); if (r) - return r; + goto failed; + } + + r = radeon_ib_ring_tests(rdev); + if (r) + DRM_ERROR("ib ring test failed (%d).\n", r); + + /* + * Turks/Thames GPU will freeze whole laptop if DPM is not restarted + * after the CP ring have chew one packet at least. Hence here we stop + * and restart DPM after the radeon_ib_ring_tests(). + */ + if (rdev->pm.dpm_enabled && + (rdev->pm.pm_method == PM_METHOD_DPM) && + (rdev->family == CHIP_TURKS) && + (rdev->flags & RADEON_IS_MOBILITY)) { + mutex_lock(&rdev->pm.mutex); + radeon_dpm_disable(rdev); + radeon_dpm_enable(rdev); + mutex_unlock(&rdev->pm.mutex); } + if ((radeon_testing & 1)) { if (rdev->accel_working) radeon_test_moves(rdev); @@ -1151,11 +1544,16 @@ int radeon_device_init(struct radeon_device *rdev, DRM_INFO("radeon: acceleration disabled, skipping benchmarks\n"); } return 0; + +failed: +#ifdef notyet + if (runtime) + vga_switcheroo_fini_domain_pm_ops(rdev->dev); +#endif + return r; } -#ifdef __linux__ static void radeon_debugfs_remove_files(struct radeon_device *rdev); -#endif /** * radeon_device_fini - tear down the driver @@ -1172,18 +1570,31 @@ void radeon_device_fini(struct radeon_device *rdev) /* evict vram memory */ radeon_bo_evict_vram(rdev); radeon_fini(rdev); -#ifdef notyet vga_switcheroo_unregister_client(rdev->pdev); + if (rdev->flags & RADEON_IS_PX) + vga_switcheroo_fini_domain_pm_ops(rdev->dev); vga_client_register(rdev->pdev, NULL, NULL, NULL); +#ifdef __linux__ if (rdev->rio_mem) pci_iounmap(rdev->pdev, rdev->rio_mem); rdev->rio_mem = NULL; iounmap(rdev->rmmio); rdev->rmmio = NULL; - radeon_debugfs_remove_files(rdev); +#else + if (rdev->rio_mem_size > 0) + bus_space_unmap(rdev->iot, rdev->rio_mem, rdev->rio_mem_size); + rdev->rio_mem_size = 0; + + if (rdev->rmmio_size > 0) + bus_space_unmap(rdev->memt, rdev->rmmio_bsh, rdev->rmmio_size); + rdev->rmmio_size = 0; #endif + if (rdev->family >= CHIP_BONAIRE) + radeon_doorbell_fini(rdev); + radeon_debugfs_remove_files(rdev); } + /* * Suspend & resume. */ @@ -1197,22 +1608,17 @@ void radeon_device_fini(struct radeon_device *rdev) * Returns 0 for success or an error on failure. * Called at driver suspend. */ -int radeon_suspend_kms(struct drm_device *dev) +int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon) { struct radeon_device *rdev; struct drm_crtc *crtc; struct drm_connector *connector; int i, r; - bool force_completion = false; if (dev == NULL || dev->dev_private == NULL) { return -ENODEV; } -#ifdef notyet - if (state.event == PM_EVENT_PRETHAW) { - return 0; - } -#endif + rdev = dev->dev_private; if (rdev->shutdown) return 0; @@ -1224,16 +1630,28 @@ int radeon_suspend_kms(struct drm_device *dev) drm_kms_helper_poll_disable(dev); + drm_modeset_lock_all(dev); /* turn off display hw */ list_for_each_entry(connector, &dev->mode_config.connector_list, head) { drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF); } + drm_modeset_unlock_all(dev); - /* unpin the front buffers */ + /* unpin the front buffers and cursors */ list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct radeon_framebuffer *rfb = to_radeon_framebuffer(crtc->primary->fb); struct radeon_bo *robj; + if (radeon_crtc->cursor_bo) { + struct radeon_bo *robj = gem_to_radeon_bo(radeon_crtc->cursor_bo); + r = radeon_bo_reserve(robj, false); + if (r == 0) { + radeon_bo_unpin(robj); + radeon_bo_unreserve(robj); + } + } + if (rfb == NULL || rfb->obj == NULL) { continue; } @@ -1250,23 +1668,17 @@ int radeon_suspend_kms(struct drm_device *dev) /* evict vram memory */ radeon_bo_evict_vram(rdev); - mutex_lock(&rdev->ring_lock); /* wait for gpu to finish processing current batch */ for (i = 0; i < RADEON_NUM_RINGS; i++) { - r = radeon_fence_wait_empty_locked(rdev, i); + r = radeon_fence_wait_empty(rdev, i); if (r) { /* delay GPU reset to resume */ - force_completion = true; + radeon_fence_driver_force_completion(rdev, i); } } - if (force_completion) { - radeon_fence_driver_force_completion(rdev); - } - mutex_unlock(&rdev->ring_lock); radeon_save_bios_scratch_regs(rdev); - radeon_pm_suspend(rdev); radeon_suspend(rdev); radeon_hpd_fini(rdev); /* evict remaining vram memory */ @@ -1274,17 +1686,18 @@ int radeon_suspend_kms(struct drm_device *dev) radeon_agp_suspend(rdev); -#ifdef notyet pci_save_state(dev->pdev); - if (state.event == PM_EVENT_SUSPEND) { + if (suspend) { /* Shut down the device */ pci_disable_device(dev->pdev); pci_set_power_state(dev->pdev, PCI_D3hot); } -#endif - console_lock(); - radeon_fbdev_set_suspend(rdev, 1); - console_unlock(); + + if (fbcon) { + console_lock(); + radeon_fbdev_set_suspend(rdev, 1); + console_unlock(); + } return 0; } @@ -1297,10 +1710,11 @@ int radeon_suspend_kms(struct drm_device *dev) * Returns 0 for success or an error on failure. * Called at driver resume. */ -int radeon_resume_kms(struct drm_device *dev) +int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon) { struct drm_connector *connector; struct radeon_device *rdev = dev->dev_private; + struct drm_crtc *crtc; int r; #ifdef notyet @@ -1308,15 +1722,18 @@ int radeon_resume_kms(struct drm_device *dev) return 0; #endif - console_lock(); -#ifdef notyet - pci_set_power_state(dev->pdev, PCI_D0); - pci_restore_state(dev->pdev); - if (pci_enable_device(dev->pdev)) { - console_unlock(); - return -1; + if (fbcon) { + console_lock(); + } + if (resume) { + pci_set_power_state(dev->pdev, PCI_D0); + pci_restore_state(dev->pdev); + if (pci_enable_device(dev->pdev)) { + if (fbcon) + console_unlock(); + return -1; + } } -#endif /* resume AGP if in use */ radeon_agp_resume(rdev); radeon_resume(rdev); @@ -1325,11 +1742,40 @@ int radeon_resume_kms(struct drm_device *dev) if (r) DRM_ERROR("ib ring test failed (%d).\n", r); - radeon_pm_resume(rdev); + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + /* do dpm late init */ + r = radeon_pm_late_init(rdev); + if (r) { + rdev->pm.dpm_enabled = false; + DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n"); + } + } else { + /* resume old pm late */ + radeon_pm_resume(rdev); + } + radeon_restore_bios_scratch_regs(rdev); - radeon_fbdev_set_suspend(rdev, 0); - console_unlock(); + /* pin cursors */ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); + + if (radeon_crtc->cursor_bo) { + struct radeon_bo *robj = gem_to_radeon_bo(radeon_crtc->cursor_bo); + r = radeon_bo_reserve(robj, false); + if (r == 0) { + /* Only 27 bit offset for legacy cursor */ + r = radeon_bo_pin_restricted(robj, + RADEON_GEM_DOMAIN_VRAM, + ASIC_IS_AVIVO(rdev) ? + 0 : 1 << 27, + &radeon_crtc->cursor_addr); + if (r != 0) + DRM_ERROR("Failed to pin cursor BO (%d)\n", r); + radeon_bo_unreserve(robj); + } + } + } /* init dig PHYs, disp eng pll */ if (rdev->is_atom_bios) { @@ -1346,13 +1792,27 @@ int radeon_resume_kms(struct drm_device *dev) /* reset hpd state */ radeon_hpd_init(rdev); /* blat the mode back in */ - drm_helper_resume_force_mode(dev); - /* turn on display hw */ - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { - drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); + if (fbcon) { + drm_helper_resume_force_mode(dev); + /* turn on display hw */ + drm_modeset_lock_all(dev); + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); + } + drm_modeset_unlock_all(dev); } drm_kms_helper_poll_enable(dev); + + /* set the power state here in case we are a PX system or headless */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) + radeon_pm_compute_clocks(rdev); + + if (fbcon) { + radeon_fbdev_set_suspend(rdev, 0); + console_unlock(); + } + return 0; } @@ -1375,10 +1835,19 @@ int radeon_gpu_reset(struct radeon_device *rdev) int resched; down_write(&rdev->exclusive_lock); + + if (!rdev->needs_reset) { + up_write(&rdev->exclusive_lock); + return 0; + } + + atomic_inc(&rdev->gpu_reset_counter); + radeon_save_bios_scratch_regs(rdev); /* block TTM */ resched = ttm_bo_lock_delayed_workqueue(&rdev->mman.bdev); radeon_suspend(rdev); + radeon_hpd_fini(rdev); for (i = 0; i < RADEON_NUM_RINGS; ++i) { ring_sizes[i] = radeon_ring_backup(rdev, &rdev->ring[i], @@ -1390,7 +1859,6 @@ int radeon_gpu_reset(struct radeon_device *rdev) } } -retry: r = radeon_asic_reset(rdev); if (!r) { dev_info(rdev->dev, "GPU reset succeeded, trying to resume\n"); @@ -1399,44 +1867,75 @@ retry: radeon_restore_bios_scratch_regs(rdev); - if (!r) { - for (i = 0; i < RADEON_NUM_RINGS; ++i) { + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + if (!r && ring_data[i]) { radeon_ring_restore(rdev, &rdev->ring[i], ring_sizes[i], ring_data[i]); - ring_sizes[i] = 0; - ring_data[i] = NULL; + } else { + radeon_fence_driver_force_completion(rdev, i); + kfree(ring_data[i]); } + } - r = radeon_ib_ring_tests(rdev); + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + /* do dpm late init */ + r = radeon_pm_late_init(rdev); if (r) { - dev_err(rdev->dev, "ib ring test failed (%d).\n", r); - if (saved) { - saved = false; - radeon_suspend(rdev); - goto retry; - } + rdev->pm.dpm_enabled = false; + DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n"); } } else { - radeon_fence_driver_force_completion(rdev); - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - kfree(ring_data[i]); + /* resume old pm late */ + radeon_pm_resume(rdev); + } + + /* init dig PHYs, disp eng pll */ + if (rdev->is_atom_bios) { + radeon_atom_encoder_init(rdev); + radeon_atom_disp_eng_pll_init(rdev); + /* turn on the BL */ + if (rdev->mode_info.bl_encoder) { + u8 bl_level = radeon_get_backlight_level(rdev, + rdev->mode_info.bl_encoder); + radeon_set_backlight_level(rdev, rdev->mode_info.bl_encoder, + bl_level); } } + /* reset hpd state */ + radeon_hpd_init(rdev); + + ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched); + + rdev->in_reset = true; + rdev->needs_reset = false; + +#ifdef notyet + downgrade_write(&rdev->exclusive_lock); +#endif drm_helper_resume_force_mode(rdev->ddev); - ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched); - if (r) { + /* set the power state here in case we are a PX system or headless */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) + radeon_pm_compute_clocks(rdev); + + if (!r) { + r = radeon_ib_ring_tests(rdev); + if (r && saved) + r = -EAGAIN; + } else { /* bad news, how to tell it to userspace ? */ dev_info(rdev->dev, "GPU reset failed\n"); } - up_write(&rdev->exclusive_lock); + rdev->needs_reset = r == -EAGAIN; + rdev->in_reset = false; + + up_read(&rdev->exclusive_lock); return r; } -#ifdef __linux__ /* * Debugfs */ @@ -1500,4 +1999,3 @@ void radeon_debugfs_cleanup(struct drm_minor *minor) { } #endif -#endif // __linux__ diff --git a/sys/dev/pci/drm/radeon/radeon_devlist.h b/sys/dev/pci/drm/radeon/radeon_devlist.h index 67edd716f53..e333d35624e 100644 --- a/sys/dev/pci/drm/radeon/radeon_devlist.h +++ b/sys/dev/pci/drm/radeon/radeon_devlist.h @@ -1,5 +1,27 @@ /* THIS FILE AUTOMATICALLY GENERATED. DO NOT EDIT. */ static const struct pci_matchid radeon_devices[] = { + { 0x1002, 0x1304 }, + { 0x1002, 0x1305 }, + { 0x1002, 0x1306 }, + { 0x1002, 0x1307 }, + { 0x1002, 0x1309 }, + { 0x1002, 0x130A }, + { 0x1002, 0x130B }, + { 0x1002, 0x130C }, + { 0x1002, 0x130D }, + { 0x1002, 0x130E }, + { 0x1002, 0x130F }, + { 0x1002, 0x1310 }, + { 0x1002, 0x1311 }, + { 0x1002, 0x1312 }, + { 0x1002, 0x1313 }, + { 0x1002, 0x1315 }, + { 0x1002, 0x1316 }, + { 0x1002, 0x1317 }, + { 0x1002, 0x1318 }, + { 0x1002, 0x131B }, + { 0x1002, 0x131C }, + { 0x1002, 0x131D }, { 0x1002, 0x3150 }, { 0x1002, 0x3151 }, { 0x1002, 0x3152 }, @@ -53,7 +75,6 @@ static const struct pci_matchid radeon_devices[] = { { 0x1002, 0x4C64 }, { 0x1002, 0x4C66 }, { 0x1002, 0x4C67 }, - { 0x1002, 0x4C6E }, { 0x1002, 0x4E44 }, { 0x1002, 0x4E45 }, { 0x1002, 0x4E46 }, @@ -140,6 +161,40 @@ static const struct pci_matchid radeon_devices[] = { { 0x1002, 0x5e4c }, { 0x1002, 0x5e4d }, { 0x1002, 0x5e4f }, + { 0x1002, 0x6600 }, + { 0x1002, 0x6601 }, + { 0x1002, 0x6602 }, + { 0x1002, 0x6603 }, + { 0x1002, 0x6604 }, + { 0x1002, 0x6605 }, + { 0x1002, 0x6606 }, + { 0x1002, 0x6607 }, + { 0x1002, 0x6608 }, + { 0x1002, 0x6610 }, + { 0x1002, 0x6611 }, + { 0x1002, 0x6613 }, + { 0x1002, 0x6617 }, + { 0x1002, 0x6620 }, + { 0x1002, 0x6621 }, + { 0x1002, 0x6623 }, + { 0x1002, 0x6631 }, + { 0x1002, 0x6640 }, + { 0x1002, 0x6641 }, + { 0x1002, 0x6646 }, + { 0x1002, 0x6647 }, + { 0x1002, 0x6649 }, + { 0x1002, 0x6650 }, + { 0x1002, 0x6651 }, + { 0x1002, 0x6658 }, + { 0x1002, 0x665c }, + { 0x1002, 0x665d }, + { 0x1002, 0x665f }, + { 0x1002, 0x6660 }, + { 0x1002, 0x6663 }, + { 0x1002, 0x6664 }, + { 0x1002, 0x6665 }, + { 0x1002, 0x6667 }, + { 0x1002, 0x666F }, { 0x1002, 0x6700 }, { 0x1002, 0x6701 }, { 0x1002, 0x6702 }, @@ -214,6 +269,18 @@ static const struct pci_matchid radeon_devices[] = { { 0x1002, 0x679B }, { 0x1002, 0x679E }, { 0x1002, 0x679F }, + { 0x1002, 0x67A0 }, + { 0x1002, 0x67A1 }, + { 0x1002, 0x67A2 }, + { 0x1002, 0x67A8 }, + { 0x1002, 0x67A9 }, + { 0x1002, 0x67AA }, + { 0x1002, 0x67B0 }, + { 0x1002, 0x67B1 }, + { 0x1002, 0x67B8 }, + { 0x1002, 0x67B9 }, + { 0x1002, 0x67BA }, + { 0x1002, 0x67BE }, { 0x1002, 0x6800 }, { 0x1002, 0x6801 }, { 0x1002, 0x6802 }, @@ -238,6 +305,7 @@ static const struct pci_matchid radeon_devices[] = { { 0x1002, 0x6829 }, { 0x1002, 0x682A }, { 0x1002, 0x682B }, + { 0x1002, 0x682C }, { 0x1002, 0x682D }, { 0x1002, 0x682F }, { 0x1002, 0x6830 }, @@ -562,6 +630,38 @@ static const struct pci_matchid radeon_devices[] = { { 0x1002, 0x9808 }, { 0x1002, 0x9809 }, { 0x1002, 0x980A }, + { 0x1002, 0x9830 }, + { 0x1002, 0x9831 }, + { 0x1002, 0x9832 }, + { 0x1002, 0x9833 }, + { 0x1002, 0x9834 }, + { 0x1002, 0x9835 }, + { 0x1002, 0x9836 }, + { 0x1002, 0x9837 }, + { 0x1002, 0x9838 }, + { 0x1002, 0x9839 }, + { 0x1002, 0x983a }, + { 0x1002, 0x983b }, + { 0x1002, 0x983c }, + { 0x1002, 0x983d }, + { 0x1002, 0x983e }, + { 0x1002, 0x983f }, + { 0x1002, 0x9850 }, + { 0x1002, 0x9851 }, + { 0x1002, 0x9852 }, + { 0x1002, 0x9853 }, + { 0x1002, 0x9854 }, + { 0x1002, 0x9855 }, + { 0x1002, 0x9856 }, + { 0x1002, 0x9857 }, + { 0x1002, 0x9858 }, + { 0x1002, 0x9859 }, + { 0x1002, 0x985A }, + { 0x1002, 0x985B }, + { 0x1002, 0x985C }, + { 0x1002, 0x985D }, + { 0x1002, 0x985E }, + { 0x1002, 0x985F }, { 0x1002, 0x9900 }, { 0x1002, 0x9901 }, { 0x1002, 0x9903 }, diff --git a/sys/dev/pci/drm/radeon/radeon_display.c b/sys/dev/pci/drm/radeon/radeon_display.c index bd47050d20f..821c42c85b1 100644 --- a/sys/dev/pci/drm/radeon/radeon_display.c +++ b/sys/dev/pci/drm/radeon/radeon_display.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_display.c,v 1.15 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2007-8 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -34,6 +33,7 @@ #include #include + static void avivo_crtc_load_lut(struct drm_crtc *crtc) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); @@ -64,7 +64,8 @@ static void avivo_crtc_load_lut(struct drm_crtc *crtc) (radeon_crtc->lut_b[i] << 0)); } - WREG32(AVIVO_D1GRPH_LUT_SEL + radeon_crtc->crtc_offset, radeon_crtc->crtc_id); + /* Only change bit 0 of LUT_SEL, other bits are set elsewhere */ + WREG32_P(AVIVO_D1GRPH_LUT_SEL + radeon_crtc->crtc_offset, radeon_crtc->crtc_id, ~1); } static void dce4_crtc_load_lut(struct drm_crtc *crtc) @@ -150,11 +151,17 @@ static void dce5_crtc_load_lut(struct drm_crtc *crtc) (NI_GRPH_REGAMMA_MODE(NI_REGAMMA_BYPASS) | NI_OVL_REGAMMA_MODE(NI_REGAMMA_BYPASS))); WREG32(NI_OUTPUT_CSC_CONTROL + radeon_crtc->crtc_offset, - (NI_OUTPUT_CSC_GRPH_MODE(NI_OUTPUT_CSC_BYPASS) | + (NI_OUTPUT_CSC_GRPH_MODE(radeon_crtc->output_csc) | NI_OUTPUT_CSC_OVL_MODE(NI_OUTPUT_CSC_BYPASS))); /* XXX match this to the depth of the crtc fmt block, move to modeset? */ WREG32(0x6940 + radeon_crtc->crtc_offset, 0); - + if (ASIC_IS_DCE8(rdev)) { + /* XXX this only needs to be programmed once per crtc at startup, + * not sure where the best place for it is + */ + WREG32(CIK_ALPHA_CONTROL + radeon_crtc->crtc_offset, + CIK_CURSOR_ALPHA_BLND_ENA); + } } static void legacy_crtc_load_lut(struct drm_crtc *crtc) @@ -241,15 +248,21 @@ static void radeon_crtc_destroy(struct drm_crtc *crtc) struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); drm_crtc_cleanup(crtc); + destroy_workqueue(radeon_crtc->flip_queue); kfree(radeon_crtc); } -/* - * Handle unpin events outside the interrupt handler proper. +/** + * radeon_unpin_work_func - unpin old buffer object + * + * @__work - kernel work item + * + * Unpin the old frame buffer object outside of the interrupt handler */ -static void radeon_unpin_work_func(void *arg1) +static void radeon_unpin_work_func(struct work_struct *__work) { - struct radeon_unpin_work *work = arg1; + struct radeon_flip_work *work = + container_of(__work, struct radeon_flip_work, unpin_work); int r; /* unpin of the old buffer */ @@ -267,41 +280,50 @@ static void radeon_unpin_work_func(void *arg1) kfree(work); } -void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id) +void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; - struct radeon_unpin_work *work; unsigned long flags; u32 update_pending; int vpos, hpos; + /* can happen during initialization */ + if (radeon_crtc == NULL) + return; + + /* Skip the pageflip completion check below (based on polling) on + * asics which reliably support hw pageflip completion irqs. pflip + * irqs are a reliable and race-free method of handling pageflip + * completion detection. A use_pflipirq module parameter < 2 allows + * to override this in case of asics with faulty pflip irqs. + * A module parameter of 0 would only use this polling based path, + * a parameter of 1 would use pflip irq only as a backup to this + * path, as in Linux 3.16. + */ + if ((radeon_use_pflipirq == 2) && ASIC_IS_DCE4(rdev)) + return; + spin_lock_irqsave(&rdev->ddev->event_lock, flags); - work = radeon_crtc->unpin_work; - if (work == NULL || - (work->fence && !radeon_fence_signaled(work->fence))) { + if (radeon_crtc->flip_status != RADEON_FLIP_SUBMITTED) { + DRM_DEBUG_DRIVER("radeon_crtc->flip_status = %d != " + "RADEON_FLIP_SUBMITTED(%d)\n", + radeon_crtc->flip_status, + RADEON_FLIP_SUBMITTED); spin_unlock_irqrestore(&rdev->ddev->event_lock, flags); return; } - /* New pageflip, or just completion of a previous one? */ - if (!radeon_crtc->deferred_flip_completion) { - /* do the flip (mmio) */ - update_pending = radeon_page_flip(rdev, crtc_id, work->new_crtc_base); - } else { - /* This is just a completion of a flip queued in crtc - * at last invocation. Make sure we go directly to - * completion routine. - */ - update_pending = 0; - radeon_crtc->deferred_flip_completion = 0; - } + + update_pending = radeon_page_flip_pending(rdev, crtc_id); /* Has the pageflip already completed in crtc, or is it certain * to complete in this vblank? */ if (update_pending && (DRM_SCANOUTPOS_VALID & radeon_get_crtc_scanoutpos(rdev->ddev, - crtc_id, 0, - &vpos, &hpos, NULL, NULL, &rdev->mode_info.crtcs[crtc_id]->base.hwmode)) && + crtc_id, + USE_REAL_VBLANKSTART, + &vpos, &hpos, NULL, NULL, + &rdev->mode_info.crtcs[crtc_id]->base.hwmode)) && ((vpos >= (99 * rdev->mode_info.crtcs[crtc_id]->base.hwmode.crtc_vdisplay)/100) || (vpos < 0 && !ASIC_IS_AVIVO(rdev)))) { /* crtc didn't flip in this target vblank interval, @@ -312,19 +334,43 @@ void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id) */ update_pending = 0; } - if (update_pending) { - /* crtc didn't flip in this target vblank interval, - * but flip is pending in crtc. It will complete it - * in next vblank interval, so complete the flip at - * next vblank irq. - */ - radeon_crtc->deferred_flip_completion = 1; + spin_unlock_irqrestore(&rdev->ddev->event_lock, flags); + if (!update_pending) + radeon_crtc_handle_flip(rdev, crtc_id); +} + +/** + * radeon_crtc_handle_flip - page flip completed + * + * @rdev: radeon device pointer + * @crtc_id: crtc number this event is for + * + * Called when we are sure that a page flip for this crtc is completed. + */ +void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id) +{ + struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; + struct radeon_flip_work *work; + unsigned long flags; + + /* this can happen at init */ + if (radeon_crtc == NULL) + return; + + spin_lock_irqsave(&rdev->ddev->event_lock, flags); + work = radeon_crtc->flip_work; + if (radeon_crtc->flip_status != RADEON_FLIP_SUBMITTED) { + DRM_DEBUG_DRIVER("radeon_crtc->flip_status = %d != " + "RADEON_FLIP_SUBMITTED(%d)\n", + radeon_crtc->flip_status, + RADEON_FLIP_SUBMITTED); spin_unlock_irqrestore(&rdev->ddev->event_lock, flags); return; } - /* Pageflip (will be) certainly completed in this vblank. Clean up. */ - radeon_crtc->unpin_work = NULL; + /* Pageflip completed. Clean up. */ + radeon_crtc->flip_status = RADEON_FLIP_NONE; + radeon_crtc->flip_work = NULL; /* wakeup userspace */ if (work->event) @@ -333,9 +379,119 @@ void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id) spin_unlock_irqrestore(&rdev->ddev->event_lock, flags); drm_vblank_put(rdev->ddev, radeon_crtc->crtc_id); - radeon_fence_unref(&work->fence); - radeon_post_page_flip(work->rdev, work->crtc_id); - task_add(systq, &work->task); + radeon_irq_kms_pflip_irq_put(rdev, work->crtc_id); + queue_work(radeon_crtc->flip_queue, &work->unpin_work); +} + +/** + * radeon_flip_work_func - page flip framebuffer + * + * @work - kernel work item + * + * Wait for the buffer object to become idle and do the actual page flip + */ +static void radeon_flip_work_func(struct work_struct *__work) +{ + struct radeon_flip_work *work = + container_of(__work, struct radeon_flip_work, flip_work); + struct radeon_device *rdev = work->rdev; + struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[work->crtc_id]; + + struct drm_crtc *crtc = &radeon_crtc->base; + unsigned long flags; + int r; + int vpos, hpos, stat, min_udelay = 0; + unsigned repcnt = 4; + struct drm_vblank_crtc *vblank = &crtc->dev->vblank[work->crtc_id]; + + down_read(&rdev->exclusive_lock); + if (work->fence) { + struct radeon_fence *fence; + + fence = to_radeon_fence(work->fence); + if (fence && fence->rdev == rdev) { + r = radeon_fence_wait(fence, false); + if (r == -EDEADLK) { + up_read(&rdev->exclusive_lock); + do { + r = radeon_gpu_reset(rdev); + } while (r == -EAGAIN); + down_read(&rdev->exclusive_lock); + } + } else + r = fence_wait(work->fence, false); + + if (r) + DRM_ERROR("failed to wait on page flip fence (%d)!\n", r); + + /* We continue with the page flip even if we failed to wait on + * the fence, otherwise the DRM core and userspace will be + * confused about which BO the CRTC is scanning out + */ + + fence_put(work->fence); + work->fence = NULL; + } + + /* We borrow the event spin lock for protecting flip_status */ + spin_lock_irqsave(&crtc->dev->event_lock, flags); + + /* set the proper interrupt */ + radeon_irq_kms_pflip_irq_get(rdev, radeon_crtc->crtc_id); + + /* If this happens to execute within the "virtually extended" vblank + * interval before the start of the real vblank interval then it needs + * to delay programming the mmio flip until the real vblank is entered. + * This prevents completing a flip too early due to the way we fudge + * our vblank counter and vblank timestamps in order to work around the + * problem that the hw fires vblank interrupts before actual start of + * vblank (when line buffer refilling is done for a frame). It + * complements the fudging logic in radeon_get_crtc_scanoutpos() for + * timestamping and radeon_get_vblank_counter_kms() for vblank counts. + * + * In practice this won't execute very often unless on very fast + * machines because the time window for this to happen is very small. + */ + while (radeon_crtc->enabled && --repcnt) { + /* GET_DISTANCE_TO_VBLANKSTART returns distance to real vblank + * start in hpos, and to the "fudged earlier" vblank start in + * vpos. + */ + stat = radeon_get_crtc_scanoutpos(rdev->ddev, work->crtc_id, + GET_DISTANCE_TO_VBLANKSTART, + &vpos, &hpos, NULL, NULL, + &crtc->hwmode); + + if ((stat & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE)) != + (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE) || + !(vpos >= 0 && hpos <= 0)) + break; + + /* Sleep at least until estimated real start of hw vblank */ + min_udelay = (-hpos + 1) * max(vblank->linedur_ns / 1000, 5); + if (min_udelay > vblank->framedur_ns / 2000) { + /* Don't wait ridiculously long - something is wrong */ + repcnt = 0; + break; + } + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + usleep_range(min_udelay, 2 * min_udelay); + spin_lock_irqsave(&crtc->dev->event_lock, flags); + }; + + if (!repcnt) + DRM_DEBUG_DRIVER("Delay problem on crtc %d: min_udelay %d, " + "framedur %d, linedur %d, stat %d, vpos %d, " + "hpos %d\n", work->crtc_id, min_udelay, + vblank->framedur_ns / 1000, + vblank->linedur_ns / 1000, stat, vpos, hpos); + + /* do the flip (mmio) */ + radeon_page_flip(rdev, radeon_crtc->crtc_id, work->base); + + radeon_crtc->flip_status = RADEON_FLIP_SUBMITTED; + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + up_read(&rdev->exclusive_lock); } static int radeon_crtc_page_flip(struct drm_crtc *crtc, @@ -349,69 +505,57 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc, struct radeon_framebuffer *old_radeon_fb; struct radeon_framebuffer *new_radeon_fb; struct drm_gem_object *obj; - struct radeon_bo *rbo; - struct radeon_unpin_work *work; + struct radeon_flip_work *work; + struct radeon_bo *new_rbo; + uint32_t tiling_flags, pitch_pixels; + uint64_t base; unsigned long flags; - u32 tiling_flags, pitch_pixels; - u64 base; int r; work = kzalloc(sizeof *work, GFP_KERNEL); if (work == NULL) return -ENOMEM; - work->event = event; + INIT_WORK(&work->flip_work, radeon_flip_work_func); + INIT_WORK(&work->unpin_work, radeon_unpin_work_func); + work->rdev = rdev; work->crtc_id = radeon_crtc->crtc_id; - old_radeon_fb = to_radeon_framebuffer(crtc->primary->fb); - new_radeon_fb = to_radeon_framebuffer(fb); + work->event = event; + /* schedule unpin of the old buffer */ + old_radeon_fb = to_radeon_framebuffer(crtc->primary->fb); obj = old_radeon_fb->obj; + /* take a reference to the old object */ drm_gem_object_reference(obj); - rbo = gem_to_radeon_bo(obj); - work->old_rbo = rbo; - obj = new_radeon_fb->obj; - rbo = gem_to_radeon_bo(obj); - - spin_lock(&rbo->tbo.bdev->fence_lock); - if (rbo->tbo.sync_obj) - work->fence = radeon_fence_ref(rbo->tbo.sync_obj); - spin_unlock(&rbo->tbo.bdev->fence_lock); + work->old_rbo = gem_to_radeon_bo(obj); - task_set(&work->task, radeon_unpin_work_func, work); - - /* We borrow the event spin lock for protecting unpin_work */ - spin_lock_irqsave(&dev->event_lock, flags); - if (radeon_crtc->unpin_work) { - DRM_DEBUG_DRIVER("flip queue: crtc already busy\n"); - r = -EBUSY; - goto unlock_free; - } - radeon_crtc->unpin_work = work; - radeon_crtc->deferred_flip_completion = 0; - spin_unlock_irqrestore(&dev->event_lock, flags); + new_radeon_fb = to_radeon_framebuffer(fb); + obj = new_radeon_fb->obj; + new_rbo = gem_to_radeon_bo(obj); /* pin the new buffer */ - DRM_DEBUG_DRIVER("flip-ioctl() cur_fbo = %p, cur_bbo = %p\n", - work->old_rbo, rbo); + DRM_DEBUG_DRIVER("flip-ioctl() cur_rbo = %p, new_rbo = %p\n", + work->old_rbo, new_rbo); - r = radeon_bo_reserve(rbo, false); + r = radeon_bo_reserve(new_rbo, false); if (unlikely(r != 0)) { DRM_ERROR("failed to reserve new rbo buffer before flip\n"); - goto pflip_cleanup; + goto cleanup; } /* Only 27 bit offset for legacy CRTC */ - r = radeon_bo_pin_restricted(rbo, RADEON_GEM_DOMAIN_VRAM, + r = radeon_bo_pin_restricted(new_rbo, RADEON_GEM_DOMAIN_VRAM, ASIC_IS_AVIVO(rdev) ? 0 : 1 << 27, &base); if (unlikely(r != 0)) { - radeon_bo_unreserve(rbo); + radeon_bo_unreserve(new_rbo); r = -EINVAL; DRM_ERROR("failed to pin new rbo buffer before flip\n"); - goto pflip_cleanup; + goto cleanup; } - radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL); - radeon_bo_unreserve(rbo); + work->fence = fence_get(reservation_object_get_excl(new_rbo->tbo.resv)); + radeon_bo_get_tiling_flags(new_rbo, &tiling_flags, NULL); + radeon_bo_unreserve(new_rbo); if (!ASIC_IS_AVIVO(rdev)) { /* crtc offset is from display base addr not FB location */ @@ -448,52 +592,103 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc, } base &= ~7; } + work->base = base; - spin_lock_irqsave(&dev->event_lock, flags); - work->new_crtc_base = base; - spin_unlock_irqrestore(&dev->event_lock, flags); - - /* update crtc fb */ - crtc->primary->fb = fb; - - r = drm_vblank_get(dev, radeon_crtc->crtc_id); + r = drm_vblank_get(crtc->dev, radeon_crtc->crtc_id); if (r) { DRM_ERROR("failed to get vblank before flip\n"); - goto pflip_cleanup1; + goto pflip_cleanup; } - /* set the proper interrupt */ - radeon_pre_page_flip(rdev, radeon_crtc->crtc_id); + /* We borrow the event spin lock for protecting flip_work */ + spin_lock_irqsave(&crtc->dev->event_lock, flags); + + if (radeon_crtc->flip_status != RADEON_FLIP_NONE) { + DRM_DEBUG_DRIVER("flip queue: crtc already busy\n"); + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + r = -EBUSY; + goto vblank_cleanup; + } + radeon_crtc->flip_status = RADEON_FLIP_PENDING; + radeon_crtc->flip_work = work; + + /* update crtc fb */ + crtc->primary->fb = fb; + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + + queue_work(radeon_crtc->flip_queue, &work->flip_work); return 0; -pflip_cleanup1: - if (unlikely(radeon_bo_reserve(rbo, false) != 0)) { +vblank_cleanup: + drm_vblank_put(crtc->dev, radeon_crtc->crtc_id); + +pflip_cleanup: + if (unlikely(radeon_bo_reserve(new_rbo, false) != 0)) { DRM_ERROR("failed to reserve new rbo in error path\n"); - goto pflip_cleanup; + goto cleanup; } - if (unlikely(radeon_bo_unpin(rbo) != 0)) { + if (unlikely(radeon_bo_unpin(new_rbo) != 0)) { DRM_ERROR("failed to unpin new rbo in error path\n"); } - radeon_bo_unreserve(rbo); + radeon_bo_unreserve(new_rbo); -pflip_cleanup: - spin_lock_irqsave(&dev->event_lock, flags); - radeon_crtc->unpin_work = NULL; -unlock_free: - spin_unlock_irqrestore(&dev->event_lock, flags); - drm_gem_object_unreference_unlocked(old_radeon_fb->obj); - radeon_fence_unref(&work->fence); +cleanup: + drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base); + fence_put(work->fence); kfree(work); - return r; } +static int +radeon_crtc_set_config(struct drm_mode_set *set) +{ + struct drm_device *dev; + struct radeon_device *rdev; + struct drm_crtc *crtc; + bool active = false; + int ret; + + if (!set || !set->crtc) + return -EINVAL; + + dev = set->crtc->dev; + + ret = pm_runtime_get_sync(dev->dev); + if (ret < 0) + return ret; + + ret = drm_crtc_helper_set_config(set); + + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) + if (crtc->enabled) + active = true; + + pm_runtime_mark_last_busy(dev->dev); + + rdev = dev->dev_private; + /* if we have active crtcs and we don't have a power ref, + take the current one */ + if (active && !rdev->have_disp_power_ref) { + rdev->have_disp_power_ref = true; + return ret; + } + /* if we have no active crtcs, then drop the power ref + we got before */ + if (!active && rdev->have_disp_power_ref) { + pm_runtime_put_autosuspend(dev->dev); + rdev->have_disp_power_ref = false; + } + + /* drop the power reference we got coming in here */ + pm_runtime_put_autosuspend(dev->dev); + return ret; +} static const struct drm_crtc_funcs radeon_crtc_funcs = { - .cursor_set = radeon_crtc_cursor_set, + .cursor_set2 = radeon_crtc_cursor_set2, .cursor_move = radeon_crtc_cursor_move, .gamma_set = radeon_crtc_gamma_set, - .set_config = drm_crtc_helper_set_config, + .set_config = radeon_crtc_set_config, .destroy = radeon_crtc_destroy, .page_flip = radeon_crtc_page_flip, }; @@ -512,8 +707,19 @@ static void radeon_crtc_init(struct drm_device *dev, int index) drm_mode_crtc_set_gamma_size(&radeon_crtc->base, 256); radeon_crtc->crtc_id = index; + radeon_crtc->flip_queue = create_singlethread_workqueue("radeon-crtc"); rdev->mode_info.crtcs[index] = radeon_crtc; + if (rdev->family >= CHIP_BONAIRE) { + radeon_crtc->max_cursor_width = CIK_CURSOR_WIDTH; + radeon_crtc->max_cursor_height = CIK_CURSOR_HEIGHT; + } else { + radeon_crtc->max_cursor_width = CURSOR_WIDTH; + radeon_crtc->max_cursor_height = CURSOR_HEIGHT; + } + dev->mode_config.cursor_width = radeon_crtc->max_cursor_width; + dev->mode_config.cursor_height = radeon_crtc->max_cursor_height; + #if 0 radeon_crtc->mode_set.crtc = &radeon_crtc->base; radeon_crtc->mode_set.connectors = (struct drm_connector **)(radeon_crtc + 1); @@ -533,7 +739,7 @@ static void radeon_crtc_init(struct drm_device *dev, int index) } #ifdef DRMDEBUG -static const char *encoder_names[37] = { +static const char *encoder_names[38] = { "NONE", "INTERNAL_LVDS", "INTERNAL_TMDS1", @@ -570,7 +776,8 @@ static const char *encoder_names[37] = { "INTERNAL_UNIPHY2", "NUTMEG", "TRAVIS", - "INTERNAL_VCE" + "INTERNAL_VCE", + "INTERNAL_UNIPHY3", }; static const char *hpd_names[6] = { @@ -581,11 +788,10 @@ static const char *hpd_names[6] = { "HPD5", "HPD6", }; -#endif /* DRMDEBUG */ +#endif static void radeon_print_display_setup(struct drm_device *dev) { -#ifdef DRMDEBUG struct drm_connector *connector; struct radeon_connector *radeon_connector; struct drm_encoder *encoder; @@ -658,7 +864,6 @@ static void radeon_print_display_setup(struct drm_device *dev) } i++; } -#endif /* DRMDEBUG */ } static bool radeon_setup_enc_conn(struct drm_device *dev) @@ -688,176 +893,230 @@ static bool radeon_setup_enc_conn(struct drm_device *dev) return ret; } -int radeon_ddc_get_modes(struct radeon_connector *radeon_connector) +/* avivo */ + +/** + * avivo_reduce_ratio - fractional number reduction + * + * @nom: nominator + * @den: denominator + * @nom_min: minimum value for nominator + * @den_min: minimum value for denominator + * + * Find the greatest common divisor and apply it on both nominator and + * denominator, but make nominator and denominator are at least as large + * as their minimum values. + */ +static void avivo_reduce_ratio(unsigned *nom, unsigned *den, + unsigned nom_min, unsigned den_min) { - struct drm_device *dev = radeon_connector->base.dev; - struct radeon_device *rdev = dev->dev_private; - int ret = 0; - - /* on hw with routers, select right port */ - if (radeon_connector->router.ddc_valid) - radeon_router_select_ddc_port(radeon_connector); - - if (radeon_connector_encoder_get_dp_bridge_encoder_id(&radeon_connector->base) != - ENCODER_OBJECT_ID_NONE) { - struct radeon_connector_atom_dig *dig = radeon_connector->con_priv; - - if (dig->dp_i2c_bus) - radeon_connector->edid = drm_get_edid(&radeon_connector->base, - &dig->dp_i2c_bus->adapter); - } else if ((radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_DisplayPort) || - (radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)) { - struct radeon_connector_atom_dig *dig = radeon_connector->con_priv; - - if ((dig->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT || - dig->dp_sink_type == CONNECTOR_OBJECT_ID_eDP) && dig->dp_i2c_bus) - radeon_connector->edid = drm_get_edid(&radeon_connector->base, - &dig->dp_i2c_bus->adapter); - else if (radeon_connector->ddc_bus && !radeon_connector->edid) - radeon_connector->edid = drm_get_edid(&radeon_connector->base, - &radeon_connector->ddc_bus->adapter); - } else { - if (radeon_connector->ddc_bus && !radeon_connector->edid) - radeon_connector->edid = drm_get_edid(&radeon_connector->base, - &radeon_connector->ddc_bus->adapter); + unsigned tmp; + + /* reduce the numbers to a simpler ratio */ + tmp = gcd(*nom, *den); + *nom /= tmp; + *den /= tmp; + + /* make sure nominator is large enough */ + if (*nom < nom_min) { + tmp = DIV_ROUND_UP(nom_min, *nom); + *nom *= tmp; + *den *= tmp; } - if (!radeon_connector->edid) { - if (rdev->is_atom_bios) { - /* some laptops provide a hardcoded edid in rom for LCDs */ - if (((radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_LVDS) || - (radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP))) - radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev); - } else - /* some servers provide a hardcoded edid in rom for KVMs */ - radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev); - } - if (radeon_connector->edid) { - drm_mode_connector_update_edid_property(&radeon_connector->base, radeon_connector->edid); - ret = drm_add_edid_modes(&radeon_connector->base, radeon_connector->edid); - drm_edid_to_eld(&radeon_connector->base, radeon_connector->edid); - return ret; + /* make sure the denominator is large enough */ + if (*den < den_min) { + tmp = DIV_ROUND_UP(den_min, *den); + *nom *= tmp; + *den *= tmp; } - drm_mode_connector_update_edid_property(&radeon_connector->base, NULL); - return 0; } -/* avivo */ -static void avivo_get_fb_div(struct radeon_pll *pll, - u32 target_clock, - u32 post_div, - u32 ref_div, - u32 *fb_div, - u32 *frac_fb_div) +/** + * avivo_get_fb_ref_div - feedback and ref divider calculation + * + * @nom: nominator + * @den: denominator + * @post_div: post divider + * @fb_div_max: feedback divider maximum + * @ref_div_max: reference divider maximum + * @fb_div: resulting feedback divider + * @ref_div: resulting reference divider + * + * Calculate feedback and reference divider for a given post divider. Makes + * sure we stay within the limits. + */ +static void avivo_get_fb_ref_div(unsigned nom, unsigned den, unsigned post_div, + unsigned fb_div_max, unsigned ref_div_max, + unsigned *fb_div, unsigned *ref_div) { - u32 tmp = post_div * ref_div; + /* limit reference * post divider to a maximum */ + ref_div_max = max(min(100 / post_div, ref_div_max), 1u); - tmp *= target_clock; - *fb_div = tmp / pll->reference_freq; - *frac_fb_div = tmp % pll->reference_freq; + /* get matching reference and feedback divider */ + *ref_div = min(max(DIV_ROUND_CLOSEST(den, post_div), 1u), ref_div_max); + *fb_div = DIV_ROUND_CLOSEST(nom * *ref_div * post_div, den); - if (*fb_div > pll->max_feedback_div) - *fb_div = pll->max_feedback_div; - else if (*fb_div < pll->min_feedback_div) - *fb_div = pll->min_feedback_div; + /* limit fb divider to its maximum */ + if (*fb_div > fb_div_max) { + *ref_div = DIV_ROUND_CLOSEST(*ref_div * fb_div_max, *fb_div); + *fb_div = fb_div_max; + } } -static u32 avivo_get_post_div(struct radeon_pll *pll, - u32 target_clock) +/** + * radeon_compute_pll_avivo - compute PLL paramaters + * + * @pll: information about the PLL + * @dot_clock_p: resulting pixel clock + * fb_div_p: resulting feedback divider + * frac_fb_div_p: fractional part of the feedback divider + * ref_div_p: resulting reference divider + * post_div_p: resulting reference divider + * + * Try to calculate the PLL parameters to generate the given frequency: + * dot_clock = (ref_freq * feedback_div) / (ref_div * post_div) + */ +void radeon_compute_pll_avivo(struct radeon_pll *pll, + u32 freq, + u32 *dot_clock_p, + u32 *fb_div_p, + u32 *frac_fb_div_p, + u32 *ref_div_p, + u32 *post_div_p) { - u32 vco, post_div, tmp; + unsigned target_clock = pll->flags & RADEON_PLL_USE_FRAC_FB_DIV ? + freq : freq / 10; - if (pll->flags & RADEON_PLL_USE_POST_DIV) - return pll->post_div; + unsigned fb_div_min, fb_div_max, fb_div; + unsigned post_div_min, post_div_max, post_div; + unsigned ref_div_min, ref_div_max, ref_div; + unsigned post_div_best, diff_best; + unsigned nom, den; - if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) { - if (pll->flags & RADEON_PLL_IS_LCD) - vco = pll->lcd_pll_out_min; - else - vco = pll->pll_out_min; - } else { - if (pll->flags & RADEON_PLL_IS_LCD) - vco = pll->lcd_pll_out_max; - else - vco = pll->pll_out_max; + /* determine allowed feedback divider range */ + fb_div_min = pll->min_feedback_div; + fb_div_max = pll->max_feedback_div; + + if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) { + fb_div_min *= 10; + fb_div_max *= 10; } - post_div = vco / target_clock; - tmp = vco % target_clock; + /* determine allowed ref divider range */ + if (pll->flags & RADEON_PLL_USE_REF_DIV) + ref_div_min = pll->reference_div; + else + ref_div_min = pll->min_ref_div; + + if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV && + pll->flags & RADEON_PLL_USE_REF_DIV) + ref_div_max = pll->reference_div; + else if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) + /* fix for problems on RS880 */ + ref_div_max = min(pll->max_ref_div, 7u); + else + ref_div_max = pll->max_ref_div; - if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) { - if (tmp) - post_div++; + /* determine allowed post divider range */ + if (pll->flags & RADEON_PLL_USE_POST_DIV) { + post_div_min = pll->post_div; + post_div_max = pll->post_div; } else { - if (!tmp) - post_div--; + unsigned vco_min, vco_max; + + if (pll->flags & RADEON_PLL_IS_LCD) { + vco_min = pll->lcd_pll_out_min; + vco_max = pll->lcd_pll_out_max; + } else { + vco_min = pll->pll_out_min; + vco_max = pll->pll_out_max; + } + + if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) { + vco_min *= 10; + vco_max *= 10; + } + + post_div_min = vco_min / target_clock; + if ((target_clock * post_div_min) < vco_min) + ++post_div_min; + if (post_div_min < pll->min_post_div) + post_div_min = pll->min_post_div; + + post_div_max = vco_max / target_clock; + if ((target_clock * post_div_max) > vco_max) + --post_div_max; + if (post_div_max > pll->max_post_div) + post_div_max = pll->max_post_div; } - if (post_div > pll->max_post_div) - post_div = pll->max_post_div; - else if (post_div < pll->min_post_div) - post_div = pll->min_post_div; + /* represent the searched ratio as fractional number */ + nom = target_clock; + den = pll->reference_freq; - return post_div; -} + /* reduce the numbers to a simpler ratio */ + avivo_reduce_ratio(&nom, &den, fb_div_min, post_div_min); -#define MAX_TOLERANCE 10 + /* now search for a post divider */ + if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) + post_div_best = post_div_min; + else + post_div_best = post_div_max; + diff_best = ~0; -void radeon_compute_pll_avivo(struct radeon_pll *pll, - u32 freq, - u32 *dot_clock_p, - u32 *fb_div_p, - u32 *frac_fb_div_p, - u32 *ref_div_p, - u32 *post_div_p) -{ - u32 target_clock = freq / 10; - u32 post_div = avivo_get_post_div(pll, target_clock); - u32 ref_div = pll->min_ref_div; - u32 fb_div = 0, frac_fb_div = 0, tmp; + for (post_div = post_div_min; post_div <= post_div_max; ++post_div) { + unsigned diff; + avivo_get_fb_ref_div(nom, den, post_div, fb_div_max, + ref_div_max, &fb_div, &ref_div); + diff = abs(target_clock - (pll->reference_freq * fb_div) / + (ref_div * post_div)); - if (pll->flags & RADEON_PLL_USE_REF_DIV) - ref_div = pll->reference_div; + if (diff < diff_best || (diff == diff_best && + !(pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP))) { - if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) { - avivo_get_fb_div(pll, target_clock, post_div, ref_div, &fb_div, &frac_fb_div); - frac_fb_div = (100 * frac_fb_div) / pll->reference_freq; - if (frac_fb_div >= 5) { - frac_fb_div -= 5; - frac_fb_div = frac_fb_div / 10; - frac_fb_div++; + post_div_best = post_div; + diff_best = diff; } - if (frac_fb_div >= 10) { - fb_div++; - frac_fb_div = 0; + } + post_div = post_div_best; + + /* get the feedback and reference divider for the optimal value */ + avivo_get_fb_ref_div(nom, den, post_div, fb_div_max, ref_div_max, + &fb_div, &ref_div); + + /* reduce the numbers to a simpler ratio once more */ + /* this also makes sure that the reference divider is large enough */ + avivo_reduce_ratio(&fb_div, &ref_div, fb_div_min, ref_div_min); + + /* avoid high jitter with small fractional dividers */ + if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV && (fb_div % 10)) { + fb_div_min = max(fb_div_min, (9 - (fb_div % 10)) * 20 + 50); + if (fb_div < fb_div_min) { + unsigned tmp = DIV_ROUND_UP(fb_div_min, fb_div); + fb_div *= tmp; + ref_div *= tmp; } + } + + /* and finally save the result */ + if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) { + *fb_div_p = fb_div / 10; + *frac_fb_div_p = fb_div % 10; } else { - while (ref_div <= pll->max_ref_div) { - avivo_get_fb_div(pll, target_clock, post_div, ref_div, - &fb_div, &frac_fb_div); - if (frac_fb_div >= (pll->reference_freq / 2)) - fb_div++; - frac_fb_div = 0; - tmp = (pll->reference_freq * fb_div) / (post_div * ref_div); - tmp = (tmp * 10000) / target_clock; - - if (tmp > (10000 + MAX_TOLERANCE)) - ref_div++; - else if (tmp >= (10000 - MAX_TOLERANCE)) - break; - else - ref_div++; - } + *fb_div_p = fb_div; + *frac_fb_div_p = 0; } - *dot_clock_p = ((pll->reference_freq * fb_div * 10) + (pll->reference_freq * frac_fb_div)) / - (ref_div * post_div * 10); - *fb_div_p = fb_div; - *frac_fb_div_p = frac_fb_div; + *dot_clock_p = ((pll->reference_freq * *fb_div_p * 10) + + (pll->reference_freq * *frac_fb_div_p)) / + (ref_div * post_div * 10); *ref_div_p = ref_div; *post_div_p = post_div; - DRM_DEBUG_KMS("%d, pll dividers - fb: %d.%d ref: %d, post %d\n", - *dot_clock_p, fb_div, frac_fb_div, ref_div, post_div); + + DRM_DEBUG_KMS("%d - %d, pll dividers - fb: %d.%d ref: %d, post %d\n", + freq, *dot_clock_p * 10, *fb_div_p, *frac_fb_div_p, + ref_div, post_div); } /* pre-avivo */ @@ -867,7 +1126,7 @@ static inline uint32_t radeon_div(uint64_t n, uint32_t d) n += d / 2; - mod = (n /= d); + mod = do_div(n, d); return n; } @@ -1089,12 +1348,12 @@ radeon_framebuffer_init(struct drm_device *dev, { int ret; rfb->obj = obj; + drm_helper_mode_fill_fb_struct(&rfb->base, mode_cmd); ret = drm_framebuffer_init(dev, &rfb->base, &radeon_fb_funcs); if (ret) { rfb->obj = NULL; return ret; } - drm_helper_mode_fill_fb_struct(&rfb->base, mode_cmd); return 0; } @@ -1114,6 +1373,14 @@ radeon_user_framebuffer_create(struct drm_device *dev, return ERR_PTR(-ENOENT); } +#ifdef notyet + /* Handle is imported dma-buf, so cannot be migrated to VRAM for scanout */ + if (obj->import_attach) { + DRM_DEBUG_KMS("Cannot create framebuffer from imported dma_buf\n"); + return ERR_PTR(-EINVAL); + } +#endif + radeon_fb = kzalloc(sizeof(*radeon_fb), GFP_KERNEL); if (radeon_fb == NULL) { drm_gem_object_unreference_unlocked(obj); @@ -1163,6 +1430,25 @@ static struct drm_prop_enum_list radeon_underscan_enum_list[] = { UNDERSCAN_AUTO, "auto" }, }; +static struct drm_prop_enum_list radeon_audio_enum_list[] = +{ { RADEON_AUDIO_DISABLE, "off" }, + { RADEON_AUDIO_ENABLE, "on" }, + { RADEON_AUDIO_AUTO, "auto" }, +}; + +/* XXX support different dither options? spatial, temporal, both, etc. */ +static struct drm_prop_enum_list radeon_dither_enum_list[] = +{ { RADEON_FMT_DITHER_DISABLE, "off" }, + { RADEON_FMT_DITHER_ENABLE, "on" }, +}; + +static struct drm_prop_enum_list radeon_output_csc_enum_list[] = +{ { RADEON_OUTPUT_CSC_BYPASS, "bypass" }, + { RADEON_OUTPUT_CSC_TVRGB, "tvrgb" }, + { RADEON_OUTPUT_CSC_YCBCR601, "ycbcr601" }, + { RADEON_OUTPUT_CSC_YCBCR709, "ycbcr709" }, +}; + static int radeon_modeset_create_props(struct radeon_device *rdev) { int sz; @@ -1213,6 +1499,24 @@ static int radeon_modeset_create_props(struct radeon_device *rdev) if (!rdev->mode_info.underscan_vborder_property) return -ENOMEM; + sz = ARRAY_SIZE(radeon_audio_enum_list); + rdev->mode_info.audio_property = + drm_property_create_enum(rdev->ddev, 0, + "audio", + radeon_audio_enum_list, sz); + + sz = ARRAY_SIZE(radeon_dither_enum_list); + rdev->mode_info.dither_property = + drm_property_create_enum(rdev->ddev, 0, + "dither", + radeon_dither_enum_list, sz); + + sz = ARRAY_SIZE(radeon_output_csc_enum_list); + rdev->mode_info.output_csc_property = + drm_property_create_enum(rdev->ddev, 0, + "output_csc", + radeon_output_csc_enum_list, sz); + return 0; } @@ -1246,41 +1550,41 @@ static void radeon_afmt_init(struct radeon_device *rdev) for (i = 0; i < RADEON_MAX_AFMT_BLOCKS; i++) rdev->mode_info.afmt[i] = NULL; - if (ASIC_IS_DCE6(rdev)) { - /* todo */ + if (ASIC_IS_NODCE(rdev)) { + /* nothing to do */ } else if (ASIC_IS_DCE4(rdev)) { + static uint32_t eg_offsets[] = { + EVERGREEN_CRTC0_REGISTER_OFFSET, + EVERGREEN_CRTC1_REGISTER_OFFSET, + EVERGREEN_CRTC2_REGISTER_OFFSET, + EVERGREEN_CRTC3_REGISTER_OFFSET, + EVERGREEN_CRTC4_REGISTER_OFFSET, + EVERGREEN_CRTC5_REGISTER_OFFSET, + 0x13830 - 0x7030, + }; + int num_afmt; + + /* DCE8 has 7 audio blocks tied to DIG encoders */ + /* DCE6 has 6 audio blocks tied to DIG encoders */ /* DCE4/5 has 6 audio blocks tied to DIG encoders */ /* DCE4.1 has 2 audio blocks tied to DIG encoders */ - rdev->mode_info.afmt[0] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL); - if (rdev->mode_info.afmt[0]) { - rdev->mode_info.afmt[0]->offset = EVERGREEN_CRTC0_REGISTER_OFFSET; - rdev->mode_info.afmt[0]->id = 0; - } - rdev->mode_info.afmt[1] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL); - if (rdev->mode_info.afmt[1]) { - rdev->mode_info.afmt[1]->offset = EVERGREEN_CRTC1_REGISTER_OFFSET; - rdev->mode_info.afmt[1]->id = 1; - } - if (!ASIC_IS_DCE41(rdev)) { - rdev->mode_info.afmt[2] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL); - if (rdev->mode_info.afmt[2]) { - rdev->mode_info.afmt[2]->offset = EVERGREEN_CRTC2_REGISTER_OFFSET; - rdev->mode_info.afmt[2]->id = 2; - } - rdev->mode_info.afmt[3] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL); - if (rdev->mode_info.afmt[3]) { - rdev->mode_info.afmt[3]->offset = EVERGREEN_CRTC3_REGISTER_OFFSET; - rdev->mode_info.afmt[3]->id = 3; - } - rdev->mode_info.afmt[4] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL); - if (rdev->mode_info.afmt[4]) { - rdev->mode_info.afmt[4]->offset = EVERGREEN_CRTC4_REGISTER_OFFSET; - rdev->mode_info.afmt[4]->id = 4; - } - rdev->mode_info.afmt[5] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL); - if (rdev->mode_info.afmt[5]) { - rdev->mode_info.afmt[5]->offset = EVERGREEN_CRTC5_REGISTER_OFFSET; - rdev->mode_info.afmt[5]->id = 5; + if (ASIC_IS_DCE8(rdev)) + num_afmt = 7; + else if (ASIC_IS_DCE6(rdev)) + num_afmt = 6; + else if (ASIC_IS_DCE5(rdev)) + num_afmt = 6; + else if (ASIC_IS_DCE41(rdev)) + num_afmt = 2; + else /* DCE4 */ + num_afmt = 6; + + BUG_ON(num_afmt > ARRAY_SIZE(eg_offsets)); + for (i = 0; i < num_afmt; i++) { + rdev->mode_info.afmt[i] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL); + if (rdev->mode_info.afmt[i]) { + rdev->mode_info.afmt[i]->offset = eg_offsets[i]; + rdev->mode_info.afmt[i]->id = i; } } } else if (ASIC_IS_DCE3(rdev)) { @@ -1386,12 +1690,12 @@ int radeon_modeset_init(struct radeon_device *rdev) /* setup afmt */ radeon_afmt_init(rdev); - /* Initialize power management */ - radeon_pm_init(rdev); - radeon_fbdev_init(rdev); drm_kms_helper_poll_init(rdev->ddev); + /* do pm late init */ + ret = radeon_pm_late_init(rdev); + return 0; } @@ -1399,7 +1703,6 @@ void radeon_modeset_fini(struct radeon_device *rdev) { radeon_fbdev_fini(rdev); kfree(rdev->mode_info.bios_hardcoded_edid); - radeon_pm_fini(rdev); if (rdev->mode_info.mode_config_initialized) { radeon_afmt_fini(rdev); @@ -1472,7 +1775,7 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc, (!(mode->flags & DRM_MODE_FLAG_INTERLACE)) && ((radeon_encoder->underscan_type == UNDERSCAN_ON) || ((radeon_encoder->underscan_type == UNDERSCAN_AUTO) && - drm_detect_hdmi_monitor(radeon_connector->edid) && + drm_detect_hdmi_monitor(radeon_connector_edid(connector)) && is_hdtv_mode(mode)))) { if (radeon_encoder->underscan_hborder != 0) radeon_crtc->h_border = radeon_encoder->underscan_hborder; @@ -1518,12 +1821,27 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc, } /* - * Retrieve current video scanout position of crtc on a given gpu. + * Retrieve current video scanout position of crtc on a given gpu, and + * an optional accurate timestamp of when query happened. * * \param dev Device to query. * \param crtc Crtc to query. + * \param flags Flags from caller (DRM_CALLED_FROM_VBLIRQ or 0). + * For driver internal use only also supports these flags: + * + * USE_REAL_VBLANKSTART to use the real start of vblank instead + * of a fudged earlier start of vblank. + * + * GET_DISTANCE_TO_VBLANKSTART to return distance to the + * fudged earlier start of vblank in *vpos and the distance + * to true start of vblank in *hpos. + * * \param *vpos Location where vertical scanout position should be stored. * \param *hpos Location where horizontal scanout position should go. + * \param *stime Target location for timestamp taken immediately before + * scanout position query. Can be NULL to skip timestamp. + * \param *etime Target location for timestamp taken immediately after + * scanout position query. Can be NULL to skip timestamp. * * Returns vpos as a positive number while in active scanout area. * Returns vpos as a negative number inside vblank, counting the number @@ -1539,8 +1857,10 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc, * unknown small number of scanlines wrt. real scanout position. * */ -int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int crtc, unsigned int flags, - int *vpos, int *hpos, ktime_t *stime, ktime_t *etime, const struct drm_display_mode *mode) +int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, + unsigned int flags, int *vpos, int *hpos, + ktime_t *stime, ktime_t *etime, + const struct drm_display_mode *mode) { u32 stat_crtc = 0, vbl = 0, position = 0; int vbl_start, vbl_end, vtotal, ret = 0; @@ -1555,42 +1875,42 @@ int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int crtc, unsign *stime = ktime_get(); if (ASIC_IS_DCE4(rdev)) { - if (crtc == 0) { + if (pipe == 0) { vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + EVERGREEN_CRTC0_REGISTER_OFFSET); position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + EVERGREEN_CRTC0_REGISTER_OFFSET); ret |= DRM_SCANOUTPOS_VALID; } - if (crtc == 1) { + if (pipe == 1) { vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + EVERGREEN_CRTC1_REGISTER_OFFSET); position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + EVERGREEN_CRTC1_REGISTER_OFFSET); ret |= DRM_SCANOUTPOS_VALID; } - if (crtc == 2) { + if (pipe == 2) { vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + EVERGREEN_CRTC2_REGISTER_OFFSET); position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + EVERGREEN_CRTC2_REGISTER_OFFSET); ret |= DRM_SCANOUTPOS_VALID; } - if (crtc == 3) { + if (pipe == 3) { vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + EVERGREEN_CRTC3_REGISTER_OFFSET); position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + EVERGREEN_CRTC3_REGISTER_OFFSET); ret |= DRM_SCANOUTPOS_VALID; } - if (crtc == 4) { + if (pipe == 4) { vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + EVERGREEN_CRTC4_REGISTER_OFFSET); position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + EVERGREEN_CRTC4_REGISTER_OFFSET); ret |= DRM_SCANOUTPOS_VALID; } - if (crtc == 5) { + if (pipe == 5) { vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + EVERGREEN_CRTC5_REGISTER_OFFSET); position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + @@ -1598,19 +1918,19 @@ int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int crtc, unsign ret |= DRM_SCANOUTPOS_VALID; } } else if (ASIC_IS_AVIVO(rdev)) { - if (crtc == 0) { + if (pipe == 0) { vbl = RREG32(AVIVO_D1CRTC_V_BLANK_START_END); position = RREG32(AVIVO_D1CRTC_STATUS_POSITION); ret |= DRM_SCANOUTPOS_VALID; } - if (crtc == 1) { + if (pipe == 1) { vbl = RREG32(AVIVO_D2CRTC_V_BLANK_START_END); position = RREG32(AVIVO_D2CRTC_STATUS_POSITION); ret |= DRM_SCANOUTPOS_VALID; } } else { /* Pre-AVIVO: Different encoding of scanout pos and vblank interval. */ - if (crtc == 0) { + if (pipe == 0) { /* Assume vbl_end == 0, get vbl_start from * upper 16 bits. */ @@ -1624,7 +1944,7 @@ int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int crtc, unsign ret |= DRM_SCANOUTPOS_VALID; } - if (crtc == 1) { + if (pipe == 1) { vbl = (RREG32(RADEON_CRTC2_V_TOTAL_DISP) & RADEON_CRTC_V_DISP) >> RADEON_CRTC_V_DISP_SHIFT; position = (RREG32(RADEON_CRTC2_VLINE_CRNT_VLINE) >> 16) & RADEON_CRTC_V_TOTAL; @@ -1655,14 +1975,44 @@ int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int crtc, unsign } else { /* No: Fake something reasonable which gives at least ok results. */ - vbl_start = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vdisplay; + vbl_start = mode->crtc_vdisplay; vbl_end = 0; } + /* Called from driver internal vblank counter query code? */ + if (flags & GET_DISTANCE_TO_VBLANKSTART) { + /* Caller wants distance from real vbl_start in *hpos */ + *hpos = *vpos - vbl_start; + } + + /* Fudge vblank to start a few scanlines earlier to handle the + * problem that vblank irqs fire a few scanlines before start + * of vblank. Some driver internal callers need the true vblank + * start to be used and signal this via the USE_REAL_VBLANKSTART flag. + * + * The cause of the "early" vblank irq is that the irq is triggered + * by the line buffer logic when the line buffer read position enters + * the vblank, whereas our crtc scanout position naturally lags the + * line buffer read position. + */ + if (!(flags & USE_REAL_VBLANKSTART)) + vbl_start -= rdev->mode_info.crtcs[pipe]->lb_vblank_lead_lines; + /* Test scanout position against vblank region. */ if ((*vpos < vbl_start) && (*vpos >= vbl_end)) in_vbl = false; + /* In vblank? */ + if (in_vbl) + ret |= DRM_SCANOUTPOS_IN_VBLANK; + + /* Called from driver internal vblank counter query code? */ + if (flags & GET_DISTANCE_TO_VBLANKSTART) { + /* Caller wants distance from fudged earlier vbl_start */ + *vpos -= vbl_start; + return ret; + } + /* Check if inside vblank area and apply corrective offsets: * vpos will then be >=0 in video scanout area, but negative * within vblank area, counting down the number of lines until @@ -1671,16 +2021,12 @@ int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int crtc, unsign /* Inside "upper part" of vblank area? Apply corrective offset if so: */ if (in_vbl && (*vpos >= vbl_start)) { - vtotal = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vtotal; + vtotal = mode->crtc_vtotal; *vpos = *vpos - vtotal; } /* Correct for shifted end of vbl at vbl_end. */ *vpos = *vpos - vbl_end; - /* In vblank? */ - if (in_vbl) - ret |= DRM_SCANOUTPOS_IN_VBLANK; - return ret; } diff --git a/sys/dev/pci/drm/radeon/radeon_dp_mst.c b/sys/dev/pci/drm/radeon/radeon_dp_mst.c new file mode 100644 index 00000000000..55f42ce53c6 --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_dp_mst.c @@ -0,0 +1,794 @@ + +#include +#include +#include + +#include "radeon.h" +#include "atom.h" +#include "ni_reg.h" + +static struct radeon_encoder *radeon_dp_create_fake_mst_encoder(struct radeon_connector *connector); + +static int radeon_atom_set_enc_offset(int id) +{ + static const int offsets[] = { EVERGREEN_CRTC0_REGISTER_OFFSET, + EVERGREEN_CRTC1_REGISTER_OFFSET, + EVERGREEN_CRTC2_REGISTER_OFFSET, + EVERGREEN_CRTC3_REGISTER_OFFSET, + EVERGREEN_CRTC4_REGISTER_OFFSET, + EVERGREEN_CRTC5_REGISTER_OFFSET, + 0x13830 - 0x7030 }; + + return offsets[id]; +} + +static int radeon_dp_mst_set_be_cntl(struct radeon_encoder *primary, + struct radeon_encoder_mst *mst_enc, + enum radeon_hpd_id hpd, bool enable) +{ + struct drm_device *dev = primary->base.dev; + struct radeon_device *rdev = dev->dev_private; + uint32_t reg; + int retries = 0; + uint32_t temp; + + reg = RREG32(NI_DIG_BE_CNTL + primary->offset); + + /* set MST mode */ + reg &= ~NI_DIG_FE_DIG_MODE(7); + reg |= NI_DIG_FE_DIG_MODE(NI_DIG_MODE_DP_MST); + + if (enable) + reg |= NI_DIG_FE_SOURCE_SELECT(1 << mst_enc->fe); + else + reg &= ~NI_DIG_FE_SOURCE_SELECT(1 << mst_enc->fe); + + reg |= NI_DIG_HPD_SELECT(hpd); + DRM_DEBUG_KMS("writing 0x%08x 0x%08x\n", NI_DIG_BE_CNTL + primary->offset, reg); + WREG32(NI_DIG_BE_CNTL + primary->offset, reg); + + if (enable) { + uint32_t offset = radeon_atom_set_enc_offset(mst_enc->fe); + + do { + temp = RREG32(NI_DIG_FE_CNTL + offset); + } while ((temp & NI_DIG_SYMCLK_FE_ON) && retries++ < 10000); + if (retries == 10000) + DRM_ERROR("timed out waiting for FE %d %d\n", primary->offset, mst_enc->fe); + } + return 0; +} + +static int radeon_dp_mst_set_stream_attrib(struct radeon_encoder *primary, + int stream_number, + int fe, + int slots) +{ + struct drm_device *dev = primary->base.dev; + struct radeon_device *rdev = dev->dev_private; + u32 temp, val; + int retries = 0; + int satreg, satidx; + + satreg = stream_number >> 1; + satidx = stream_number & 1; + + temp = RREG32(NI_DP_MSE_SAT0 + satreg + primary->offset); + + val = NI_DP_MSE_SAT_SLOT_COUNT0(slots) | NI_DP_MSE_SAT_SRC0(fe); + + val <<= (16 * satidx); + + temp &= ~(0xffff << (16 * satidx)); + + temp |= val; + + DRM_DEBUG_KMS("writing 0x%08x 0x%08x\n", NI_DP_MSE_SAT0 + satreg + primary->offset, temp); + WREG32(NI_DP_MSE_SAT0 + satreg + primary->offset, temp); + + WREG32(NI_DP_MSE_SAT_UPDATE + primary->offset, 1); + + do { + temp = RREG32(NI_DP_MSE_SAT_UPDATE + primary->offset); + } while ((temp & 0x1) && retries++ < 10000); + + if (retries == 10000) + DRM_ERROR("timed out waitin for SAT update %d\n", primary->offset); + + /* MTP 16 ? */ + return 0; +} + +static int radeon_dp_mst_update_stream_attribs(struct radeon_connector *mst_conn, + struct radeon_encoder *primary) +{ + struct drm_device *dev = mst_conn->base.dev; + struct stream_attribs new_attribs[6]; + int i; + int idx = 0; + struct radeon_connector *radeon_connector; + struct drm_connector *connector; + + memset(new_attribs, 0, sizeof(new_attribs)); + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + struct radeon_encoder *subenc; + struct radeon_encoder_mst *mst_enc; + + radeon_connector = to_radeon_connector(connector); + if (!radeon_connector->is_mst_connector) + continue; + + if (radeon_connector->mst_port != mst_conn) + continue; + + subenc = radeon_connector->mst_encoder; + mst_enc = subenc->enc_priv; + + if (!mst_enc->enc_active) + continue; + + new_attribs[idx].fe = mst_enc->fe; + new_attribs[idx].slots = drm_dp_mst_get_vcpi_slots(&mst_conn->mst_mgr, mst_enc->port); + idx++; + } + + for (i = 0; i < idx; i++) { + if (new_attribs[i].fe != mst_conn->cur_stream_attribs[i].fe || + new_attribs[i].slots != mst_conn->cur_stream_attribs[i].slots) { + radeon_dp_mst_set_stream_attrib(primary, i, new_attribs[i].fe, new_attribs[i].slots); + mst_conn->cur_stream_attribs[i].fe = new_attribs[i].fe; + mst_conn->cur_stream_attribs[i].slots = new_attribs[i].slots; + } + } + + for (i = idx; i < mst_conn->enabled_attribs; i++) { + radeon_dp_mst_set_stream_attrib(primary, i, 0, 0); + mst_conn->cur_stream_attribs[i].fe = 0; + mst_conn->cur_stream_attribs[i].slots = 0; + } + mst_conn->enabled_attribs = idx; + return 0; +} + +static int radeon_dp_mst_set_vcp_size(struct radeon_encoder *mst, uint32_t x, uint32_t y) +{ + struct drm_device *dev = mst->base.dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder_mst *mst_enc = mst->enc_priv; + uint32_t val, temp; + uint32_t offset = radeon_atom_set_enc_offset(mst_enc->fe); + int retries = 0; + + val = NI_DP_MSE_RATE_X(x) | NI_DP_MSE_RATE_Y(y); + + WREG32(NI_DP_MSE_RATE_CNTL + offset, val); + + do { + temp = RREG32(NI_DP_MSE_RATE_UPDATE + offset); + } while ((temp & 0x1) && (retries++ < 10000)); + + if (retries >= 10000) + DRM_ERROR("timed out wait for rate cntl %d\n", mst_enc->fe); + return 0; +} + +static int radeon_dp_mst_get_ddc_modes(struct drm_connector *connector) +{ + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + struct radeon_connector *master = radeon_connector->mst_port; + struct edid *edid; + int ret = 0; + + edid = drm_dp_mst_get_edid(connector, &master->mst_mgr, radeon_connector->port); + radeon_connector->edid = edid; + DRM_DEBUG_KMS("edid retrieved %p\n", edid); + if (radeon_connector->edid) { + drm_mode_connector_update_edid_property(&radeon_connector->base, radeon_connector->edid); + ret = drm_add_edid_modes(&radeon_connector->base, radeon_connector->edid); + drm_edid_to_eld(&radeon_connector->base, radeon_connector->edid); + return ret; + } + drm_mode_connector_update_edid_property(&radeon_connector->base, NULL); + + return ret; +} + +static int radeon_dp_mst_get_modes(struct drm_connector *connector) +{ + return radeon_dp_mst_get_ddc_modes(connector); +} + +static enum drm_mode_status +radeon_dp_mst_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + /* TODO - validate mode against available PBN for link */ + if (mode->clock < 10000) + return MODE_CLOCK_LOW; + + if (mode->flags & DRM_MODE_FLAG_DBLCLK) + return MODE_H_ILLEGAL; + + return MODE_OK; +} + +struct drm_encoder *radeon_mst_best_encoder(struct drm_connector *connector) +{ + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + + return &radeon_connector->mst_encoder->base; +} + +static const struct drm_connector_helper_funcs radeon_dp_mst_connector_helper_funcs = { + .get_modes = radeon_dp_mst_get_modes, + .mode_valid = radeon_dp_mst_mode_valid, + .best_encoder = radeon_mst_best_encoder, +}; + +static enum drm_connector_status +radeon_dp_mst_detect(struct drm_connector *connector, bool force) +{ + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + struct radeon_connector *master = radeon_connector->mst_port; + + return drm_dp_mst_detect_port(connector, &master->mst_mgr, radeon_connector->port); +} + +static void +radeon_dp_mst_connector_destroy(struct drm_connector *connector) +{ + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + struct radeon_encoder *radeon_encoder = radeon_connector->mst_encoder; + + drm_encoder_cleanup(&radeon_encoder->base); + kfree(radeon_encoder); + drm_connector_cleanup(connector); + kfree(radeon_connector); +} + +static int radeon_connector_dpms(struct drm_connector *connector, int mode) +{ + DRM_DEBUG_KMS("\n"); + return 0; +} + +static const struct drm_connector_funcs radeon_dp_mst_connector_funcs = { + .dpms = radeon_connector_dpms, + .detect = radeon_dp_mst_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = radeon_dp_mst_connector_destroy, +}; + +static struct drm_connector *radeon_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr, + struct drm_dp_mst_port *port, + const char *pathprop) +{ + struct radeon_connector *master = container_of(mgr, struct radeon_connector, mst_mgr); + struct drm_device *dev = master->base.dev; + struct radeon_connector *radeon_connector; + struct drm_connector *connector; + + radeon_connector = kzalloc(sizeof(*radeon_connector), GFP_KERNEL); + if (!radeon_connector) + return NULL; + + radeon_connector->is_mst_connector = true; + connector = &radeon_connector->base; + radeon_connector->port = port; + radeon_connector->mst_port = master; + DRM_DEBUG_KMS("\n"); + + drm_connector_init(dev, connector, &radeon_dp_mst_connector_funcs, DRM_MODE_CONNECTOR_DisplayPort); + drm_connector_helper_add(connector, &radeon_dp_mst_connector_helper_funcs); + radeon_connector->mst_encoder = radeon_dp_create_fake_mst_encoder(master); + + drm_object_attach_property(&connector->base, dev->mode_config.path_property, 0); + drm_object_attach_property(&connector->base, dev->mode_config.tile_property, 0); + drm_mode_connector_set_path_property(connector, pathprop); + + return connector; +} + +static void radeon_dp_register_mst_connector(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct radeon_device *rdev = dev->dev_private; + + drm_modeset_lock_all(dev); + radeon_fb_add_connector(rdev, connector); + drm_modeset_unlock_all(dev); + + drm_connector_register(connector); +} + +static void radeon_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr, + struct drm_connector *connector) +{ + struct radeon_connector *master = container_of(mgr, struct radeon_connector, mst_mgr); + struct drm_device *dev = master->base.dev; + struct radeon_device *rdev = dev->dev_private; + + drm_connector_unregister(connector); + /* need to nuke the connector */ + drm_modeset_lock_all(dev); + /* dpms off */ + radeon_fb_remove_connector(rdev, connector); + + drm_connector_cleanup(connector); + drm_modeset_unlock_all(dev); + + kfree(connector); + DRM_DEBUG_KMS("\n"); +} + +static void radeon_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr) +{ + struct radeon_connector *master = container_of(mgr, struct radeon_connector, mst_mgr); + struct drm_device *dev = master->base.dev; + + drm_kms_helper_hotplug_event(dev); +} + +struct drm_dp_mst_topology_cbs mst_cbs = { + .add_connector = radeon_dp_add_mst_connector, + .register_connector = radeon_dp_register_mst_connector, + .destroy_connector = radeon_dp_destroy_mst_connector, + .hotplug = radeon_dp_mst_hotplug, +}; + +struct radeon_connector *radeon_mst_find_connector(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct drm_connector *connector; + + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + if (!connector->encoder) + continue; + if (!radeon_connector->is_mst_connector) + continue; + + DRM_DEBUG_KMS("checking %p vs %p\n", connector->encoder, encoder); + if (connector->encoder == encoder) + return radeon_connector; + } + return NULL; +} + +void radeon_dp_mst_prepare_pll(struct drm_crtc *crtc, struct drm_display_mode *mode) +{ + struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(radeon_crtc->encoder); + struct radeon_encoder_mst *mst_enc = radeon_encoder->enc_priv; + struct radeon_connector *radeon_connector = radeon_mst_find_connector(&radeon_encoder->base); + int dp_clock; + struct radeon_connector_atom_dig *dig_connector = mst_enc->connector->con_priv; + + if (radeon_connector) { + radeon_connector->pixelclock_for_modeset = mode->clock; + if (radeon_connector->base.display_info.bpc) + radeon_crtc->bpc = radeon_connector->base.display_info.bpc; + else + radeon_crtc->bpc = 8; + } + + DRM_DEBUG_KMS("dp_clock %p %d\n", dig_connector, dig_connector->dp_clock); + dp_clock = dig_connector->dp_clock; + radeon_crtc->ss_enabled = + radeon_atombios_get_asic_ss_info(rdev, &radeon_crtc->ss, + ASIC_INTERNAL_SS_ON_DP, + dp_clock); +} + +static void +radeon_mst_encoder_dpms(struct drm_encoder *encoder, int mode) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder, *primary; + struct radeon_encoder_mst *mst_enc; + struct radeon_encoder_atom_dig *dig_enc; + struct radeon_connector *radeon_connector; + struct drm_crtc *crtc; + struct radeon_crtc *radeon_crtc; + int ret, slots; + + if (!ASIC_IS_DCE5(rdev)) { + DRM_ERROR("got mst dpms on non-DCE5\n"); + return; + } + + radeon_connector = radeon_mst_find_connector(encoder); + if (!radeon_connector) + return; + + radeon_encoder = to_radeon_encoder(encoder); + + mst_enc = radeon_encoder->enc_priv; + + primary = mst_enc->primary; + + dig_enc = primary->enc_priv; + + crtc = encoder->crtc; + DRM_DEBUG_KMS("got connector %d\n", dig_enc->active_mst_links); + + switch (mode) { + case DRM_MODE_DPMS_ON: + dig_enc->active_mst_links++; + + radeon_crtc = to_radeon_crtc(crtc); + + if (dig_enc->active_mst_links == 1) { + mst_enc->fe = dig_enc->dig_encoder; + mst_enc->fe_from_be = true; + atombios_set_mst_encoder_crtc_source(encoder, mst_enc->fe); + + atombios_dig_encoder_setup(&primary->base, ATOM_ENCODER_CMD_SETUP, 0); + atombios_dig_transmitter_setup2(&primary->base, ATOM_TRANSMITTER_ACTION_ENABLE, + 0, 0, dig_enc->dig_encoder); + + if (radeon_dp_needs_link_train(mst_enc->connector) || + dig_enc->active_mst_links == 1) { + radeon_dp_link_train(&primary->base, &mst_enc->connector->base); + } + + } else { + mst_enc->fe = radeon_atom_pick_dig_encoder(encoder, radeon_crtc->crtc_id); + if (mst_enc->fe == -1) + DRM_ERROR("failed to get frontend for dig encoder\n"); + mst_enc->fe_from_be = false; + atombios_set_mst_encoder_crtc_source(encoder, mst_enc->fe); + } + + DRM_DEBUG_KMS("dig encoder is %d %d %d\n", dig_enc->dig_encoder, + dig_enc->linkb, radeon_crtc->crtc_id); + + ret = drm_dp_mst_allocate_vcpi(&radeon_connector->mst_port->mst_mgr, + radeon_connector->port, + mst_enc->pbn, &slots); + ret = drm_dp_update_payload_part1(&radeon_connector->mst_port->mst_mgr); + + radeon_dp_mst_set_be_cntl(primary, mst_enc, + radeon_connector->mst_port->hpd.hpd, true); + + mst_enc->enc_active = true; + radeon_dp_mst_update_stream_attribs(radeon_connector->mst_port, primary); + radeon_dp_mst_set_vcp_size(radeon_encoder, slots, 0); + + atombios_dig_encoder_setup2(&primary->base, ATOM_ENCODER_CMD_DP_VIDEO_ON, 0, + mst_enc->fe); + ret = drm_dp_check_act_status(&radeon_connector->mst_port->mst_mgr); + + ret = drm_dp_update_payload_part2(&radeon_connector->mst_port->mst_mgr); + + break; + case DRM_MODE_DPMS_STANDBY: + case DRM_MODE_DPMS_SUSPEND: + case DRM_MODE_DPMS_OFF: + DRM_ERROR("DPMS OFF %d\n", dig_enc->active_mst_links); + + if (!mst_enc->enc_active) + return; + + drm_dp_mst_reset_vcpi_slots(&radeon_connector->mst_port->mst_mgr, mst_enc->port); + ret = drm_dp_update_payload_part1(&radeon_connector->mst_port->mst_mgr); + + drm_dp_check_act_status(&radeon_connector->mst_port->mst_mgr); + /* and this can also fail */ + drm_dp_update_payload_part2(&radeon_connector->mst_port->mst_mgr); + + drm_dp_mst_deallocate_vcpi(&radeon_connector->mst_port->mst_mgr, mst_enc->port); + + mst_enc->enc_active = false; + radeon_dp_mst_update_stream_attribs(radeon_connector->mst_port, primary); + + radeon_dp_mst_set_be_cntl(primary, mst_enc, + radeon_connector->mst_port->hpd.hpd, false); + atombios_dig_encoder_setup2(&primary->base, ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0, + mst_enc->fe); + + if (!mst_enc->fe_from_be) + radeon_atom_release_dig_encoder(rdev, mst_enc->fe); + + mst_enc->fe_from_be = false; + dig_enc->active_mst_links--; + if (dig_enc->active_mst_links == 0) { + /* drop link */ + } + + break; + } + +} + +static bool radeon_mst_mode_fixup(struct drm_encoder *encoder, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct radeon_encoder_mst *mst_enc; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + int bpp = 24; + + mst_enc = radeon_encoder->enc_priv; + + mst_enc->pbn = drm_dp_calc_pbn_mode(adjusted_mode->clock, bpp); + + mst_enc->primary->active_device = mst_enc->primary->devices & mst_enc->connector->devices; + DRM_DEBUG_KMS("setting active device to %08x from %08x %08x for encoder %d\n", + mst_enc->primary->active_device, mst_enc->primary->devices, + mst_enc->connector->devices, mst_enc->primary->base.encoder_type); + + + drm_mode_set_crtcinfo(adjusted_mode, 0); + { + struct radeon_connector_atom_dig *dig_connector; + dig_connector = mst_enc->connector->con_priv; + dig_connector->dp_lane_count = drm_dp_max_lane_count(dig_connector->dpcd); + dig_connector->dp_clock = drm_dp_max_link_rate(dig_connector->dpcd); + DRM_DEBUG_KMS("dig clock %p %d %d\n", dig_connector, + dig_connector->dp_lane_count, dig_connector->dp_clock); + } + return true; +} + +static void radeon_mst_encoder_prepare(struct drm_encoder *encoder) +{ + struct radeon_connector *radeon_connector; + struct radeon_encoder *radeon_encoder, *primary; + struct radeon_encoder_mst *mst_enc; + struct radeon_encoder_atom_dig *dig_enc; + + radeon_connector = radeon_mst_find_connector(encoder); + if (!radeon_connector) { + DRM_DEBUG_KMS("failed to find connector %p\n", encoder); + return; + } + radeon_encoder = to_radeon_encoder(encoder); + + radeon_mst_encoder_dpms(encoder, DRM_MODE_DPMS_OFF); + + mst_enc = radeon_encoder->enc_priv; + + primary = mst_enc->primary; + + dig_enc = primary->enc_priv; + + mst_enc->port = radeon_connector->port; + + if (dig_enc->dig_encoder == -1) { + dig_enc->dig_encoder = radeon_atom_pick_dig_encoder(&primary->base, -1); + primary->offset = radeon_atom_set_enc_offset(dig_enc->dig_encoder); + atombios_set_mst_encoder_crtc_source(encoder, dig_enc->dig_encoder); + + + } + DRM_DEBUG_KMS("%d %d\n", dig_enc->dig_encoder, primary->offset); +} + +static void +radeon_mst_encoder_mode_set(struct drm_encoder *encoder, + struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + DRM_DEBUG_KMS("\n"); +} + +static void radeon_mst_encoder_commit(struct drm_encoder *encoder) +{ + radeon_mst_encoder_dpms(encoder, DRM_MODE_DPMS_ON); + DRM_DEBUG_KMS("\n"); +} + +static const struct drm_encoder_helper_funcs radeon_mst_helper_funcs = { + .dpms = radeon_mst_encoder_dpms, + .mode_fixup = radeon_mst_mode_fixup, + .prepare = radeon_mst_encoder_prepare, + .mode_set = radeon_mst_encoder_mode_set, + .commit = radeon_mst_encoder_commit, +}; + +void radeon_dp_mst_encoder_destroy(struct drm_encoder *encoder) +{ + drm_encoder_cleanup(encoder); + kfree(encoder); +} + +static const struct drm_encoder_funcs radeon_dp_mst_enc_funcs = { + .destroy = radeon_dp_mst_encoder_destroy, +}; + +static struct radeon_encoder * +radeon_dp_create_fake_mst_encoder(struct radeon_connector *connector) +{ + struct drm_device *dev = connector->base.dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder; + struct radeon_encoder_mst *mst_enc; + struct drm_encoder *encoder; + const struct drm_connector_helper_funcs *connector_funcs = connector->base.helper_private; + struct drm_encoder *enc_master = connector_funcs->best_encoder(&connector->base); + + DRM_DEBUG_KMS("enc master is %p\n", enc_master); + radeon_encoder = kzalloc(sizeof(*radeon_encoder), GFP_KERNEL); + if (!radeon_encoder) + return NULL; + + radeon_encoder->enc_priv = kzalloc(sizeof(*mst_enc), GFP_KERNEL); + if (!radeon_encoder->enc_priv) { + kfree(radeon_encoder); + return NULL; + } + encoder = &radeon_encoder->base; + switch (rdev->num_crtc) { + case 1: + encoder->possible_crtcs = 0x1; + break; + case 2: + default: + encoder->possible_crtcs = 0x3; + break; + case 4: + encoder->possible_crtcs = 0xf; + break; + case 6: + encoder->possible_crtcs = 0x3f; + break; + } + + drm_encoder_init(dev, &radeon_encoder->base, &radeon_dp_mst_enc_funcs, + DRM_MODE_ENCODER_DPMST); + drm_encoder_helper_add(encoder, &radeon_mst_helper_funcs); + + mst_enc = radeon_encoder->enc_priv; + mst_enc->connector = connector; + mst_enc->primary = to_radeon_encoder(enc_master); + radeon_encoder->is_mst_encoder = true; + return radeon_encoder; +} + +int +radeon_dp_mst_init(struct radeon_connector *radeon_connector) +{ + struct drm_device *dev = radeon_connector->base.dev; + + if (!radeon_connector->ddc_bus->has_aux) + return 0; + + radeon_connector->mst_mgr.cbs = &mst_cbs; + return drm_dp_mst_topology_mgr_init(&radeon_connector->mst_mgr, dev->dev, + &radeon_connector->ddc_bus->aux, 16, 6, + radeon_connector->base.base.id); +} + +int +radeon_dp_mst_probe(struct radeon_connector *radeon_connector) +{ + struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv; + struct drm_device *dev = radeon_connector->base.dev; + struct radeon_device *rdev = dev->dev_private; + int ret; + u8 msg[1]; + + if (!radeon_mst) + return 0; + + if (!ASIC_IS_DCE5(rdev)) + return 0; + + if (dig_connector->dpcd[DP_DPCD_REV] < 0x12) + return 0; + + ret = drm_dp_dpcd_read(&radeon_connector->ddc_bus->aux, DP_MSTM_CAP, msg, + 1); + if (ret) { + if (msg[0] & DP_MST_CAP) { + DRM_DEBUG_KMS("Sink is MST capable\n"); + dig_connector->is_mst = true; + } else { + DRM_DEBUG_KMS("Sink is not MST capable\n"); + dig_connector->is_mst = false; + } + + } + drm_dp_mst_topology_mgr_set_mst(&radeon_connector->mst_mgr, + dig_connector->is_mst); + return dig_connector->is_mst; +} + +int +radeon_dp_mst_check_status(struct radeon_connector *radeon_connector) +{ + struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv; + int retry; + + if (dig_connector->is_mst) { + u8 esi[16] = { 0 }; + int dret; + int ret = 0; + bool handled; + + dret = drm_dp_dpcd_read(&radeon_connector->ddc_bus->aux, + DP_SINK_COUNT_ESI, esi, 8); +go_again: + if (dret == 8) { + DRM_DEBUG_KMS("got esi %02x %02x %02x\n", esi[0], esi[1], esi[2]); + ret = drm_dp_mst_hpd_irq(&radeon_connector->mst_mgr, esi, &handled); + + if (handled) { + for (retry = 0; retry < 3; retry++) { + int wret; + wret = drm_dp_dpcd_write(&radeon_connector->ddc_bus->aux, + DP_SINK_COUNT_ESI + 1, &esi[1], 3); + if (wret == 3) + break; + } + + dret = drm_dp_dpcd_read(&radeon_connector->ddc_bus->aux, + DP_SINK_COUNT_ESI, esi, 8); + if (dret == 8) { + DRM_DEBUG_KMS("got esi2 %02x %02x %02x\n", esi[0], esi[1], esi[2]); + goto go_again; + } + } else + ret = 0; + + return ret; + } else { + DRM_DEBUG_KMS("failed to get ESI - device may have failed %d\n", ret); + dig_connector->is_mst = false; + drm_dp_mst_topology_mgr_set_mst(&radeon_connector->mst_mgr, + dig_connector->is_mst); + /* send a hotplug event */ + } + } + return -EINVAL; +} + +#if defined(CONFIG_DEBUG_FS) + +static int radeon_debugfs_mst_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *)m->private; + struct drm_device *dev = node->minor->dev; + struct drm_connector *connector; + struct radeon_connector *radeon_connector; + struct radeon_connector_atom_dig *dig_connector; + int i; + + drm_modeset_lock_all(dev); + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort) + continue; + + radeon_connector = to_radeon_connector(connector); + dig_connector = radeon_connector->con_priv; + if (radeon_connector->is_mst_connector) + continue; + if (!dig_connector->is_mst) + continue; + drm_dp_mst_dump_topology(m, &radeon_connector->mst_mgr); + + for (i = 0; i < radeon_connector->enabled_attribs; i++) + seq_printf(m, "attrib %d: %d %d\n", i, + radeon_connector->cur_stream_attribs[i].fe, + radeon_connector->cur_stream_attribs[i].slots); + } + drm_modeset_unlock_all(dev); + return 0; +} + +static struct drm_info_list radeon_debugfs_mst_list[] = { + {"radeon_mst_info", &radeon_debugfs_mst_info, 0, NULL}, +}; +#endif + +int radeon_mst_debugfs_init(struct radeon_device *rdev) +{ +#if defined(CONFIG_DEBUG_FS) + return radeon_debugfs_add_files(rdev, radeon_debugfs_mst_list, 1); +#endif + return 0; +} diff --git a/sys/dev/pci/drm/radeon/radeon_drv.c b/sys/dev/pci/drm/radeon/radeon_drv.c new file mode 100644 index 00000000000..3939bc06e75 --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_drv.c @@ -0,0 +1,718 @@ +/** + * \file radeon_drv.c + * ATI Radeon driver + * + * \author Gareth Hughes + */ + +/* + * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include "radeon_drv.h" +#include "radeon.h" + +#include + +#include +#include "radeon_kfd.h" + +/* + * KMS wrapper. + * - 2.0.0 - initial interface + * - 2.1.0 - add square tiling interface + * - 2.2.0 - add r6xx/r7xx const buffer support + * - 2.3.0 - add MSPOS + 3D texture + r500 VAP regs + * - 2.4.0 - add crtc id query + * - 2.5.0 - add get accel 2 to work around ddx breakage for evergreen + * - 2.6.0 - add tiling config query (r6xx+), add initial HiZ support (r300->r500) + * 2.7.0 - fixups for r600 2D tiling support. (no external ABI change), add eg dyn gpr regs + * 2.8.0 - pageflip support, r500 US_FORMAT regs. r500 ARGB2101010 colorbuf, r300->r500 CMASK, clock crystal query + * 2.9.0 - r600 tiling (s3tc,rgtc) working, SET_PREDICATION packet 3 on r600 + eg, backend query + * 2.10.0 - fusion 2D tiling + * 2.11.0 - backend map, initial compute support for the CS checker + * 2.12.0 - RADEON_CS_KEEP_TILING_FLAGS + * 2.13.0 - virtual memory support, streamout + * 2.14.0 - add evergreen tiling informations + * 2.15.0 - add max_pipes query + * 2.16.0 - fix evergreen 2D tiled surface calculation + * 2.17.0 - add STRMOUT_BASE_UPDATE for r7xx + * 2.18.0 - r600-eg: allow "invalid" DB formats + * 2.19.0 - r600-eg: MSAA textures + * 2.20.0 - r600-si: RADEON_INFO_TIMESTAMP query + * 2.21.0 - r600-r700: FMASK and CMASK + * 2.22.0 - r600 only: RESOLVE_BOX allowed + * 2.23.0 - allow STRMOUT_BASE_UPDATE on RS780 and RS880 + * 2.24.0 - eg only: allow MIP_ADDRESS=0 for MSAA textures + * 2.25.0 - eg+: new info request for num SE and num SH + * 2.26.0 - r600-eg: fix htile size computation + * 2.27.0 - r600-SI: Add CS ioctl support for async DMA + * 2.28.0 - r600-eg: Add MEM_WRITE packet support + * 2.29.0 - R500 FP16 color clear registers + * 2.30.0 - fix for FMASK texturing + * 2.31.0 - Add fastfb support for rs690 + * 2.32.0 - new info request for rings working + * 2.33.0 - Add SI tiling mode array query + * 2.34.0 - Add CIK tiling mode array query + * 2.35.0 - Add CIK macrotile mode array query + * 2.36.0 - Fix CIK DCE tiling setup + * 2.37.0 - allow GS ring setup on r6xx/r7xx + * 2.38.0 - RADEON_GEM_OP (GET_INITIAL_DOMAIN, SET_INITIAL_DOMAIN), + * CIK: 1D and linear tiling modes contain valid PIPE_CONFIG + * 2.39.0 - Add INFO query for number of active CUs + * 2.40.0 - Add RADEON_GEM_GTT_WC/UC, flush HDP cache before submitting + * CS to GPU on >= r600 + * 2.41.0 - evergreen/cayman: Add SET_BASE/DRAW_INDIRECT command parsing support + * 2.42.0 - Add VCE/VUI (Video Usability Information) support + * 2.43.0 - RADEON_INFO_GPU_RESET_COUNTER + */ +#define KMS_DRIVER_MAJOR 2 +#define KMS_DRIVER_MINOR 43 +#define KMS_DRIVER_PATCHLEVEL 0 +int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags); +int radeon_driver_unload_kms(struct drm_device *dev); +void radeon_driver_lastclose_kms(struct drm_device *dev); +int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv); +void radeon_driver_postclose_kms(struct drm_device *dev, + struct drm_file *file_priv); +void radeon_driver_preclose_kms(struct drm_device *dev, + struct drm_file *file_priv); +int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon); +int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon); +u32 radeon_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe); +int radeon_enable_vblank_kms(struct drm_device *dev, unsigned int pipe); +void radeon_disable_vblank_kms(struct drm_device *dev, unsigned int pipe); +int radeon_get_vblank_timestamp_kms(struct drm_device *dev, unsigned int pipe, + int *max_error, + struct timeval *vblank_time, + unsigned flags); +void radeon_driver_irq_preinstall_kms(struct drm_device *dev); +int radeon_driver_irq_postinstall_kms(struct drm_device *dev); +void radeon_driver_irq_uninstall_kms(struct drm_device *dev); +irqreturn_t radeon_driver_irq_handler_kms(int irq, void *arg); +void radeon_gem_object_free(struct drm_gem_object *obj); +int radeon_gem_object_open(struct drm_gem_object *obj, + struct drm_file *file_priv); +void radeon_gem_object_close(struct drm_gem_object *obj, + struct drm_file *file_priv); +struct dma_buf *radeon_gem_prime_export(struct drm_device *dev, + struct drm_gem_object *gobj, + int flags); +extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int crtc, + unsigned int flags, int *vpos, int *hpos, + ktime_t *stime, ktime_t *etime, + const struct drm_display_mode *mode); +extern bool radeon_is_px(struct drm_device *dev); +extern const struct drm_ioctl_desc radeon_ioctls_kms[]; +extern int radeon_max_kms_ioctl; +extern struct uvm_object *radeon_mmap(struct drm_device *, voff_t, vsize_t); +int radeon_mode_dumb_mmap(struct drm_file *filp, + struct drm_device *dev, + uint32_t handle, uint64_t *offset_p); +int radeon_mode_dumb_create(struct drm_file *file_priv, + struct drm_device *dev, + struct drm_mode_create_dumb *args); +#ifdef notyet +struct sg_table *radeon_gem_prime_get_sg_table(struct drm_gem_object *obj); +struct drm_gem_object *radeon_gem_prime_import_sg_table(struct drm_device *dev, + struct dma_buf_attachment *, + struct sg_table *sg); +int radeon_gem_prime_pin(struct drm_gem_object *obj); +void radeon_gem_prime_unpin(struct drm_gem_object *obj); +struct reservation_object *radeon_gem_prime_res_obj(struct drm_gem_object *); +void *radeon_gem_prime_vmap(struct drm_gem_object *obj); +void radeon_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr); +#endif +extern long radeon_kms_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg); + +#if defined(CONFIG_DEBUG_FS) +int radeon_debugfs_init(struct drm_minor *minor); +void radeon_debugfs_cleanup(struct drm_minor *minor); +#endif + +/* atpx handler */ +#if defined(CONFIG_VGA_SWITCHEROO) +void radeon_register_atpx_handler(void); +void radeon_unregister_atpx_handler(void); +#else +#ifdef notyet +static inline void radeon_register_atpx_handler(void) {} +static inline void radeon_unregister_atpx_handler(void) {} +#endif +#endif + +int radeon_no_wb; +int radeon_modeset = -1; +int radeon_dynclks = -1; +int radeon_r4xx_atom = 0; +int radeon_agpmode = 0; +int radeon_vram_limit = 0; +int radeon_gart_size = -1; /* auto */ +int radeon_benchmarking = 0; +int radeon_testing = 0; +int radeon_connector_table = 0; +int radeon_tv = 1; +int radeon_audio = -1; +int radeon_disp_priority = 0; +int radeon_hw_i2c = 0; +int radeon_pcie_gen2 = -1; +int radeon_msi = -1; +int radeon_lockup_timeout = 10000; +int radeon_fastfb = 0; +int radeon_dpm = -1; +int radeon_aspm = -1; +int radeon_runtime_pm = -1; +int radeon_hard_reset = 0; +int radeon_vm_size = 8; +int radeon_vm_block_size = -1; +int radeon_deep_color = 0; +int radeon_use_pflipirq = 2; +int radeon_bapm = -1; +int radeon_backlight = -1; +int radeon_auxch = -1; +int radeon_mst = 0; + +MODULE_PARM_DESC(no_wb, "Disable AGP writeback for scratch registers"); +module_param_named(no_wb, radeon_no_wb, int, 0444); + +MODULE_PARM_DESC(modeset, "Disable/Enable modesetting"); +module_param_named(modeset, radeon_modeset, int, 0400); + +MODULE_PARM_DESC(dynclks, "Disable/Enable dynamic clocks"); +module_param_named(dynclks, radeon_dynclks, int, 0444); + +MODULE_PARM_DESC(r4xx_atom, "Enable ATOMBIOS modesetting for R4xx"); +module_param_named(r4xx_atom, radeon_r4xx_atom, int, 0444); + +MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes"); +module_param_named(vramlimit, radeon_vram_limit, int, 0600); + +MODULE_PARM_DESC(agpmode, "AGP Mode (-1 == PCI)"); +module_param_named(agpmode, radeon_agpmode, int, 0444); + +MODULE_PARM_DESC(gartsize, "Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)"); +module_param_named(gartsize, radeon_gart_size, int, 0600); + +MODULE_PARM_DESC(benchmark, "Run benchmark"); +module_param_named(benchmark, radeon_benchmarking, int, 0444); + +MODULE_PARM_DESC(test, "Run tests"); +module_param_named(test, radeon_testing, int, 0444); + +MODULE_PARM_DESC(connector_table, "Force connector table"); +module_param_named(connector_table, radeon_connector_table, int, 0444); + +MODULE_PARM_DESC(tv, "TV enable (0 = disable)"); +module_param_named(tv, radeon_tv, int, 0444); + +MODULE_PARM_DESC(audio, "Audio enable (-1 = auto, 0 = disable, 1 = enable)"); +module_param_named(audio, radeon_audio, int, 0444); + +MODULE_PARM_DESC(disp_priority, "Display Priority (0 = auto, 1 = normal, 2 = high)"); +module_param_named(disp_priority, radeon_disp_priority, int, 0444); + +MODULE_PARM_DESC(hw_i2c, "hw i2c engine enable (0 = disable)"); +module_param_named(hw_i2c, radeon_hw_i2c, int, 0444); + +MODULE_PARM_DESC(pcie_gen2, "PCIE Gen2 mode (-1 = auto, 0 = disable, 1 = enable)"); +module_param_named(pcie_gen2, radeon_pcie_gen2, int, 0444); + +MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)"); +module_param_named(msi, radeon_msi, int, 0444); + +MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (default 10000 = 10 seconds, 0 = disable)"); +module_param_named(lockup_timeout, radeon_lockup_timeout, int, 0444); + +MODULE_PARM_DESC(fastfb, "Direct FB access for IGP chips (0 = disable, 1 = enable)"); +module_param_named(fastfb, radeon_fastfb, int, 0444); + +MODULE_PARM_DESC(dpm, "DPM support (1 = enable, 0 = disable, -1 = auto)"); +module_param_named(dpm, radeon_dpm, int, 0444); + +MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)"); +module_param_named(aspm, radeon_aspm, int, 0444); + +MODULE_PARM_DESC(runpm, "PX runtime pm (1 = force enable, 0 = disable, -1 = PX only default)"); +module_param_named(runpm, radeon_runtime_pm, int, 0444); + +MODULE_PARM_DESC(hard_reset, "PCI config reset (1 = force enable, 0 = disable (default))"); +module_param_named(hard_reset, radeon_hard_reset, int, 0444); + +MODULE_PARM_DESC(vm_size, "VM address space size in gigabytes (default 4GB)"); +module_param_named(vm_size, radeon_vm_size, int, 0444); + +MODULE_PARM_DESC(vm_block_size, "VM page table size in bits (default depending on vm_size)"); +module_param_named(vm_block_size, radeon_vm_block_size, int, 0444); + +MODULE_PARM_DESC(deep_color, "Deep Color support (1 = enable, 0 = disable (default))"); +module_param_named(deep_color, radeon_deep_color, int, 0444); + +MODULE_PARM_DESC(use_pflipirq, "Pflip irqs for pageflip completion (0 = disable, 1 = as fallback, 2 = exclusive (default))"); +module_param_named(use_pflipirq, radeon_use_pflipirq, int, 0444); + +MODULE_PARM_DESC(bapm, "BAPM support (1 = enable, 0 = disable, -1 = auto)"); +module_param_named(bapm, radeon_bapm, int, 0444); + +MODULE_PARM_DESC(backlight, "backlight support (1 = enable, 0 = disable, -1 = auto)"); +module_param_named(backlight, radeon_backlight, int, 0444); + +MODULE_PARM_DESC(auxch, "Use native auxch experimental support (1 = enable, 0 = disable, -1 = auto)"); +module_param_named(auxch, radeon_auxch, int, 0444); + +MODULE_PARM_DESC(mst, "DisplayPort MST experimental support (1 = enable, 0 = disable)"); +module_param_named(mst, radeon_mst, int, 0444); + +const struct drm_pcidev radeondrm_pciidlist[] = { + radeon_PCI_IDS +}; + +MODULE_DEVICE_TABLE(pci, pciidlist); + +#ifdef CONFIG_DRM_RADEON_UMS + +static int radeon_suspend(struct drm_device *dev, pm_message_t state) +{ + drm_radeon_private_t *dev_priv = dev->dev_private; + + if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) + return 0; + + /* Disable *all* interrupts */ + if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) + RADEON_WRITE(R500_DxMODE_INT_MASK, 0); + RADEON_WRITE(RADEON_GEN_INT_CNTL, 0); + return 0; +} + +static int radeon_resume(struct drm_device *dev) +{ + drm_radeon_private_t *dev_priv = dev->dev_private; + + if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) + return 0; + + /* Restore interrupt registers */ + if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS600) + RADEON_WRITE(R500_DxMODE_INT_MASK, dev_priv->r500_disp_irq_reg); + RADEON_WRITE(RADEON_GEN_INT_CNTL, dev_priv->irq_enable_reg); + return 0; +} + + +static const struct file_operations radeon_driver_old_fops = { + .owner = THIS_MODULE, + .open = drm_open, + .release = drm_release, + .unlocked_ioctl = drm_ioctl, + .mmap = drm_legacy_mmap, + .poll = drm_poll, + .read = drm_read, +#ifdef CONFIG_COMPAT + .compat_ioctl = radeon_compat_ioctl, +#endif + .llseek = noop_llseek, +}; + +static struct drm_driver driver_old = { + .driver_features = + DRIVER_USE_AGP | DRIVER_PCI_DMA | DRIVER_SG | + DRIVER_HAVE_IRQ | DRIVER_HAVE_DMA | DRIVER_IRQ_SHARED, + .dev_priv_size = sizeof(drm_radeon_buf_priv_t), + .load = radeon_driver_load, + .firstopen = radeon_driver_firstopen, + .open = radeon_driver_open, + .preclose = radeon_driver_preclose, + .postclose = radeon_driver_postclose, + .lastclose = radeon_driver_lastclose, + .set_busid = drm_pci_set_busid, + .unload = radeon_driver_unload, + .suspend = radeon_suspend, + .resume = radeon_resume, + .get_vblank_counter = radeon_get_vblank_counter, + .enable_vblank = radeon_enable_vblank, + .disable_vblank = radeon_disable_vblank, + .master_create = radeon_master_create, + .master_destroy = radeon_master_destroy, + .irq_preinstall = radeon_driver_irq_preinstall, + .irq_postinstall = radeon_driver_irq_postinstall, + .irq_uninstall = radeon_driver_irq_uninstall, + .irq_handler = radeon_driver_irq_handler, + .ioctls = radeon_ioctls, + .dma_ioctl = radeon_cp_buffers, + .fops = &radeon_driver_old_fops, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .date = DRIVER_DATE, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, + .patchlevel = DRIVER_PATCHLEVEL, +}; + +#endif + +#ifdef notyet +static struct drm_driver kms_driver; +#endif + +#ifdef notyet +static int radeon_kick_out_firmware_fb(struct pci_dev *pdev) +{ + struct apertures_struct *ap; + bool primary = false; + + ap = alloc_apertures(1); + if (!ap) + return -ENOMEM; + + ap->ranges[0].base = pci_resource_start(pdev, 0); + ap->ranges[0].size = pci_resource_len(pdev, 0); + +#ifdef CONFIG_X86 + primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW; +#endif + remove_conflicting_framebuffers(ap, "radeondrmfb", primary); + kfree(ap); + + return 0; +} +#endif + +#ifdef __linux__ +static int radeon_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + int ret; + + /* Get rid of things like offb */ + ret = radeon_kick_out_firmware_fb(pdev); + if (ret) + return ret; + + return drm_get_pci_dev(pdev, ent, &kms_driver); +} + +static void +radeon_pci_remove(struct pci_dev *pdev) +{ + struct drm_device *dev = pci_get_drvdata(pdev); + + drm_put_dev(dev); +} + +static int radeon_pmops_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + return radeon_suspend_kms(drm_dev, true, true); +} + +static int radeon_pmops_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + return radeon_resume_kms(drm_dev, true, true); +} + +static int radeon_pmops_freeze(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + return radeon_suspend_kms(drm_dev, false, true); +} + +static int radeon_pmops_thaw(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + return radeon_resume_kms(drm_dev, false, true); +} + +static int radeon_pmops_runtime_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + int ret; + + if (!radeon_is_px(drm_dev)) { + pm_runtime_forbid(dev); + return -EBUSY; + } + + drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; + drm_kms_helper_poll_disable(drm_dev); + vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_OFF); + + ret = radeon_suspend_kms(drm_dev, false, false); + pci_save_state(pdev); + pci_disable_device(pdev); + pci_ignore_hotplug(pdev); + pci_set_power_state(pdev, PCI_D3cold); + drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF; + + return 0; +} + +static int radeon_pmops_runtime_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + int ret; + + if (!radeon_is_px(drm_dev)) + return -EINVAL; + + drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; + + pci_set_power_state(pdev, PCI_D0); + pci_restore_state(pdev); + ret = pci_enable_device(pdev); + if (ret) + return ret; + pci_set_master(pdev); + + ret = radeon_resume_kms(drm_dev, false, false); + drm_kms_helper_poll_enable(drm_dev); + vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_ON); + drm_dev->switch_power_state = DRM_SWITCH_POWER_ON; + return 0; +} + +static int radeon_pmops_runtime_idle(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + struct drm_crtc *crtc; + + if (!radeon_is_px(drm_dev)) { + pm_runtime_forbid(dev); + return -EBUSY; + } + + list_for_each_entry(crtc, &drm_dev->mode_config.crtc_list, head) { + if (crtc->enabled) { + DRM_DEBUG_DRIVER("failing to power off - crtc active\n"); + return -EBUSY; + } + } + + pm_runtime_mark_last_busy(dev); + pm_runtime_autosuspend(dev); + /* we don't want the main rpm_idle to call suspend - we want to autosuspend */ + return 1; +} + +long radeon_drm_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg) +{ + struct drm_file *file_priv = filp->private_data; + struct drm_device *dev; + long ret; + dev = file_priv->minor->dev; + ret = pm_runtime_get_sync(dev->dev); + if (ret < 0) + return ret; + + ret = drm_ioctl(filp, cmd, arg); + + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + return ret; +} + +static const struct dev_pm_ops radeon_pm_ops = { + .suspend = radeon_pmops_suspend, + .resume = radeon_pmops_resume, + .freeze = radeon_pmops_freeze, + .thaw = radeon_pmops_thaw, + .poweroff = radeon_pmops_freeze, + .restore = radeon_pmops_resume, + .runtime_suspend = radeon_pmops_runtime_suspend, + .runtime_resume = radeon_pmops_runtime_resume, + .runtime_idle = radeon_pmops_runtime_idle, +}; + +static const struct file_operations radeon_driver_kms_fops = { + .owner = THIS_MODULE, + .open = drm_open, + .release = drm_release, + .unlocked_ioctl = radeon_drm_ioctl, + .mmap = radeon_mmap, + .poll = drm_poll, + .read = drm_read, +#ifdef CONFIG_COMPAT + .compat_ioctl = radeon_kms_compat_ioctl, +#endif +}; +#endif /* __linux__ */ + +struct drm_driver kms_driver = { + .driver_features = + DRIVER_USE_AGP | + DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | + DRIVER_PRIME | DRIVER_RENDER, +#ifdef notyet + .load = radeon_driver_load_kms, +#endif + .open = radeon_driver_open_kms, + .mmap = radeon_mmap, + .preclose = radeon_driver_preclose_kms, + .postclose = radeon_driver_postclose_kms, + .lastclose = radeon_driver_lastclose_kms, +#ifdef notyet + .set_busid = drm_pci_set_busid, + .unload = radeon_driver_unload_kms, +#endif + .get_vblank_counter = radeon_get_vblank_counter_kms, + .enable_vblank = radeon_enable_vblank_kms, + .disable_vblank = radeon_disable_vblank_kms, + .get_vblank_timestamp = radeon_get_vblank_timestamp_kms, + .get_scanout_position = radeon_get_crtc_scanoutpos, +#if defined(CONFIG_DEBUG_FS) + .debugfs_init = radeon_debugfs_init, + .debugfs_cleanup = radeon_debugfs_cleanup, +#endif + .irq_preinstall = radeon_driver_irq_preinstall_kms, + .irq_postinstall = radeon_driver_irq_postinstall_kms, + .irq_uninstall = radeon_driver_irq_uninstall_kms, + .irq_handler = radeon_driver_irq_handler_kms, + .ioctls = radeon_ioctls_kms, + .gem_free_object = radeon_gem_object_free, + .gem_open_object = radeon_gem_object_open, + .gem_close_object = radeon_gem_object_close, + .gem_size = sizeof(struct radeon_bo), + .dumb_create = radeon_mode_dumb_create, + .dumb_map_offset = radeon_mode_dumb_mmap, + .dumb_destroy = drm_gem_dumb_destroy, +#ifdef notyet + .fops = &radeon_driver_kms_fops, + + .prime_handle_to_fd = drm_gem_prime_handle_to_fd, + .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_export = radeon_gem_prime_export, + .gem_prime_import = drm_gem_prime_import, + .gem_prime_pin = radeon_gem_prime_pin, + .gem_prime_unpin = radeon_gem_prime_unpin, + .gem_prime_res_obj = radeon_gem_prime_res_obj, + .gem_prime_get_sg_table = radeon_gem_prime_get_sg_table, + .gem_prime_import_sg_table = radeon_gem_prime_import_sg_table, + .gem_prime_vmap = radeon_gem_prime_vmap, + .gem_prime_vunmap = radeon_gem_prime_vunmap, +#endif + + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .date = DRIVER_DATE, + .major = KMS_DRIVER_MAJOR, + .minor = KMS_DRIVER_MINOR, + .patchlevel = KMS_DRIVER_PATCHLEVEL, +}; + +#ifdef notyet +static struct drm_driver *driver; +#endif +#ifdef __linux__ +static struct pci_driver *pdriver; + +#ifdef CONFIG_DRM_RADEON_UMS +static struct pci_driver radeon_pci_driver = { + .name = DRIVER_NAME, + .id_table = pciidlist, +}; +#endif + +static struct pci_driver radeon_kms_pci_driver = { + .name = DRIVER_NAME, + .id_table = pciidlist, + .probe = radeon_pci_probe, + .remove = radeon_pci_remove, + .driver.pm = &radeon_pm_ops, +}; +#endif /* __linux__ */ + +#ifdef notyet +static int __init radeon_init(void) +{ +#ifdef CONFIG_VGA_CONSOLE + if (vgacon_text_force() && radeon_modeset == -1) { + DRM_INFO("VGACON disable radeon kernel modesetting.\n"); + radeon_modeset = 0; + } +#endif + /* set to modesetting by default if not nomodeset */ + if (radeon_modeset == -1) + radeon_modeset = 1; + + if (radeon_modeset == 1) { + DRM_INFO("radeon kernel modesetting enabled.\n"); + driver = &kms_driver; +#ifdef __linux__ + pdriver = &radeon_kms_pci_driver; +#endif + driver->driver_features |= DRIVER_MODESET; + driver->num_ioctls = radeon_max_kms_ioctl; + radeon_register_atpx_handler(); + + } else { +#ifdef CONFIG_DRM_RADEON_UMS + DRM_INFO("radeon userspace modesetting enabled.\n"); + driver = &driver_old; + pdriver = &radeon_pci_driver; + driver->driver_features &= ~DRIVER_MODESET; + driver->num_ioctls = radeon_max_ioctl; +#else + DRM_ERROR("No UMS support in radeon module!\n"); + return -EINVAL; +#endif + } + + radeon_kfd_init(); + + /* let modprobe override vga console setting */ +#ifdef notyet + return drm_pci_init(driver, pdriver); +#else + STUB(); + return -ENOSYS; +#endif +} + +static void __exit radeon_exit(void) +{ + STUB(); + radeon_kfd_fini(); +#ifdef notyet + drm_pci_exit(driver, pdriver); +#endif + radeon_unregister_atpx_handler(); +} +#endif + +module_init(radeon_init); +module_exit(radeon_exit); + +MODULE_AUTHOR(DRIVER_AUTHOR); +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("GPL and additional rights"); diff --git a/sys/dev/pci/drm/radeon/radeon_drv.h b/sys/dev/pci/drm/radeon/radeon_drv.h new file mode 100644 index 00000000000..1b962db8dfc --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_drv.h @@ -0,0 +1,2165 @@ +/* radeon_drv.h -- Private header for radeon driver -*- linux-c -*- + * + * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas. + * Copyright 2000 VA Linux Systems, Inc., Fremont, California. + * All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Kevin E. Martin + * Gareth Hughes + */ + +#ifndef __RADEON_DRV_H__ +#define __RADEON_DRV_H__ + +#include "radeon_family.h" + +/* General customization: + */ + +#define DRIVER_AUTHOR "Gareth Hughes, Keith Whitwell, others." + +#define DRIVER_NAME "radeon" +#define DRIVER_DESC "ATI Radeon" +#define DRIVER_DATE "20080528" + +/* Interface history: + * + * 1.1 - ?? + * 1.2 - Add vertex2 ioctl (keith) + * - Add stencil capability to clear ioctl (gareth, keith) + * - Increase MAX_TEXTURE_LEVELS (brian) + * 1.3 - Add cmdbuf ioctl (keith) + * - Add support for new radeon packets (keith) + * - Add getparam ioctl (keith) + * - Add flip-buffers ioctl, deprecate fullscreen foo (keith). + * 1.4 - Add scratch registers to get_param ioctl. + * 1.5 - Add r200 packets to cmdbuf ioctl + * - Add r200 function to init ioctl + * - Add 'scalar2' instruction to cmdbuf + * 1.6 - Add static GART memory manager + * Add irq handler (won't be turned on unless X server knows to) + * Add irq ioctls and irq_active getparam. + * Add wait command for cmdbuf ioctl + * Add GART offset query for getparam + * 1.7 - Add support for cube map registers: R200_PP_CUBIC_FACES_[0..5] + * and R200_PP_CUBIC_OFFSET_F1_[0..5]. + * Added packets R200_EMIT_PP_CUBIC_FACES_[0..5] and + * R200_EMIT_PP_CUBIC_OFFSETS_[0..5]. (brian) + * 1.8 - Remove need to call cleanup ioctls on last client exit (keith) + * Add 'GET' queries for starting additional clients on different VT's. + * 1.9 - Add DRM_IOCTL_RADEON_CP_RESUME ioctl. + * Add texture rectangle support for r100. + * 1.10- Add SETPARAM ioctl; first parameter to set is FB_LOCATION, which + * clients use to tell the DRM where they think the framebuffer is + * located in the card's address space + * 1.11- Add packet R200_EMIT_RB3D_BLENDCOLOR to support GL_EXT_blend_color + * and GL_EXT_blend_[func|equation]_separate on r200 + * 1.12- Add R300 CP microcode support - this just loads the CP on r300 + * (No 3D support yet - just microcode loading). + * 1.13- Add packet R200_EMIT_TCL_POINT_SPRITE_CNTL for ARB_point_parameters + * - Add hyperz support, add hyperz flags to clear ioctl. + * 1.14- Add support for color tiling + * - Add R100/R200 surface allocation/free support + * 1.15- Add support for texture micro tiling + * - Add support for r100 cube maps + * 1.16- Add R200_EMIT_PP_TRI_PERF_CNTL packet to support brilinear + * texture filtering on r200 + * 1.17- Add initial support for R300 (3D). + * 1.18- Add support for GL_ATI_fragment_shader, new packets + * R200_EMIT_PP_AFS_0/1, R200_EMIT_PP_TXCTLALL_0-5 (replaces + * R200_EMIT_PP_TXFILTER_0-5, 2 more regs) and R200_EMIT_ATF_TFACTOR + * (replaces R200_EMIT_TFACTOR_0 (8 consts instead of 6) + * 1.19- Add support for gart table in FB memory and PCIE r300 + * 1.20- Add support for r300 texrect + * 1.21- Add support for card type getparam + * 1.22- Add support for texture cache flushes (R300_TX_CNTL) + * 1.23- Add new radeon memory map work from benh + * 1.24- Add general-purpose packet for manipulating scratch registers (r300) + * 1.25- Add support for r200 vertex programs (R200_EMIT_VAP_PVS_CNTL, + * new packet type) + * 1.26- Add support for variable size PCI(E) gart aperture + * 1.27- Add support for IGP GART + * 1.28- Add support for VBL on CRTC2 + * 1.29- R500 3D cmd buffer support + * 1.30- Add support for occlusion queries + * 1.31- Add support for num Z pipes from GET_PARAM + * 1.32- fixes for rv740 setup + * 1.33- Add r6xx/r7xx const buffer support + * 1.34- fix evergreen/cayman GS register + */ +#define DRIVER_MAJOR 1 +#define DRIVER_MINOR 34 +#define DRIVER_PATCHLEVEL 0 + +long radeon_drm_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg); + +/* The rest of the file is DEPRECATED! */ +#ifdef CONFIG_DRM_RADEON_UMS + +enum radeon_cp_microcode_version { + UCODE_R100, + UCODE_R200, + UCODE_R300, +}; + +typedef struct drm_radeon_freelist { + unsigned int age; + struct drm_buf *buf; + struct drm_radeon_freelist *next; + struct drm_radeon_freelist *prev; +} drm_radeon_freelist_t; + +typedef struct drm_radeon_ring_buffer { + u32 *start; + u32 *end; + int size; + int size_l2qw; + + int rptr_update; /* Double Words */ + int rptr_update_l2qw; /* log2 Quad Words */ + + int fetch_size; /* Double Words */ + int fetch_size_l2ow; /* log2 Oct Words */ + + u32 tail; + u32 tail_mask; + int space; + + int high_mark; +} drm_radeon_ring_buffer_t; + +typedef struct drm_radeon_depth_clear_t { + u32 rb3d_cntl; + u32 rb3d_zstencilcntl; + u32 se_cntl; +} drm_radeon_depth_clear_t; + +struct drm_radeon_driver_file_fields { + int64_t radeon_fb_delta; +}; + +struct mem_block { + struct mem_block *next; + struct mem_block *prev; + int start; + int size; + struct drm_file *file_priv; /* NULL: free, -1: heap, other: real files */ +}; + +struct radeon_surface { + int refcount; + u32 lower; + u32 upper; + u32 flags; +}; + +struct radeon_virt_surface { + int surface_index; + u32 lower; + u32 upper; + u32 flags; + struct drm_file *file_priv; +#define PCIGART_FILE_PRIV ((void *) -1L) +}; + +#define RADEON_FLUSH_EMITED (1 << 0) +#define RADEON_PURGE_EMITED (1 << 1) + +struct drm_radeon_master_private { + drm_local_map_t *sarea; + drm_radeon_sarea_t *sarea_priv; +}; + +typedef struct drm_radeon_private { + drm_radeon_ring_buffer_t ring; + + u32 fb_location; + u32 fb_size; + int new_memmap; + + int gart_size; + u32 gart_vm_start; + unsigned long gart_buffers_offset; + + int cp_mode; + int cp_running; + + drm_radeon_freelist_t *head; + drm_radeon_freelist_t *tail; + int last_buf; + int writeback_works; + + int usec_timeout; + + int microcode_version; + + struct { + u32 boxes; + int freelist_timeouts; + int freelist_loops; + int requested_bufs; + int last_frame_reads; + int last_clear_reads; + int clears; + int texture_uploads; + } stats; + + int do_boxes; + int page_flipping; + + u32 color_fmt; + unsigned int front_offset; + unsigned int front_pitch; + unsigned int back_offset; + unsigned int back_pitch; + + u32 depth_fmt; + unsigned int depth_offset; + unsigned int depth_pitch; + + u32 front_pitch_offset; + u32 back_pitch_offset; + u32 depth_pitch_offset; + + drm_radeon_depth_clear_t depth_clear; + + unsigned long ring_offset; + unsigned long ring_rptr_offset; + unsigned long buffers_offset; + unsigned long gart_textures_offset; + + drm_local_map_t *sarea; + drm_local_map_t *cp_ring; + drm_local_map_t *ring_rptr; + drm_local_map_t *gart_textures; + + struct mem_block *gart_heap; + struct mem_block *fb_heap; + + /* SW interrupt */ + wait_queue_head_t swi_queue; + atomic_t swi_emitted; + int vblank_crtc; + uint32_t irq_enable_reg; + uint32_t r500_disp_irq_reg; + + struct radeon_surface surfaces[RADEON_MAX_SURFACES]; + struct radeon_virt_surface virt_surfaces[2 * RADEON_MAX_SURFACES]; + + unsigned long pcigart_offset; + unsigned int pcigart_offset_set; + struct drm_ati_pcigart_info gart_info; + + u32 scratch_ages[5]; + + int have_z_offset; + + /* starting from here on, data is preserved across an open */ + uint32_t flags; /* see radeon_chip_flags */ + resource_size_t fb_aper_offset; + + int num_gb_pipes; + int num_z_pipes; + int track_flush; + drm_local_map_t *mmio; + + /* r6xx/r7xx pipe/shader config */ + int r600_max_pipes; + int r600_max_tile_pipes; + int r600_max_simds; + int r600_max_backends; + int r600_max_gprs; + int r600_max_threads; + int r600_max_stack_entries; + int r600_max_hw_contexts; + int r600_max_gs_threads; + int r600_sx_max_export_size; + int r600_sx_max_export_pos_size; + int r600_sx_max_export_smx_size; + int r600_sq_num_cf_insts; + int r700_sx_num_of_sets; + int r700_sc_prim_fifo_size; + int r700_sc_hiz_tile_fifo_size; + int r700_sc_earlyz_tile_fifo_fize; + int r600_group_size; + int r600_npipes; + int r600_nbanks; + + struct rwlock cs_mutex; + u32 cs_id_scnt; + u32 cs_id_wcnt; + /* r6xx/r7xx drm blit vertex buffer */ + struct drm_buf *blit_vb; + + /* firmware */ + const struct firmware *me_fw, *pfp_fw; +} drm_radeon_private_t; + +typedef struct drm_radeon_buf_priv { + u32 age; +} drm_radeon_buf_priv_t; + +struct drm_buffer; + +typedef struct drm_radeon_kcmd_buffer { + int bufsz; + struct drm_buffer *buffer; + int nbox; + struct drm_clip_rect __user *boxes; +} drm_radeon_kcmd_buffer_t; + +extern int radeon_no_wb; +extern struct drm_ioctl_desc radeon_ioctls[]; +extern int radeon_max_ioctl; + +extern u32 radeon_get_ring_head(drm_radeon_private_t *dev_priv); +extern void radeon_set_ring_head(drm_radeon_private_t *dev_priv, u32 val); + +#define GET_RING_HEAD(dev_priv) radeon_get_ring_head(dev_priv) +#define SET_RING_HEAD(dev_priv, val) radeon_set_ring_head(dev_priv, val) + +/* Check whether the given hardware address is inside the framebuffer or the + * GART area. + */ +static __inline__ int radeon_check_offset(drm_radeon_private_t *dev_priv, + u64 off) +{ + u32 fb_start = dev_priv->fb_location; + u32 fb_end = fb_start + dev_priv->fb_size - 1; + u32 gart_start = dev_priv->gart_vm_start; + u32 gart_end = gart_start + dev_priv->gart_size - 1; + + return ((off >= fb_start && off <= fb_end) || + (off >= gart_start && off <= gart_end)); +} + +/* radeon_state.c */ +extern void radeon_cp_discard_buffer(struct drm_device *dev, struct drm_master *master, struct drm_buf *buf); + + /* radeon_cp.c */ +extern int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv); +extern void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc); +extern void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base); + +extern void radeon_freelist_reset(struct drm_device * dev); +extern struct drm_buf *radeon_freelist_get(struct drm_device * dev); + +extern int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n); + +extern int radeon_do_cp_idle(drm_radeon_private_t * dev_priv); + +extern int radeon_driver_preinit(struct drm_device *dev, unsigned long flags); +extern int radeon_presetup(struct drm_device *dev); +extern int radeon_driver_postcleanup(struct drm_device *dev); + +extern int radeon_mem_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_mem_free(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_mem_init_heap(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern void radeon_mem_takedown(struct mem_block **heap); +extern void radeon_mem_release(struct drm_file *file_priv, + struct mem_block *heap); + +extern void radeon_enable_bm(struct drm_radeon_private *dev_priv); +extern u32 radeon_read_ring_rptr(drm_radeon_private_t *dev_priv, u32 off); +extern void radeon_write_ring_rptr(drm_radeon_private_t *dev_priv, u32 off, u32 val); + + /* radeon_irq.c */ +extern void radeon_irq_set_state(struct drm_device *dev, u32 mask, int state); +extern int radeon_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv); +extern int radeon_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv); + +extern void radeon_do_release(struct drm_device * dev); +extern u32 radeon_get_vblank_counter(struct drm_device *dev, unsigned int pipe); +extern int radeon_enable_vblank(struct drm_device *dev, unsigned int pipe); +extern void radeon_disable_vblank(struct drm_device *dev, unsigned int pipe); +extern irqreturn_t radeon_driver_irq_handler(int irq, void *arg); +extern void radeon_driver_irq_preinstall(struct drm_device * dev); +extern int radeon_driver_irq_postinstall(struct drm_device *dev); +extern void radeon_driver_irq_uninstall(struct drm_device * dev); +extern void radeon_enable_interrupt(struct drm_device *dev); +extern int radeon_vblank_crtc_get(struct drm_device *dev); +extern int radeon_vblank_crtc_set(struct drm_device *dev, int64_t value); + +extern int radeon_driver_load(struct drm_device *dev, unsigned long flags); +extern int radeon_driver_unload(struct drm_device *dev); +extern int radeon_driver_firstopen(struct drm_device *dev); +extern void radeon_driver_preclose(struct drm_device *dev, + struct drm_file *file_priv); +extern void radeon_driver_postclose(struct drm_device *dev, + struct drm_file *file_priv); +extern void radeon_driver_lastclose(struct drm_device * dev); +extern int radeon_driver_open(struct drm_device *dev, + struct drm_file *file_priv); +extern long radeon_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg); + +extern int radeon_master_create(struct drm_device *dev, struct drm_master *master); +extern void radeon_master_destroy(struct drm_device *dev, struct drm_master *master); +extern void radeon_cp_dispatch_flip(struct drm_device *dev, struct drm_master *master); +/* r300_cmdbuf.c */ +extern void r300_init_reg_flags(struct drm_device *dev); + +extern int r300_do_cp_cmdbuf(struct drm_device *dev, + struct drm_file *file_priv, + drm_radeon_kcmd_buffer_t *cmdbuf); + +/* r600_cp.c */ +extern int r600_do_engine_reset(struct drm_device *dev); +extern int r600_do_cleanup_cp(struct drm_device *dev); +extern int r600_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init, + struct drm_file *file_priv); +extern int r600_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv); +extern int r600_do_cp_idle(drm_radeon_private_t *dev_priv); +extern void r600_do_cp_start(drm_radeon_private_t *dev_priv); +extern void r600_do_cp_reset(drm_radeon_private_t *dev_priv); +extern void r600_do_cp_stop(drm_radeon_private_t *dev_priv); +extern int r600_cp_dispatch_indirect(struct drm_device *dev, + struct drm_buf *buf, int start, int end); +extern int r600_page_table_init(struct drm_device *dev); +extern void r600_page_table_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info); +extern int r600_cs_legacy_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv); +extern void r600_cp_dispatch_swap(struct drm_device *dev, struct drm_file *file_priv); +extern int r600_cp_dispatch_texture(struct drm_device *dev, + struct drm_file *file_priv, + drm_radeon_texture_t *tex, + drm_radeon_tex_image_t *image); +/* r600_blit.c */ +extern int r600_prepare_blit_copy(struct drm_device *dev, struct drm_file *file_priv); +extern void r600_done_blit_copy(struct drm_device *dev); +extern void r600_blit_copy(struct drm_device *dev, + uint64_t src_gpu_addr, uint64_t dst_gpu_addr, + int size_bytes); +extern void r600_blit_swap(struct drm_device *dev, + uint64_t src_gpu_addr, uint64_t dst_gpu_addr, + int sx, int sy, int dx, int dy, + int w, int h, int src_pitch, int dst_pitch, int cpp); + +/* Flags for stats.boxes + */ +#define RADEON_BOX_DMA_IDLE 0x1 +#define RADEON_BOX_RING_FULL 0x2 +#define RADEON_BOX_FLIP 0x4 +#define RADEON_BOX_WAIT_IDLE 0x8 +#define RADEON_BOX_TEXTURE_LOAD 0x10 + +/* Register definitions, register access macros and drmAddMap constants + * for Radeon kernel driver. + */ +#define RADEON_MM_INDEX 0x0000 +#define RADEON_MM_DATA 0x0004 + +#define RADEON_AGP_COMMAND 0x0f60 +#define RADEON_AGP_COMMAND_PCI_CONFIG 0x0060 /* offset in PCI config */ +# define RADEON_AGP_ENABLE (1<<8) +#define RADEON_AUX_SCISSOR_CNTL 0x26f0 +# define RADEON_EXCLUSIVE_SCISSOR_0 (1 << 24) +# define RADEON_EXCLUSIVE_SCISSOR_1 (1 << 25) +# define RADEON_EXCLUSIVE_SCISSOR_2 (1 << 26) +# define RADEON_SCISSOR_0_ENABLE (1 << 28) +# define RADEON_SCISSOR_1_ENABLE (1 << 29) +# define RADEON_SCISSOR_2_ENABLE (1 << 30) + +/* + * PCIE radeons (rv370/rv380, rv410, r423/r430/r480, r5xx) + * don't have an explicit bus mastering disable bit. It's handled + * by the PCI D-states. PMI_BM_DIS disables D-state bus master + * handling, not bus mastering itself. + */ +#define RADEON_BUS_CNTL 0x0030 +/* r1xx, r2xx, r300, r(v)350, r420/r481, rs400/rs480 */ +# define RADEON_BUS_MASTER_DIS (1 << 6) +/* rs600/rs690/rs740 */ +# define RS600_BUS_MASTER_DIS (1 << 14) +# define RS600_MSI_REARM (1 << 20) +/* see RS400_MSI_REARM in AIC_CNTL for rs480 */ + +#define RADEON_BUS_CNTL1 0x0034 +# define RADEON_PMI_BM_DIS (1 << 2) +# define RADEON_PMI_INT_DIS (1 << 3) + +#define RV370_BUS_CNTL 0x004c +# define RV370_PMI_BM_DIS (1 << 5) +# define RV370_PMI_INT_DIS (1 << 6) + +#define RADEON_MSI_REARM_EN 0x0160 +/* rv370/rv380, rv410, r423/r430/r480, r5xx */ +# define RV370_MSI_REARM_EN (1 << 0) + +#define RADEON_CLOCK_CNTL_DATA 0x000c +# define RADEON_PLL_WR_EN (1 << 7) +#define RADEON_CLOCK_CNTL_INDEX 0x0008 +#define RADEON_CONFIG_APER_SIZE 0x0108 +#define RADEON_CONFIG_MEMSIZE 0x00f8 +#define RADEON_CRTC_OFFSET 0x0224 +#define RADEON_CRTC_OFFSET_CNTL 0x0228 +# define RADEON_CRTC_TILE_EN (1 << 15) +# define RADEON_CRTC_OFFSET_FLIP_CNTL (1 << 16) +#define RADEON_CRTC2_OFFSET 0x0324 +#define RADEON_CRTC2_OFFSET_CNTL 0x0328 + +#define RADEON_PCIE_INDEX 0x0030 +#define RADEON_PCIE_DATA 0x0034 +#define RADEON_PCIE_TX_GART_CNTL 0x10 +# define RADEON_PCIE_TX_GART_EN (1 << 0) +# define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_PASS_THRU (0 << 1) +# define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_CLAMP_LO (1 << 1) +# define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD (3 << 1) +# define RADEON_PCIE_TX_GART_MODE_32_128_CACHE (0 << 3) +# define RADEON_PCIE_TX_GART_MODE_8_4_128_CACHE (1 << 3) +# define RADEON_PCIE_TX_GART_CHK_RW_VALID_EN (1 << 5) +# define RADEON_PCIE_TX_GART_INVALIDATE_TLB (1 << 8) +#define RADEON_PCIE_TX_DISCARD_RD_ADDR_LO 0x11 +#define RADEON_PCIE_TX_DISCARD_RD_ADDR_HI 0x12 +#define RADEON_PCIE_TX_GART_BASE 0x13 +#define RADEON_PCIE_TX_GART_START_LO 0x14 +#define RADEON_PCIE_TX_GART_START_HI 0x15 +#define RADEON_PCIE_TX_GART_END_LO 0x16 +#define RADEON_PCIE_TX_GART_END_HI 0x17 + +#define RS480_NB_MC_INDEX 0x168 +# define RS480_NB_MC_IND_WR_EN (1 << 8) +#define RS480_NB_MC_DATA 0x16c + +#define RS690_MC_INDEX 0x78 +# define RS690_MC_INDEX_MASK 0x1ff +# define RS690_MC_INDEX_WR_EN (1 << 9) +# define RS690_MC_INDEX_WR_ACK 0x7f +#define RS690_MC_DATA 0x7c + +/* MC indirect registers */ +#define RS480_MC_MISC_CNTL 0x18 +# define RS480_DISABLE_GTW (1 << 1) +/* switch between MCIND GART and MM GART registers. 0 = mmgart, 1 = mcind gart */ +# define RS480_GART_INDEX_REG_EN (1 << 12) +# define RS690_BLOCK_GFX_D3_EN (1 << 14) +#define RS480_K8_FB_LOCATION 0x1e +#define RS480_GART_FEATURE_ID 0x2b +# define RS480_HANG_EN (1 << 11) +# define RS480_TLB_ENABLE (1 << 18) +# define RS480_P2P_ENABLE (1 << 19) +# define RS480_GTW_LAC_EN (1 << 25) +# define RS480_2LEVEL_GART (0 << 30) +# define RS480_1LEVEL_GART (1 << 30) +# define RS480_PDC_EN (1 << 31) +#define RS480_GART_BASE 0x2c +#define RS480_GART_CACHE_CNTRL 0x2e +# define RS480_GART_CACHE_INVALIDATE (1 << 0) /* wait for it to clear */ +#define RS480_AGP_ADDRESS_SPACE_SIZE 0x38 +# define RS480_GART_EN (1 << 0) +# define RS480_VA_SIZE_32MB (0 << 1) +# define RS480_VA_SIZE_64MB (1 << 1) +# define RS480_VA_SIZE_128MB (2 << 1) +# define RS480_VA_SIZE_256MB (3 << 1) +# define RS480_VA_SIZE_512MB (4 << 1) +# define RS480_VA_SIZE_1GB (5 << 1) +# define RS480_VA_SIZE_2GB (6 << 1) +#define RS480_AGP_MODE_CNTL 0x39 +# define RS480_POST_GART_Q_SIZE (1 << 18) +# define RS480_NONGART_SNOOP (1 << 19) +# define RS480_AGP_RD_BUF_SIZE (1 << 20) +# define RS480_REQ_TYPE_SNOOP_SHIFT 22 +# define RS480_REQ_TYPE_SNOOP_MASK 0x3 +# define RS480_REQ_TYPE_SNOOP_DIS (1 << 24) +#define RS480_MC_MISC_UMA_CNTL 0x5f +#define RS480_MC_MCLK_CNTL 0x7a +#define RS480_MC_UMA_DUALCH_CNTL 0x86 + +#define RS690_MC_FB_LOCATION 0x100 +#define RS690_MC_AGP_LOCATION 0x101 +#define RS690_MC_AGP_BASE 0x102 +#define RS690_MC_AGP_BASE_2 0x103 + +#define RS600_MC_INDEX 0x70 +# define RS600_MC_ADDR_MASK 0xffff +# define RS600_MC_IND_SEQ_RBS_0 (1 << 16) +# define RS600_MC_IND_SEQ_RBS_1 (1 << 17) +# define RS600_MC_IND_SEQ_RBS_2 (1 << 18) +# define RS600_MC_IND_SEQ_RBS_3 (1 << 19) +# define RS600_MC_IND_AIC_RBS (1 << 20) +# define RS600_MC_IND_CITF_ARB0 (1 << 21) +# define RS600_MC_IND_CITF_ARB1 (1 << 22) +# define RS600_MC_IND_WR_EN (1 << 23) +#define RS600_MC_DATA 0x74 + +#define RS600_MC_STATUS 0x0 +# define RS600_MC_IDLE (1 << 1) +#define RS600_MC_FB_LOCATION 0x4 +#define RS600_MC_AGP_LOCATION 0x5 +#define RS600_AGP_BASE 0x6 +#define RS600_AGP_BASE_2 0x7 +#define RS600_MC_CNTL1 0x9 +# define RS600_ENABLE_PAGE_TABLES (1 << 26) +#define RS600_MC_PT0_CNTL 0x100 +# define RS600_ENABLE_PT (1 << 0) +# define RS600_EFFECTIVE_L2_CACHE_SIZE(x) ((x) << 15) +# define RS600_EFFECTIVE_L2_QUEUE_SIZE(x) ((x) << 21) +# define RS600_INVALIDATE_ALL_L1_TLBS (1 << 28) +# define RS600_INVALIDATE_L2_CACHE (1 << 29) +#define RS600_MC_PT0_CONTEXT0_CNTL 0x102 +# define RS600_ENABLE_PAGE_TABLE (1 << 0) +# define RS600_PAGE_TABLE_TYPE_FLAT (0 << 1) +#define RS600_MC_PT0_SYSTEM_APERTURE_LOW_ADDR 0x112 +#define RS600_MC_PT0_SYSTEM_APERTURE_HIGH_ADDR 0x114 +#define RS600_MC_PT0_CONTEXT0_DEFAULT_READ_ADDR 0x11c +#define RS600_MC_PT0_CONTEXT0_FLAT_BASE_ADDR 0x12c +#define RS600_MC_PT0_CONTEXT0_FLAT_START_ADDR 0x13c +#define RS600_MC_PT0_CONTEXT0_FLAT_END_ADDR 0x14c +#define RS600_MC_PT0_CLIENT0_CNTL 0x16c +# define RS600_ENABLE_TRANSLATION_MODE_OVERRIDE (1 << 0) +# define RS600_TRANSLATION_MODE_OVERRIDE (1 << 1) +# define RS600_SYSTEM_ACCESS_MODE_MASK (3 << 8) +# define RS600_SYSTEM_ACCESS_MODE_PA_ONLY (0 << 8) +# define RS600_SYSTEM_ACCESS_MODE_USE_SYS_MAP (1 << 8) +# define RS600_SYSTEM_ACCESS_MODE_IN_SYS (2 << 8) +# define RS600_SYSTEM_ACCESS_MODE_NOT_IN_SYS (3 << 8) +# define RS600_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASSTHROUGH (0 << 10) +# define RS600_SYSTEM_APERTURE_UNMAPPED_ACCESS_DEFAULT_PAGE (1 << 10) +# define RS600_EFFECTIVE_L1_CACHE_SIZE(x) ((x) << 11) +# define RS600_ENABLE_FRAGMENT_PROCESSING (1 << 14) +# define RS600_EFFECTIVE_L1_QUEUE_SIZE(x) ((x) << 15) +# define RS600_INVALIDATE_L1_TLB (1 << 20) + +#define R520_MC_IND_INDEX 0x70 +#define R520_MC_IND_WR_EN (1 << 24) +#define R520_MC_IND_DATA 0x74 + +#define RV515_MC_FB_LOCATION 0x01 +#define RV515_MC_AGP_LOCATION 0x02 +#define RV515_MC_AGP_BASE 0x03 +#define RV515_MC_AGP_BASE_2 0x04 + +#define R520_MC_FB_LOCATION 0x04 +#define R520_MC_AGP_LOCATION 0x05 +#define R520_MC_AGP_BASE 0x06 +#define R520_MC_AGP_BASE_2 0x07 + +#define RADEON_MPP_TB_CONFIG 0x01c0 +#define RADEON_MEM_CNTL 0x0140 +#define RADEON_MEM_SDRAM_MODE_REG 0x0158 +#define RADEON_AGP_BASE_2 0x015c /* r200+ only */ +#define RS480_AGP_BASE_2 0x0164 +#define RADEON_AGP_BASE 0x0170 + +/* pipe config regs */ +#define R400_GB_PIPE_SELECT 0x402c +#define RV530_GB_PIPE_SELECT2 0x4124 +#define R500_DYN_SCLK_PWMEM_PIPE 0x000d /* PLL */ +#define R300_GB_TILE_CONFIG 0x4018 +# define R300_ENABLE_TILING (1 << 0) +# define R300_PIPE_COUNT_RV350 (0 << 1) +# define R300_PIPE_COUNT_R300 (3 << 1) +# define R300_PIPE_COUNT_R420_3P (6 << 1) +# define R300_PIPE_COUNT_R420 (7 << 1) +# define R300_TILE_SIZE_8 (0 << 4) +# define R300_TILE_SIZE_16 (1 << 4) +# define R300_TILE_SIZE_32 (2 << 4) +# define R300_SUBPIXEL_1_12 (0 << 16) +# define R300_SUBPIXEL_1_16 (1 << 16) +#define R300_DST_PIPE_CONFIG 0x170c +# define R300_PIPE_AUTO_CONFIG (1 << 31) +#define R300_RB2D_DSTCACHE_MODE 0x3428 +# define R300_DC_AUTOFLUSH_ENABLE (1 << 8) +# define R300_DC_DC_DISABLE_IGNORE_PE (1 << 17) + +#define RADEON_RB3D_COLOROFFSET 0x1c40 +#define RADEON_RB3D_COLORPITCH 0x1c48 + +#define RADEON_SRC_X_Y 0x1590 + +#define RADEON_DP_GUI_MASTER_CNTL 0x146c +# define RADEON_GMC_SRC_PITCH_OFFSET_CNTL (1 << 0) +# define RADEON_GMC_DST_PITCH_OFFSET_CNTL (1 << 1) +# define RADEON_GMC_BRUSH_SOLID_COLOR (13 << 4) +# define RADEON_GMC_BRUSH_NONE (15 << 4) +# define RADEON_GMC_DST_16BPP (4 << 8) +# define RADEON_GMC_DST_24BPP (5 << 8) +# define RADEON_GMC_DST_32BPP (6 << 8) +# define RADEON_GMC_DST_DATATYPE_SHIFT 8 +# define RADEON_GMC_SRC_DATATYPE_COLOR (3 << 12) +# define RADEON_DP_SRC_SOURCE_MEMORY (2 << 24) +# define RADEON_DP_SRC_SOURCE_HOST_DATA (3 << 24) +# define RADEON_GMC_CLR_CMP_CNTL_DIS (1 << 28) +# define RADEON_GMC_WR_MSK_DIS (1 << 30) +# define RADEON_ROP3_S 0x00cc0000 +# define RADEON_ROP3_P 0x00f00000 +#define RADEON_DP_WRITE_MASK 0x16cc +#define RADEON_SRC_PITCH_OFFSET 0x1428 +#define RADEON_DST_PITCH_OFFSET 0x142c +#define RADEON_DST_PITCH_OFFSET_C 0x1c80 +# define RADEON_DST_TILE_LINEAR (0 << 30) +# define RADEON_DST_TILE_MACRO (1 << 30) +# define RADEON_DST_TILE_MICRO (2 << 30) +# define RADEON_DST_TILE_BOTH (3 << 30) + +#define RADEON_SCRATCH_REG0 0x15e0 +#define RADEON_SCRATCH_REG1 0x15e4 +#define RADEON_SCRATCH_REG2 0x15e8 +#define RADEON_SCRATCH_REG3 0x15ec +#define RADEON_SCRATCH_REG4 0x15f0 +#define RADEON_SCRATCH_REG5 0x15f4 +#define RADEON_SCRATCH_UMSK 0x0770 +#define RADEON_SCRATCH_ADDR 0x0774 + +#define RADEON_SCRATCHOFF( x ) (RADEON_SCRATCH_REG_OFFSET + 4*(x)) + +extern u32 radeon_get_scratch(drm_radeon_private_t *dev_priv, int index); + +#define GET_SCRATCH(dev_priv, x) radeon_get_scratch(dev_priv, x) + +#define R600_SCRATCH_REG0 0x8500 +#define R600_SCRATCH_REG1 0x8504 +#define R600_SCRATCH_REG2 0x8508 +#define R600_SCRATCH_REG3 0x850c +#define R600_SCRATCH_REG4 0x8510 +#define R600_SCRATCH_REG5 0x8514 +#define R600_SCRATCH_REG6 0x8518 +#define R600_SCRATCH_REG7 0x851c +#define R600_SCRATCH_UMSK 0x8540 +#define R600_SCRATCH_ADDR 0x8544 + +#define R600_SCRATCHOFF(x) (R600_SCRATCH_REG_OFFSET + 4*(x)) + +#define RADEON_GEN_INT_CNTL 0x0040 +# define RADEON_CRTC_VBLANK_MASK (1 << 0) +# define RADEON_CRTC2_VBLANK_MASK (1 << 9) +# define RADEON_GUI_IDLE_INT_ENABLE (1 << 19) +# define RADEON_SW_INT_ENABLE (1 << 25) + +#define RADEON_GEN_INT_STATUS 0x0044 +# define RADEON_CRTC_VBLANK_STAT (1 << 0) +# define RADEON_CRTC_VBLANK_STAT_ACK (1 << 0) +# define RADEON_CRTC2_VBLANK_STAT (1 << 9) +# define RADEON_CRTC2_VBLANK_STAT_ACK (1 << 9) +# define RADEON_GUI_IDLE_INT_TEST_ACK (1 << 19) +# define RADEON_SW_INT_TEST (1 << 25) +# define RADEON_SW_INT_TEST_ACK (1 << 25) +# define RADEON_SW_INT_FIRE (1 << 26) +# define R500_DISPLAY_INT_STATUS (1 << 0) + +#define RADEON_HOST_PATH_CNTL 0x0130 +# define RADEON_HDP_SOFT_RESET (1 << 26) +# define RADEON_HDP_WC_TIMEOUT_MASK (7 << 28) +# define RADEON_HDP_WC_TIMEOUT_28BCLK (7 << 28) + +#define RADEON_ISYNC_CNTL 0x1724 +# define RADEON_ISYNC_ANY2D_IDLE3D (1 << 0) +# define RADEON_ISYNC_ANY3D_IDLE2D (1 << 1) +# define RADEON_ISYNC_TRIG2D_IDLE3D (1 << 2) +# define RADEON_ISYNC_TRIG3D_IDLE2D (1 << 3) +# define RADEON_ISYNC_WAIT_IDLEGUI (1 << 4) +# define RADEON_ISYNC_CPSCRATCH_IDLEGUI (1 << 5) + +#define RADEON_RBBM_GUICNTL 0x172c +# define RADEON_HOST_DATA_SWAP_NONE (0 << 0) +# define RADEON_HOST_DATA_SWAP_16BIT (1 << 0) +# define RADEON_HOST_DATA_SWAP_32BIT (2 << 0) +# define RADEON_HOST_DATA_SWAP_HDW (3 << 0) + +#define RADEON_MC_AGP_LOCATION 0x014c +#define RADEON_MC_FB_LOCATION 0x0148 +#define RADEON_MCLK_CNTL 0x0012 +# define RADEON_FORCEON_MCLKA (1 << 16) +# define RADEON_FORCEON_MCLKB (1 << 17) +# define RADEON_FORCEON_YCLKA (1 << 18) +# define RADEON_FORCEON_YCLKB (1 << 19) +# define RADEON_FORCEON_MC (1 << 20) +# define RADEON_FORCEON_AIC (1 << 21) + +#define RADEON_PP_BORDER_COLOR_0 0x1d40 +#define RADEON_PP_BORDER_COLOR_1 0x1d44 +#define RADEON_PP_BORDER_COLOR_2 0x1d48 +#define RADEON_PP_CNTL 0x1c38 +# define RADEON_SCISSOR_ENABLE (1 << 1) +#define RADEON_PP_LUM_MATRIX 0x1d00 +#define RADEON_PP_MISC 0x1c14 +#define RADEON_PP_ROT_MATRIX_0 0x1d58 +#define RADEON_PP_TXFILTER_0 0x1c54 +#define RADEON_PP_TXOFFSET_0 0x1c5c +#define RADEON_PP_TXFILTER_1 0x1c6c +#define RADEON_PP_TXFILTER_2 0x1c84 + +#define R300_RB2D_DSTCACHE_CTLSTAT 0x342c /* use R300_DSTCACHE_CTLSTAT */ +#define R300_DSTCACHE_CTLSTAT 0x1714 +# define R300_RB2D_DC_FLUSH (3 << 0) +# define R300_RB2D_DC_FREE (3 << 2) +# define R300_RB2D_DC_FLUSH_ALL 0xf +# define R300_RB2D_DC_BUSY (1 << 31) +#define RADEON_RB3D_CNTL 0x1c3c +# define RADEON_ALPHA_BLEND_ENABLE (1 << 0) +# define RADEON_PLANE_MASK_ENABLE (1 << 1) +# define RADEON_DITHER_ENABLE (1 << 2) +# define RADEON_ROUND_ENABLE (1 << 3) +# define RADEON_SCALE_DITHER_ENABLE (1 << 4) +# define RADEON_DITHER_INIT (1 << 5) +# define RADEON_ROP_ENABLE (1 << 6) +# define RADEON_STENCIL_ENABLE (1 << 7) +# define RADEON_Z_ENABLE (1 << 8) +# define RADEON_ZBLOCK16 (1 << 15) +#define RADEON_RB3D_DEPTHOFFSET 0x1c24 +#define RADEON_RB3D_DEPTHCLEARVALUE 0x3230 +#define RADEON_RB3D_DEPTHPITCH 0x1c28 +#define RADEON_RB3D_PLANEMASK 0x1d84 +#define RADEON_RB3D_STENCILREFMASK 0x1d7c +#define RADEON_RB3D_ZCACHE_MODE 0x3250 +#define RADEON_RB3D_ZCACHE_CTLSTAT 0x3254 +# define RADEON_RB3D_ZC_FLUSH (1 << 0) +# define RADEON_RB3D_ZC_FREE (1 << 2) +# define RADEON_RB3D_ZC_FLUSH_ALL 0x5 +# define RADEON_RB3D_ZC_BUSY (1 << 31) +#define R300_ZB_ZCACHE_CTLSTAT 0x4f18 +# define R300_ZC_FLUSH (1 << 0) +# define R300_ZC_FREE (1 << 1) +# define R300_ZC_BUSY (1 << 31) +#define RADEON_RB3D_DSTCACHE_CTLSTAT 0x325c +# define RADEON_RB3D_DC_FLUSH (3 << 0) +# define RADEON_RB3D_DC_FREE (3 << 2) +# define RADEON_RB3D_DC_FLUSH_ALL 0xf +# define RADEON_RB3D_DC_BUSY (1 << 31) +#define R300_RB3D_DSTCACHE_CTLSTAT 0x4e4c +# define R300_RB3D_DC_FLUSH (2 << 0) +# define R300_RB3D_DC_FREE (2 << 2) +# define R300_RB3D_DC_FINISH (1 << 4) +#define RADEON_RB3D_ZSTENCILCNTL 0x1c2c +# define RADEON_Z_TEST_MASK (7 << 4) +# define RADEON_Z_TEST_ALWAYS (7 << 4) +# define RADEON_Z_HIERARCHY_ENABLE (1 << 8) +# define RADEON_STENCIL_TEST_ALWAYS (7 << 12) +# define RADEON_STENCIL_S_FAIL_REPLACE (2 << 16) +# define RADEON_STENCIL_ZPASS_REPLACE (2 << 20) +# define RADEON_STENCIL_ZFAIL_REPLACE (2 << 24) +# define RADEON_Z_COMPRESSION_ENABLE (1 << 28) +# define RADEON_FORCE_Z_DIRTY (1 << 29) +# define RADEON_Z_WRITE_ENABLE (1 << 30) +# define RADEON_Z_DECOMPRESSION_ENABLE (1 << 31) +#define RADEON_RBBM_SOFT_RESET 0x00f0 +# define RADEON_SOFT_RESET_CP (1 << 0) +# define RADEON_SOFT_RESET_HI (1 << 1) +# define RADEON_SOFT_RESET_SE (1 << 2) +# define RADEON_SOFT_RESET_RE (1 << 3) +# define RADEON_SOFT_RESET_PP (1 << 4) +# define RADEON_SOFT_RESET_E2 (1 << 5) +# define RADEON_SOFT_RESET_RB (1 << 6) +# define RADEON_SOFT_RESET_HDP (1 << 7) +/* + * 6:0 Available slots in the FIFO + * 8 Host Interface active + * 9 CP request active + * 10 FIFO request active + * 11 Host Interface retry active + * 12 CP retry active + * 13 FIFO retry active + * 14 FIFO pipeline busy + * 15 Event engine busy + * 16 CP command stream busy + * 17 2D engine busy + * 18 2D portion of render backend busy + * 20 3D setup engine busy + * 26 GA engine busy + * 27 CBA 2D engine busy + * 31 2D engine busy or 3D engine busy or FIFO not empty or CP busy or + * command stream queue not empty or Ring Buffer not empty + */ +#define RADEON_RBBM_STATUS 0x0e40 +/* Same as the previous RADEON_RBBM_STATUS; this is a mirror of that register. */ +/* #define RADEON_RBBM_STATUS 0x1740 */ +/* bits 6:0 are dword slots available in the cmd fifo */ +# define RADEON_RBBM_FIFOCNT_MASK 0x007f +# define RADEON_HIRQ_ON_RBB (1 << 8) +# define RADEON_CPRQ_ON_RBB (1 << 9) +# define RADEON_CFRQ_ON_RBB (1 << 10) +# define RADEON_HIRQ_IN_RTBUF (1 << 11) +# define RADEON_CPRQ_IN_RTBUF (1 << 12) +# define RADEON_CFRQ_IN_RTBUF (1 << 13) +# define RADEON_PIPE_BUSY (1 << 14) +# define RADEON_ENG_EV_BUSY (1 << 15) +# define RADEON_CP_CMDSTRM_BUSY (1 << 16) +# define RADEON_E2_BUSY (1 << 17) +# define RADEON_RB2D_BUSY (1 << 18) +# define RADEON_RB3D_BUSY (1 << 19) /* not used on r300 */ +# define RADEON_VAP_BUSY (1 << 20) +# define RADEON_RE_BUSY (1 << 21) /* not used on r300 */ +# define RADEON_TAM_BUSY (1 << 22) /* not used on r300 */ +# define RADEON_TDM_BUSY (1 << 23) /* not used on r300 */ +# define RADEON_PB_BUSY (1 << 24) /* not used on r300 */ +# define RADEON_TIM_BUSY (1 << 25) /* not used on r300 */ +# define RADEON_GA_BUSY (1 << 26) +# define RADEON_CBA2D_BUSY (1 << 27) +# define RADEON_RBBM_ACTIVE (1 << 31) +#define RADEON_RE_LINE_PATTERN 0x1cd0 +#define RADEON_RE_MISC 0x26c4 +#define RADEON_RE_TOP_LEFT 0x26c0 +#define RADEON_RE_WIDTH_HEIGHT 0x1c44 +#define RADEON_RE_STIPPLE_ADDR 0x1cc8 +#define RADEON_RE_STIPPLE_DATA 0x1ccc + +#define RADEON_SCISSOR_TL_0 0x1cd8 +#define RADEON_SCISSOR_BR_0 0x1cdc +#define RADEON_SCISSOR_TL_1 0x1ce0 +#define RADEON_SCISSOR_BR_1 0x1ce4 +#define RADEON_SCISSOR_TL_2 0x1ce8 +#define RADEON_SCISSOR_BR_2 0x1cec +#define RADEON_SE_COORD_FMT 0x1c50 +#define RADEON_SE_CNTL 0x1c4c +# define RADEON_FFACE_CULL_CW (0 << 0) +# define RADEON_BFACE_SOLID (3 << 1) +# define RADEON_FFACE_SOLID (3 << 3) +# define RADEON_FLAT_SHADE_VTX_LAST (3 << 6) +# define RADEON_DIFFUSE_SHADE_FLAT (1 << 8) +# define RADEON_DIFFUSE_SHADE_GOURAUD (2 << 8) +# define RADEON_ALPHA_SHADE_FLAT (1 << 10) +# define RADEON_ALPHA_SHADE_GOURAUD (2 << 10) +# define RADEON_SPECULAR_SHADE_FLAT (1 << 12) +# define RADEON_SPECULAR_SHADE_GOURAUD (2 << 12) +# define RADEON_FOG_SHADE_FLAT (1 << 14) +# define RADEON_FOG_SHADE_GOURAUD (2 << 14) +# define RADEON_VPORT_XY_XFORM_ENABLE (1 << 24) +# define RADEON_VPORT_Z_XFORM_ENABLE (1 << 25) +# define RADEON_VTX_PIX_CENTER_OGL (1 << 27) +# define RADEON_ROUND_MODE_TRUNC (0 << 28) +# define RADEON_ROUND_PREC_8TH_PIX (1 << 30) +#define RADEON_SE_CNTL_STATUS 0x2140 +#define RADEON_SE_LINE_WIDTH 0x1db8 +#define RADEON_SE_VPORT_XSCALE 0x1d98 +#define RADEON_SE_ZBIAS_FACTOR 0x1db0 +#define RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED 0x2210 +#define RADEON_SE_TCL_OUTPUT_VTX_FMT 0x2254 +#define RADEON_SE_TCL_VECTOR_INDX_REG 0x2200 +# define RADEON_VEC_INDX_OCTWORD_STRIDE_SHIFT 16 +# define RADEON_VEC_INDX_DWORD_COUNT_SHIFT 28 +#define RADEON_SE_TCL_VECTOR_DATA_REG 0x2204 +#define RADEON_SE_TCL_SCALAR_INDX_REG 0x2208 +# define RADEON_SCAL_INDX_DWORD_STRIDE_SHIFT 16 +#define RADEON_SE_TCL_SCALAR_DATA_REG 0x220C +#define RADEON_SURFACE_ACCESS_FLAGS 0x0bf8 +#define RADEON_SURFACE_ACCESS_CLR 0x0bfc +#define RADEON_SURFACE_CNTL 0x0b00 +# define RADEON_SURF_TRANSLATION_DIS (1 << 8) +# define RADEON_NONSURF_AP0_SWP_MASK (3 << 20) +# define RADEON_NONSURF_AP0_SWP_LITTLE (0 << 20) +# define RADEON_NONSURF_AP0_SWP_BIG16 (1 << 20) +# define RADEON_NONSURF_AP0_SWP_BIG32 (2 << 20) +# define RADEON_NONSURF_AP1_SWP_MASK (3 << 22) +# define RADEON_NONSURF_AP1_SWP_LITTLE (0 << 22) +# define RADEON_NONSURF_AP1_SWP_BIG16 (1 << 22) +# define RADEON_NONSURF_AP1_SWP_BIG32 (2 << 22) +#define RADEON_SURFACE0_INFO 0x0b0c +# define RADEON_SURF_PITCHSEL_MASK (0x1ff << 0) +# define RADEON_SURF_TILE_MODE_MASK (3 << 16) +# define RADEON_SURF_TILE_MODE_MACRO (0 << 16) +# define RADEON_SURF_TILE_MODE_MICRO (1 << 16) +# define RADEON_SURF_TILE_MODE_32BIT_Z (2 << 16) +# define RADEON_SURF_TILE_MODE_16BIT_Z (3 << 16) +#define RADEON_SURFACE0_LOWER_BOUND 0x0b04 +#define RADEON_SURFACE0_UPPER_BOUND 0x0b08 +# define RADEON_SURF_ADDRESS_FIXED_MASK (0x3ff << 0) +#define RADEON_SURFACE1_INFO 0x0b1c +#define RADEON_SURFACE1_LOWER_BOUND 0x0b14 +#define RADEON_SURFACE1_UPPER_BOUND 0x0b18 +#define RADEON_SURFACE2_INFO 0x0b2c +#define RADEON_SURFACE2_LOWER_BOUND 0x0b24 +#define RADEON_SURFACE2_UPPER_BOUND 0x0b28 +#define RADEON_SURFACE3_INFO 0x0b3c +#define RADEON_SURFACE3_LOWER_BOUND 0x0b34 +#define RADEON_SURFACE3_UPPER_BOUND 0x0b38 +#define RADEON_SURFACE4_INFO 0x0b4c +#define RADEON_SURFACE4_LOWER_BOUND 0x0b44 +#define RADEON_SURFACE4_UPPER_BOUND 0x0b48 +#define RADEON_SURFACE5_INFO 0x0b5c +#define RADEON_SURFACE5_LOWER_BOUND 0x0b54 +#define RADEON_SURFACE5_UPPER_BOUND 0x0b58 +#define RADEON_SURFACE6_INFO 0x0b6c +#define RADEON_SURFACE6_LOWER_BOUND 0x0b64 +#define RADEON_SURFACE6_UPPER_BOUND 0x0b68 +#define RADEON_SURFACE7_INFO 0x0b7c +#define RADEON_SURFACE7_LOWER_BOUND 0x0b74 +#define RADEON_SURFACE7_UPPER_BOUND 0x0b78 +#define RADEON_SW_SEMAPHORE 0x013c + +#define RADEON_WAIT_UNTIL 0x1720 +# define RADEON_WAIT_CRTC_PFLIP (1 << 0) +# define RADEON_WAIT_2D_IDLE (1 << 14) +# define RADEON_WAIT_3D_IDLE (1 << 15) +# define RADEON_WAIT_2D_IDLECLEAN (1 << 16) +# define RADEON_WAIT_3D_IDLECLEAN (1 << 17) +# define RADEON_WAIT_HOST_IDLECLEAN (1 << 18) + +#define RADEON_RB3D_ZMASKOFFSET 0x3234 +#define RADEON_RB3D_ZSTENCILCNTL 0x1c2c +# define RADEON_DEPTH_FORMAT_16BIT_INT_Z (0 << 0) +# define RADEON_DEPTH_FORMAT_24BIT_INT_Z (2 << 0) + +/* CP registers */ +#define RADEON_CP_ME_RAM_ADDR 0x07d4 +#define RADEON_CP_ME_RAM_RADDR 0x07d8 +#define RADEON_CP_ME_RAM_DATAH 0x07dc +#define RADEON_CP_ME_RAM_DATAL 0x07e0 + +#define RADEON_CP_RB_BASE 0x0700 +#define RADEON_CP_RB_CNTL 0x0704 +# define RADEON_BUF_SWAP_32BIT (2 << 16) +# define RADEON_RB_NO_UPDATE (1 << 27) +# define RADEON_RB_RPTR_WR_ENA (1 << 31) +#define RADEON_CP_RB_RPTR_ADDR 0x070c +#define RADEON_CP_RB_RPTR 0x0710 +#define RADEON_CP_RB_WPTR 0x0714 + +#define RADEON_CP_RB_WPTR_DELAY 0x0718 +# define RADEON_PRE_WRITE_TIMER_SHIFT 0 +# define RADEON_PRE_WRITE_LIMIT_SHIFT 23 + +#define RADEON_CP_IB_BASE 0x0738 + +#define RADEON_CP_CSQ_CNTL 0x0740 +# define RADEON_CSQ_CNT_PRIMARY_MASK (0xff << 0) +# define RADEON_CSQ_PRIDIS_INDDIS (0 << 28) +# define RADEON_CSQ_PRIPIO_INDDIS (1 << 28) +# define RADEON_CSQ_PRIBM_INDDIS (2 << 28) +# define RADEON_CSQ_PRIPIO_INDBM (3 << 28) +# define RADEON_CSQ_PRIBM_INDBM (4 << 28) +# define RADEON_CSQ_PRIPIO_INDPIO (15 << 28) + +#define R300_CP_RESYNC_ADDR 0x0778 +#define R300_CP_RESYNC_DATA 0x077c + +#define RADEON_AIC_CNTL 0x01d0 +# define RADEON_PCIGART_TRANSLATE_EN (1 << 0) +# define RS400_MSI_REARM (1 << 3) +#define RADEON_AIC_STAT 0x01d4 +#define RADEON_AIC_PT_BASE 0x01d8 +#define RADEON_AIC_LO_ADDR 0x01dc +#define RADEON_AIC_HI_ADDR 0x01e0 +#define RADEON_AIC_TLB_ADDR 0x01e4 +#define RADEON_AIC_TLB_DATA 0x01e8 + +/* CP command packets */ +#define RADEON_CP_PACKET0 0x00000000 +# define RADEON_ONE_REG_WR (1 << 15) +#define RADEON_CP_PACKET1 0x40000000 +#define RADEON_CP_PACKET2 0x80000000 +#define RADEON_CP_PACKET3 0xC0000000 +# define RADEON_CP_NOP 0x00001000 +# define RADEON_CP_NEXT_CHAR 0x00001900 +# define RADEON_CP_PLY_NEXTSCAN 0x00001D00 +# define RADEON_CP_SET_SCISSORS 0x00001E00 + /* GEN_INDX_PRIM is unsupported starting with R300 */ +# define RADEON_3D_RNDR_GEN_INDX_PRIM 0x00002300 +# define RADEON_WAIT_FOR_IDLE 0x00002600 +# define RADEON_3D_DRAW_VBUF 0x00002800 +# define RADEON_3D_DRAW_IMMD 0x00002900 +# define RADEON_3D_DRAW_INDX 0x00002A00 +# define RADEON_CP_LOAD_PALETTE 0x00002C00 +# define RADEON_3D_LOAD_VBPNTR 0x00002F00 +# define RADEON_MPEG_IDCT_MACROBLOCK 0x00003000 +# define RADEON_MPEG_IDCT_MACROBLOCK_REV 0x00003100 +# define RADEON_3D_CLEAR_ZMASK 0x00003200 +# define RADEON_CP_INDX_BUFFER 0x00003300 +# define RADEON_CP_3D_DRAW_VBUF_2 0x00003400 +# define RADEON_CP_3D_DRAW_IMMD_2 0x00003500 +# define RADEON_CP_3D_DRAW_INDX_2 0x00003600 +# define RADEON_3D_CLEAR_HIZ 0x00003700 +# define RADEON_CP_3D_CLEAR_CMASK 0x00003802 +# define RADEON_CNTL_HOSTDATA_BLT 0x00009400 +# define RADEON_CNTL_PAINT_MULTI 0x00009A00 +# define RADEON_CNTL_BITBLT_MULTI 0x00009B00 +# define RADEON_CNTL_SET_SCISSORS 0xC0001E00 + +# define R600_IT_INDIRECT_BUFFER_END 0x00001700 +# define R600_IT_SET_PREDICATION 0x00002000 +# define R600_IT_REG_RMW 0x00002100 +# define R600_IT_COND_EXEC 0x00002200 +# define R600_IT_PRED_EXEC 0x00002300 +# define R600_IT_START_3D_CMDBUF 0x00002400 +# define R600_IT_DRAW_INDEX_2 0x00002700 +# define R600_IT_CONTEXT_CONTROL 0x00002800 +# define R600_IT_DRAW_INDEX_IMMD_BE 0x00002900 +# define R600_IT_INDEX_TYPE 0x00002A00 +# define R600_IT_DRAW_INDEX 0x00002B00 +# define R600_IT_DRAW_INDEX_AUTO 0x00002D00 +# define R600_IT_DRAW_INDEX_IMMD 0x00002E00 +# define R600_IT_NUM_INSTANCES 0x00002F00 +# define R600_IT_STRMOUT_BUFFER_UPDATE 0x00003400 +# define R600_IT_INDIRECT_BUFFER_MP 0x00003800 +# define R600_IT_MEM_SEMAPHORE 0x00003900 +# define R600_IT_MPEG_INDEX 0x00003A00 +# define R600_IT_WAIT_REG_MEM 0x00003C00 +# define R600_IT_MEM_WRITE 0x00003D00 +# define R600_IT_INDIRECT_BUFFER 0x00003200 +# define R600_IT_SURFACE_SYNC 0x00004300 +# define R600_CB0_DEST_BASE_ENA (1 << 6) +# define R600_TC_ACTION_ENA (1 << 23) +# define R600_VC_ACTION_ENA (1 << 24) +# define R600_CB_ACTION_ENA (1 << 25) +# define R600_DB_ACTION_ENA (1 << 26) +# define R600_SH_ACTION_ENA (1 << 27) +# define R600_SMX_ACTION_ENA (1 << 28) +# define R600_IT_ME_INITIALIZE 0x00004400 +# define R600_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16) +# define R600_IT_COND_WRITE 0x00004500 +# define R600_IT_EVENT_WRITE 0x00004600 +# define R600_IT_EVENT_WRITE_EOP 0x00004700 +# define R600_IT_ONE_REG_WRITE 0x00005700 +# define R600_IT_SET_CONFIG_REG 0x00006800 +# define R600_SET_CONFIG_REG_OFFSET 0x00008000 +# define R600_SET_CONFIG_REG_END 0x0000ac00 +# define R600_IT_SET_CONTEXT_REG 0x00006900 +# define R600_SET_CONTEXT_REG_OFFSET 0x00028000 +# define R600_SET_CONTEXT_REG_END 0x00029000 +# define R600_IT_SET_ALU_CONST 0x00006A00 +# define R600_SET_ALU_CONST_OFFSET 0x00030000 +# define R600_SET_ALU_CONST_END 0x00032000 +# define R600_IT_SET_BOOL_CONST 0x00006B00 +# define R600_SET_BOOL_CONST_OFFSET 0x0003e380 +# define R600_SET_BOOL_CONST_END 0x00040000 +# define R600_IT_SET_LOOP_CONST 0x00006C00 +# define R600_SET_LOOP_CONST_OFFSET 0x0003e200 +# define R600_SET_LOOP_CONST_END 0x0003e380 +# define R600_IT_SET_RESOURCE 0x00006D00 +# define R600_SET_RESOURCE_OFFSET 0x00038000 +# define R600_SET_RESOURCE_END 0x0003c000 +# define R600_SQ_TEX_VTX_INVALID_TEXTURE 0x0 +# define R600_SQ_TEX_VTX_INVALID_BUFFER 0x1 +# define R600_SQ_TEX_VTX_VALID_TEXTURE 0x2 +# define R600_SQ_TEX_VTX_VALID_BUFFER 0x3 +# define R600_IT_SET_SAMPLER 0x00006E00 +# define R600_SET_SAMPLER_OFFSET 0x0003c000 +# define R600_SET_SAMPLER_END 0x0003cff0 +# define R600_IT_SET_CTL_CONST 0x00006F00 +# define R600_SET_CTL_CONST_OFFSET 0x0003cff0 +# define R600_SET_CTL_CONST_END 0x0003e200 +# define R600_IT_SURFACE_BASE_UPDATE 0x00007300 + +#define RADEON_CP_PACKET_MASK 0xC0000000 +#define RADEON_CP_PACKET_COUNT_MASK 0x3fff0000 +#define RADEON_CP_PACKET0_REG_MASK 0x000007ff +#define RADEON_CP_PACKET1_REG0_MASK 0x000007ff +#define RADEON_CP_PACKET1_REG1_MASK 0x003ff800 + +#define RADEON_VTX_Z_PRESENT (1 << 31) +#define RADEON_VTX_PKCOLOR_PRESENT (1 << 3) + +#define RADEON_PRIM_TYPE_NONE (0 << 0) +#define RADEON_PRIM_TYPE_POINT (1 << 0) +#define RADEON_PRIM_TYPE_LINE (2 << 0) +#define RADEON_PRIM_TYPE_LINE_STRIP (3 << 0) +#define RADEON_PRIM_TYPE_TRI_LIST (4 << 0) +#define RADEON_PRIM_TYPE_TRI_FAN (5 << 0) +#define RADEON_PRIM_TYPE_TRI_STRIP (6 << 0) +#define RADEON_PRIM_TYPE_TRI_TYPE2 (7 << 0) +#define RADEON_PRIM_TYPE_RECT_LIST (8 << 0) +#define RADEON_PRIM_TYPE_3VRT_POINT_LIST (9 << 0) +#define RADEON_PRIM_TYPE_3VRT_LINE_LIST (10 << 0) +#define RADEON_PRIM_TYPE_MASK 0xf +#define RADEON_PRIM_WALK_IND (1 << 4) +#define RADEON_PRIM_WALK_LIST (2 << 4) +#define RADEON_PRIM_WALK_RING (3 << 4) +#define RADEON_COLOR_ORDER_BGRA (0 << 6) +#define RADEON_COLOR_ORDER_RGBA (1 << 6) +#define RADEON_MAOS_ENABLE (1 << 7) +#define RADEON_VTX_FMT_R128_MODE (0 << 8) +#define RADEON_VTX_FMT_RADEON_MODE (1 << 8) +#define RADEON_NUM_VERTICES_SHIFT 16 + +#define RADEON_COLOR_FORMAT_CI8 2 +#define RADEON_COLOR_FORMAT_ARGB1555 3 +#define RADEON_COLOR_FORMAT_RGB565 4 +#define RADEON_COLOR_FORMAT_ARGB8888 6 +#define RADEON_COLOR_FORMAT_RGB332 7 +#define RADEON_COLOR_FORMAT_RGB8 9 +#define RADEON_COLOR_FORMAT_ARGB4444 15 + +#define RADEON_TXFORMAT_I8 0 +#define RADEON_TXFORMAT_AI88 1 +#define RADEON_TXFORMAT_RGB332 2 +#define RADEON_TXFORMAT_ARGB1555 3 +#define RADEON_TXFORMAT_RGB565 4 +#define RADEON_TXFORMAT_ARGB4444 5 +#define RADEON_TXFORMAT_ARGB8888 6 +#define RADEON_TXFORMAT_RGBA8888 7 +#define RADEON_TXFORMAT_Y8 8 +#define RADEON_TXFORMAT_VYUY422 10 +#define RADEON_TXFORMAT_YVYU422 11 +#define RADEON_TXFORMAT_DXT1 12 +#define RADEON_TXFORMAT_DXT23 14 +#define RADEON_TXFORMAT_DXT45 15 + +#define R200_PP_TXCBLEND_0 0x2f00 +#define R200_PP_TXCBLEND_1 0x2f10 +#define R200_PP_TXCBLEND_2 0x2f20 +#define R200_PP_TXCBLEND_3 0x2f30 +#define R200_PP_TXCBLEND_4 0x2f40 +#define R200_PP_TXCBLEND_5 0x2f50 +#define R200_PP_TXCBLEND_6 0x2f60 +#define R200_PP_TXCBLEND_7 0x2f70 +#define R200_SE_TCL_LIGHT_MODEL_CTL_0 0x2268 +#define R200_PP_TFACTOR_0 0x2ee0 +#define R200_SE_VTX_FMT_0 0x2088 +#define R200_SE_VAP_CNTL 0x2080 +#define R200_SE_TCL_MATRIX_SEL_0 0x2230 +#define R200_SE_TCL_TEX_PROC_CTL_2 0x22a8 +#define R200_SE_TCL_UCP_VERT_BLEND_CTL 0x22c0 +#define R200_PP_TXFILTER_5 0x2ca0 +#define R200_PP_TXFILTER_4 0x2c80 +#define R200_PP_TXFILTER_3 0x2c60 +#define R200_PP_TXFILTER_2 0x2c40 +#define R200_PP_TXFILTER_1 0x2c20 +#define R200_PP_TXFILTER_0 0x2c00 +#define R200_PP_TXOFFSET_5 0x2d78 +#define R200_PP_TXOFFSET_4 0x2d60 +#define R200_PP_TXOFFSET_3 0x2d48 +#define R200_PP_TXOFFSET_2 0x2d30 +#define R200_PP_TXOFFSET_1 0x2d18 +#define R200_PP_TXOFFSET_0 0x2d00 + +#define R200_PP_CUBIC_FACES_0 0x2c18 +#define R200_PP_CUBIC_FACES_1 0x2c38 +#define R200_PP_CUBIC_FACES_2 0x2c58 +#define R200_PP_CUBIC_FACES_3 0x2c78 +#define R200_PP_CUBIC_FACES_4 0x2c98 +#define R200_PP_CUBIC_FACES_5 0x2cb8 +#define R200_PP_CUBIC_OFFSET_F1_0 0x2d04 +#define R200_PP_CUBIC_OFFSET_F2_0 0x2d08 +#define R200_PP_CUBIC_OFFSET_F3_0 0x2d0c +#define R200_PP_CUBIC_OFFSET_F4_0 0x2d10 +#define R200_PP_CUBIC_OFFSET_F5_0 0x2d14 +#define R200_PP_CUBIC_OFFSET_F1_1 0x2d1c +#define R200_PP_CUBIC_OFFSET_F2_1 0x2d20 +#define R200_PP_CUBIC_OFFSET_F3_1 0x2d24 +#define R200_PP_CUBIC_OFFSET_F4_1 0x2d28 +#define R200_PP_CUBIC_OFFSET_F5_1 0x2d2c +#define R200_PP_CUBIC_OFFSET_F1_2 0x2d34 +#define R200_PP_CUBIC_OFFSET_F2_2 0x2d38 +#define R200_PP_CUBIC_OFFSET_F3_2 0x2d3c +#define R200_PP_CUBIC_OFFSET_F4_2 0x2d40 +#define R200_PP_CUBIC_OFFSET_F5_2 0x2d44 +#define R200_PP_CUBIC_OFFSET_F1_3 0x2d4c +#define R200_PP_CUBIC_OFFSET_F2_3 0x2d50 +#define R200_PP_CUBIC_OFFSET_F3_3 0x2d54 +#define R200_PP_CUBIC_OFFSET_F4_3 0x2d58 +#define R200_PP_CUBIC_OFFSET_F5_3 0x2d5c +#define R200_PP_CUBIC_OFFSET_F1_4 0x2d64 +#define R200_PP_CUBIC_OFFSET_F2_4 0x2d68 +#define R200_PP_CUBIC_OFFSET_F3_4 0x2d6c +#define R200_PP_CUBIC_OFFSET_F4_4 0x2d70 +#define R200_PP_CUBIC_OFFSET_F5_4 0x2d74 +#define R200_PP_CUBIC_OFFSET_F1_5 0x2d7c +#define R200_PP_CUBIC_OFFSET_F2_5 0x2d80 +#define R200_PP_CUBIC_OFFSET_F3_5 0x2d84 +#define R200_PP_CUBIC_OFFSET_F4_5 0x2d88 +#define R200_PP_CUBIC_OFFSET_F5_5 0x2d8c + +#define R200_RE_AUX_SCISSOR_CNTL 0x26f0 +#define R200_SE_VTE_CNTL 0x20b0 +#define R200_SE_TCL_OUTPUT_VTX_COMP_SEL 0x2250 +#define R200_PP_TAM_DEBUG3 0x2d9c +#define R200_PP_CNTL_X 0x2cc4 +#define R200_SE_VAP_CNTL_STATUS 0x2140 +#define R200_RE_SCISSOR_TL_0 0x1cd8 +#define R200_RE_SCISSOR_TL_1 0x1ce0 +#define R200_RE_SCISSOR_TL_2 0x1ce8 +#define R200_RB3D_DEPTHXY_OFFSET 0x1d60 +#define R200_RE_AUX_SCISSOR_CNTL 0x26f0 +#define R200_SE_VTX_STATE_CNTL 0x2180 +#define R200_RE_POINTSIZE 0x2648 +#define R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0 0x2254 + +#define RADEON_PP_TEX_SIZE_0 0x1d04 /* NPOT */ +#define RADEON_PP_TEX_SIZE_1 0x1d0c +#define RADEON_PP_TEX_SIZE_2 0x1d14 + +#define RADEON_PP_CUBIC_FACES_0 0x1d24 +#define RADEON_PP_CUBIC_FACES_1 0x1d28 +#define RADEON_PP_CUBIC_FACES_2 0x1d2c +#define RADEON_PP_CUBIC_OFFSET_T0_0 0x1dd0 /* bits [31:5] */ +#define RADEON_PP_CUBIC_OFFSET_T1_0 0x1e00 +#define RADEON_PP_CUBIC_OFFSET_T2_0 0x1e14 + +#define RADEON_SE_TCL_STATE_FLUSH 0x2284 + +#define SE_VAP_CNTL__TCL_ENA_MASK 0x00000001 +#define SE_VAP_CNTL__FORCE_W_TO_ONE_MASK 0x00010000 +#define SE_VAP_CNTL__VF_MAX_VTX_NUM__SHIFT 0x00000012 +#define SE_VTE_CNTL__VTX_XY_FMT_MASK 0x00000100 +#define SE_VTE_CNTL__VTX_Z_FMT_MASK 0x00000200 +#define SE_VTX_FMT_0__VTX_Z0_PRESENT_MASK 0x00000001 +#define SE_VTX_FMT_0__VTX_W0_PRESENT_MASK 0x00000002 +#define SE_VTX_FMT_0__VTX_COLOR_0_FMT__SHIFT 0x0000000b +#define R200_3D_DRAW_IMMD_2 0xC0003500 +#define R200_SE_VTX_FMT_1 0x208c +#define R200_RE_CNTL 0x1c50 + +#define R200_RB3D_BLENDCOLOR 0x3218 + +#define R200_SE_TCL_POINT_SPRITE_CNTL 0x22c4 + +#define R200_PP_TRI_PERF 0x2cf8 + +#define R200_PP_AFS_0 0x2f80 +#define R200_PP_AFS_1 0x2f00 /* same as txcblend_0 */ + +#define R200_VAP_PVS_CNTL_1 0x22D0 + +#define RADEON_CRTC_CRNT_FRAME 0x0214 +#define RADEON_CRTC2_CRNT_FRAME 0x0314 + +#define R500_D1CRTC_STATUS 0x609c +#define R500_D2CRTC_STATUS 0x689c +#define R500_CRTC_V_BLANK (1<<0) + +#define R500_D1CRTC_FRAME_COUNT 0x60a4 +#define R500_D2CRTC_FRAME_COUNT 0x68a4 + +#define R500_D1MODE_V_COUNTER 0x6530 +#define R500_D2MODE_V_COUNTER 0x6d30 + +#define R500_D1MODE_VBLANK_STATUS 0x6534 +#define R500_D2MODE_VBLANK_STATUS 0x6d34 +#define R500_VBLANK_OCCURED (1<<0) +#define R500_VBLANK_ACK (1<<4) +#define R500_VBLANK_STAT (1<<12) +#define R500_VBLANK_INT (1<<16) + +#define R500_DxMODE_INT_MASK 0x6540 +#define R500_D1MODE_INT_MASK (1<<0) +#define R500_D2MODE_INT_MASK (1<<8) + +#define R500_DISP_INTERRUPT_STATUS 0x7edc +#define R500_D1_VBLANK_INTERRUPT (1 << 4) +#define R500_D2_VBLANK_INTERRUPT (1 << 5) + +/* R6xx/R7xx registers */ +#define R600_MC_VM_FB_LOCATION 0x2180 +#define R600_MC_VM_AGP_TOP 0x2184 +#define R600_MC_VM_AGP_BOT 0x2188 +#define R600_MC_VM_AGP_BASE 0x218c +#define R600_MC_VM_SYSTEM_APERTURE_LOW_ADDR 0x2190 +#define R600_MC_VM_SYSTEM_APERTURE_HIGH_ADDR 0x2194 +#define R600_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR 0x2198 + +#define R700_MC_VM_FB_LOCATION 0x2024 +#define R700_MC_VM_AGP_TOP 0x2028 +#define R700_MC_VM_AGP_BOT 0x202c +#define R700_MC_VM_AGP_BASE 0x2030 +#define R700_MC_VM_SYSTEM_APERTURE_LOW_ADDR 0x2034 +#define R700_MC_VM_SYSTEM_APERTURE_HIGH_ADDR 0x2038 +#define R700_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR 0x203c + +#define R600_MCD_RD_A_CNTL 0x219c +#define R600_MCD_RD_B_CNTL 0x21a0 + +#define R600_MCD_WR_A_CNTL 0x21a4 +#define R600_MCD_WR_B_CNTL 0x21a8 + +#define R600_MCD_RD_SYS_CNTL 0x2200 +#define R600_MCD_WR_SYS_CNTL 0x2214 + +#define R600_MCD_RD_GFX_CNTL 0x21fc +#define R600_MCD_RD_HDP_CNTL 0x2204 +#define R600_MCD_RD_PDMA_CNTL 0x2208 +#define R600_MCD_RD_SEM_CNTL 0x220c +#define R600_MCD_WR_GFX_CNTL 0x2210 +#define R600_MCD_WR_HDP_CNTL 0x2218 +#define R600_MCD_WR_PDMA_CNTL 0x221c +#define R600_MCD_WR_SEM_CNTL 0x2220 + +# define R600_MCD_L1_TLB (1 << 0) +# define R600_MCD_L1_FRAG_PROC (1 << 1) +# define R600_MCD_L1_STRICT_ORDERING (1 << 2) + +# define R600_MCD_SYSTEM_ACCESS_MODE_MASK (3 << 6) +# define R600_MCD_SYSTEM_ACCESS_MODE_PA_ONLY (0 << 6) +# define R600_MCD_SYSTEM_ACCESS_MODE_USE_SYS_MAP (1 << 6) +# define R600_MCD_SYSTEM_ACCESS_MODE_IN_SYS (2 << 6) +# define R600_MCD_SYSTEM_ACCESS_MODE_NOT_IN_SYS (3 << 6) + +# define R600_MCD_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU (0 << 8) +# define R600_MCD_SYSTEM_APERTURE_UNMAPPED_ACCESS_DEFAULT_PAGE (1 << 8) + +# define R600_MCD_SEMAPHORE_MODE (1 << 10) +# define R600_MCD_WAIT_L2_QUERY (1 << 11) +# define R600_MCD_EFFECTIVE_L1_TLB_SIZE(x) ((x) << 12) +# define R600_MCD_EFFECTIVE_L1_QUEUE_SIZE(x) ((x) << 15) + +#define R700_MC_VM_MD_L1_TLB0_CNTL 0x2654 +#define R700_MC_VM_MD_L1_TLB1_CNTL 0x2658 +#define R700_MC_VM_MD_L1_TLB2_CNTL 0x265c + +#define R700_MC_VM_MB_L1_TLB0_CNTL 0x2234 +#define R700_MC_VM_MB_L1_TLB1_CNTL 0x2238 +#define R700_MC_VM_MB_L1_TLB2_CNTL 0x223c +#define R700_MC_VM_MB_L1_TLB3_CNTL 0x2240 + +# define R700_ENABLE_L1_TLB (1 << 0) +# define R700_ENABLE_L1_FRAGMENT_PROCESSING (1 << 1) +# define R700_SYSTEM_ACCESS_MODE_IN_SYS (2 << 3) +# define R700_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU (0 << 5) +# define R700_EFFECTIVE_L1_TLB_SIZE(x) ((x) << 15) +# define R700_EFFECTIVE_L1_QUEUE_SIZE(x) ((x) << 18) + +#define R700_MC_ARB_RAMCFG 0x2760 +# define R700_NOOFBANK_SHIFT 0 +# define R700_NOOFBANK_MASK 0x3 +# define R700_NOOFRANK_SHIFT 2 +# define R700_NOOFRANK_MASK 0x1 +# define R700_NOOFROWS_SHIFT 3 +# define R700_NOOFROWS_MASK 0x7 +# define R700_NOOFCOLS_SHIFT 6 +# define R700_NOOFCOLS_MASK 0x3 +# define R700_CHANSIZE_SHIFT 8 +# define R700_CHANSIZE_MASK 0x1 +# define R700_BURSTLENGTH_SHIFT 9 +# define R700_BURSTLENGTH_MASK 0x1 +#define R600_RAMCFG 0x2408 +# define R600_NOOFBANK_SHIFT 0 +# define R600_NOOFBANK_MASK 0x1 +# define R600_NOOFRANK_SHIFT 1 +# define R600_NOOFRANK_MASK 0x1 +# define R600_NOOFROWS_SHIFT 2 +# define R600_NOOFROWS_MASK 0x7 +# define R600_NOOFCOLS_SHIFT 5 +# define R600_NOOFCOLS_MASK 0x3 +# define R600_CHANSIZE_SHIFT 7 +# define R600_CHANSIZE_MASK 0x1 +# define R600_BURSTLENGTH_SHIFT 8 +# define R600_BURSTLENGTH_MASK 0x1 + +#define R600_VM_L2_CNTL 0x1400 +# define R600_VM_L2_CACHE_EN (1 << 0) +# define R600_VM_L2_FRAG_PROC (1 << 1) +# define R600_VM_ENABLE_PTE_CACHE_LRU_W (1 << 9) +# define R600_VM_L2_CNTL_QUEUE_SIZE(x) ((x) << 13) +# define R700_VM_L2_CNTL_QUEUE_SIZE(x) ((x) << 14) + +#define R600_VM_L2_CNTL2 0x1404 +# define R600_VM_L2_CNTL2_INVALIDATE_ALL_L1_TLBS (1 << 0) +# define R600_VM_L2_CNTL2_INVALIDATE_L2_CACHE (1 << 1) +#define R600_VM_L2_CNTL3 0x1408 +# define R600_VM_L2_CNTL3_BANK_SELECT_0(x) ((x) << 0) +# define R600_VM_L2_CNTL3_BANK_SELECT_1(x) ((x) << 5) +# define R600_VM_L2_CNTL3_CACHE_UPDATE_MODE(x) ((x) << 10) +# define R700_VM_L2_CNTL3_BANK_SELECT(x) ((x) << 0) +# define R700_VM_L2_CNTL3_CACHE_UPDATE_MODE(x) ((x) << 6) + +#define R600_VM_L2_STATUS 0x140c + +#define R600_VM_CONTEXT0_CNTL 0x1410 +# define R600_VM_ENABLE_CONTEXT (1 << 0) +# define R600_VM_PAGE_TABLE_DEPTH_FLAT (0 << 1) + +#define R600_VM_CONTEXT0_CNTL2 0x1430 +#define R600_VM_CONTEXT0_REQUEST_RESPONSE 0x1470 +#define R600_VM_CONTEXT0_INVALIDATION_LOW_ADDR 0x1490 +#define R600_VM_CONTEXT0_INVALIDATION_HIGH_ADDR 0x14b0 +#define R600_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR 0x1574 +#define R600_VM_CONTEXT0_PAGE_TABLE_START_ADDR 0x1594 +#define R600_VM_CONTEXT0_PAGE_TABLE_END_ADDR 0x15b4 + +#define R700_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR 0x153c +#define R700_VM_CONTEXT0_PAGE_TABLE_START_ADDR 0x155c +#define R700_VM_CONTEXT0_PAGE_TABLE_END_ADDR 0x157c + +#define R600_HDP_HOST_PATH_CNTL 0x2c00 + +#define R600_GRBM_CNTL 0x8000 +# define R600_GRBM_READ_TIMEOUT(x) ((x) << 0) + +#define R600_GRBM_STATUS 0x8010 +# define R600_CMDFIFO_AVAIL_MASK 0x1f +# define R700_CMDFIFO_AVAIL_MASK 0xf +# define R600_GUI_ACTIVE (1 << 31) +#define R600_GRBM_STATUS2 0x8014 +#define R600_GRBM_SOFT_RESET 0x8020 +# define R600_SOFT_RESET_CP (1 << 0) +#define R600_WAIT_UNTIL 0x8040 + +#define R600_CP_SEM_WAIT_TIMER 0x85bc +#define R600_CP_ME_CNTL 0x86d8 +# define R600_CP_ME_HALT (1 << 28) +#define R600_CP_QUEUE_THRESHOLDS 0x8760 +# define R600_ROQ_IB1_START(x) ((x) << 0) +# define R600_ROQ_IB2_START(x) ((x) << 8) +#define R600_CP_MEQ_THRESHOLDS 0x8764 +# define R700_STQ_SPLIT(x) ((x) << 0) +# define R600_MEQ_END(x) ((x) << 16) +# define R600_ROQ_END(x) ((x) << 24) +#define R600_CP_PERFMON_CNTL 0x87fc +#define R600_CP_RB_BASE 0xc100 +#define R600_CP_RB_CNTL 0xc104 +# define R600_RB_BUFSZ(x) ((x) << 0) +# define R600_RB_BLKSZ(x) ((x) << 8) +# define R600_BUF_SWAP_32BIT (2 << 16) +# define R600_RB_NO_UPDATE (1 << 27) +# define R600_RB_RPTR_WR_ENA (1 << 31) +#define R600_CP_RB_RPTR_WR 0xc108 +#define R600_CP_RB_RPTR_ADDR 0xc10c +#define R600_CP_RB_RPTR_ADDR_HI 0xc110 +#define R600_CP_RB_WPTR 0xc114 +#define R600_CP_RB_WPTR_ADDR 0xc118 +#define R600_CP_RB_WPTR_ADDR_HI 0xc11c +#define R600_CP_RB_RPTR 0x8700 +#define R600_CP_RB_WPTR_DELAY 0x8704 +#define R600_CP_PFP_UCODE_ADDR 0xc150 +#define R600_CP_PFP_UCODE_DATA 0xc154 +#define R600_CP_ME_RAM_RADDR 0xc158 +#define R600_CP_ME_RAM_WADDR 0xc15c +#define R600_CP_ME_RAM_DATA 0xc160 +#define R600_CP_DEBUG 0xc1fc + +#define R600_PA_CL_ENHANCE 0x8a14 +# define R600_CLIP_VTX_REORDER_ENA (1 << 0) +# define R600_NUM_CLIP_SEQ(x) ((x) << 1) +#define R600_PA_SC_LINE_STIPPLE_STATE 0x8b10 +#define R600_PA_SC_MULTI_CHIP_CNTL 0x8b20 +#define R700_PA_SC_FORCE_EOV_MAX_CNTS 0x8b24 +# define R700_FORCE_EOV_MAX_CLK_CNT(x) ((x) << 0) +# define R700_FORCE_EOV_MAX_REZ_CNT(x) ((x) << 16) +#define R600_PA_SC_AA_SAMPLE_LOCS_2S 0x8b40 +#define R600_PA_SC_AA_SAMPLE_LOCS_4S 0x8b44 +#define R600_PA_SC_AA_SAMPLE_LOCS_8S_WD0 0x8b48 +#define R600_PA_SC_AA_SAMPLE_LOCS_8S_WD1 0x8b4c +# define R600_S0_X(x) ((x) << 0) +# define R600_S0_Y(x) ((x) << 4) +# define R600_S1_X(x) ((x) << 8) +# define R600_S1_Y(x) ((x) << 12) +# define R600_S2_X(x) ((x) << 16) +# define R600_S2_Y(x) ((x) << 20) +# define R600_S3_X(x) ((x) << 24) +# define R600_S3_Y(x) ((x) << 28) +# define R600_S4_X(x) ((x) << 0) +# define R600_S4_Y(x) ((x) << 4) +# define R600_S5_X(x) ((x) << 8) +# define R600_S5_Y(x) ((x) << 12) +# define R600_S6_X(x) ((x) << 16) +# define R600_S6_Y(x) ((x) << 20) +# define R600_S7_X(x) ((x) << 24) +# define R600_S7_Y(x) ((x) << 28) +#define R600_PA_SC_FIFO_SIZE 0x8bd0 +# define R600_SC_PRIM_FIFO_SIZE(x) ((x) << 0) +# define R600_SC_HIZ_TILE_FIFO_SIZE(x) ((x) << 8) +# define R600_SC_EARLYZ_TILE_FIFO_SIZE(x) ((x) << 16) +#define R700_PA_SC_FIFO_SIZE_R7XX 0x8bcc +# define R700_SC_PRIM_FIFO_SIZE(x) ((x) << 0) +# define R700_SC_HIZ_TILE_FIFO_SIZE(x) ((x) << 12) +# define R700_SC_EARLYZ_TILE_FIFO_SIZE(x) ((x) << 20) +#define R600_PA_SC_ENHANCE 0x8bf0 +# define R600_FORCE_EOV_MAX_CLK_CNT(x) ((x) << 0) +# define R600_FORCE_EOV_MAX_TILE_CNT(x) ((x) << 12) +#define R600_PA_SC_CLIPRECT_RULE 0x2820c +#define R700_PA_SC_EDGERULE 0x28230 +#define R600_PA_SC_LINE_STIPPLE 0x28a0c +#define R600_PA_SC_MODE_CNTL 0x28a4c +#define R600_PA_SC_AA_CONFIG 0x28c04 + +#define R600_SX_EXPORT_BUFFER_SIZES 0x900c +# define R600_COLOR_BUFFER_SIZE(x) ((x) << 0) +# define R600_POSITION_BUFFER_SIZE(x) ((x) << 8) +# define R600_SMX_BUFFER_SIZE(x) ((x) << 16) +#define R600_SX_DEBUG_1 0x9054 +# define R600_SMX_EVENT_RELEASE (1 << 0) +# define R600_ENABLE_NEW_SMX_ADDRESS (1 << 16) +#define R700_SX_DEBUG_1 0x9058 +# define R700_ENABLE_NEW_SMX_ADDRESS (1 << 16) +#define R600_SX_MISC 0x28350 + +#define R600_DB_DEBUG 0x9830 +# define R600_PREZ_MUST_WAIT_FOR_POSTZ_DONE (1 << 31) +#define R600_DB_WATERMARKS 0x9838 +# define R600_DEPTH_FREE(x) ((x) << 0) +# define R600_DEPTH_FLUSH(x) ((x) << 5) +# define R600_DEPTH_PENDING_FREE(x) ((x) << 15) +# define R600_DEPTH_CACHELINE_FREE(x) ((x) << 20) +#define R700_DB_DEBUG3 0x98b0 +# define R700_DB_CLK_OFF_DELAY(x) ((x) << 11) +#define RV700_DB_DEBUG4 0x9b8c +# define RV700_DISABLE_TILE_COVERED_FOR_PS_ITER (1 << 6) + +#define R600_VGT_CACHE_INVALIDATION 0x88c4 +# define R600_CACHE_INVALIDATION(x) ((x) << 0) +# define R600_VC_ONLY 0 +# define R600_TC_ONLY 1 +# define R600_VC_AND_TC 2 +# define R700_AUTO_INVLD_EN(x) ((x) << 6) +# define R700_NO_AUTO 0 +# define R700_ES_AUTO 1 +# define R700_GS_AUTO 2 +# define R700_ES_AND_GS_AUTO 3 +#define R600_VGT_GS_PER_ES 0x88c8 +#define R600_VGT_ES_PER_GS 0x88cc +#define R600_VGT_GS_PER_VS 0x88e8 +#define R600_VGT_GS_VERTEX_REUSE 0x88d4 +#define R600_VGT_NUM_INSTANCES 0x8974 +#define R600_VGT_STRMOUT_EN 0x28ab0 +#define R600_VGT_EVENT_INITIATOR 0x28a90 +# define R600_CACHE_FLUSH_AND_INV_EVENT (0x16 << 0) +#define R600_VGT_VERTEX_REUSE_BLOCK_CNTL 0x28c58 +# define R600_VTX_REUSE_DEPTH_MASK 0xff +#define R600_VGT_OUT_DEALLOC_CNTL 0x28c5c +# define R600_DEALLOC_DIST_MASK 0x7f + +#define R600_CB_COLOR0_BASE 0x28040 +#define R600_CB_COLOR1_BASE 0x28044 +#define R600_CB_COLOR2_BASE 0x28048 +#define R600_CB_COLOR3_BASE 0x2804c +#define R600_CB_COLOR4_BASE 0x28050 +#define R600_CB_COLOR5_BASE 0x28054 +#define R600_CB_COLOR6_BASE 0x28058 +#define R600_CB_COLOR7_BASE 0x2805c +#define R600_CB_COLOR7_FRAG 0x280fc + +#define R600_CB_COLOR0_SIZE 0x28060 +#define R600_CB_COLOR0_VIEW 0x28080 +#define R600_CB_COLOR0_INFO 0x280a0 +#define R600_CB_COLOR0_TILE 0x280c0 +#define R600_CB_COLOR0_FRAG 0x280e0 +#define R600_CB_COLOR0_MASK 0x28100 + +#define AVIVO_D1MODE_VLINE_START_END 0x6538 +#define AVIVO_D2MODE_VLINE_START_END 0x6d38 +#define R600_CP_COHER_BASE 0x85f8 +#define R600_DB_DEPTH_BASE 0x2800c +#define R600_SQ_PGM_START_FS 0x28894 +#define R600_SQ_PGM_START_ES 0x28880 +#define R600_SQ_PGM_START_VS 0x28858 +#define R600_SQ_PGM_RESOURCES_VS 0x28868 +#define R600_SQ_PGM_CF_OFFSET_VS 0x288d0 +#define R600_SQ_PGM_START_GS 0x2886c +#define R600_SQ_PGM_START_PS 0x28840 +#define R600_SQ_PGM_RESOURCES_PS 0x28850 +#define R600_SQ_PGM_EXPORTS_PS 0x28854 +#define R600_SQ_PGM_CF_OFFSET_PS 0x288cc +#define R600_VGT_DMA_BASE 0x287e8 +#define R600_VGT_DMA_BASE_HI 0x287e4 +#define R600_VGT_STRMOUT_BASE_OFFSET_0 0x28b10 +#define R600_VGT_STRMOUT_BASE_OFFSET_1 0x28b14 +#define R600_VGT_STRMOUT_BASE_OFFSET_2 0x28b18 +#define R600_VGT_STRMOUT_BASE_OFFSET_3 0x28b1c +#define R600_VGT_STRMOUT_BASE_OFFSET_HI_0 0x28b44 +#define R600_VGT_STRMOUT_BASE_OFFSET_HI_1 0x28b48 +#define R600_VGT_STRMOUT_BASE_OFFSET_HI_2 0x28b4c +#define R600_VGT_STRMOUT_BASE_OFFSET_HI_3 0x28b50 +#define R600_VGT_STRMOUT_BUFFER_BASE_0 0x28ad8 +#define R600_VGT_STRMOUT_BUFFER_BASE_1 0x28ae8 +#define R600_VGT_STRMOUT_BUFFER_BASE_2 0x28af8 +#define R600_VGT_STRMOUT_BUFFER_BASE_3 0x28b08 +#define R600_VGT_STRMOUT_BUFFER_OFFSET_0 0x28adc +#define R600_VGT_STRMOUT_BUFFER_OFFSET_1 0x28aec +#define R600_VGT_STRMOUT_BUFFER_OFFSET_2 0x28afc +#define R600_VGT_STRMOUT_BUFFER_OFFSET_3 0x28b0c + +#define R600_VGT_PRIMITIVE_TYPE 0x8958 + +#define R600_PA_SC_SCREEN_SCISSOR_TL 0x28030 +#define R600_PA_SC_GENERIC_SCISSOR_TL 0x28240 +#define R600_PA_SC_WINDOW_SCISSOR_TL 0x28204 + +#define R600_TC_CNTL 0x9608 +# define R600_TC_L2_SIZE(x) ((x) << 5) +# define R600_L2_DISABLE_LATE_HIT (1 << 9) + +#define R600_ARB_POP 0x2418 +# define R600_ENABLE_TC128 (1 << 30) +#define R600_ARB_GDEC_RD_CNTL 0x246c + +#define R600_TA_CNTL_AUX 0x9508 +# define R600_DISABLE_CUBE_WRAP (1 << 0) +# define R600_DISABLE_CUBE_ANISO (1 << 1) +# define R700_GETLOD_SELECT(x) ((x) << 2) +# define R600_SYNC_GRADIENT (1 << 24) +# define R600_SYNC_WALKER (1 << 25) +# define R600_SYNC_ALIGNER (1 << 26) +# define R600_BILINEAR_PRECISION_6_BIT (0 << 31) +# define R600_BILINEAR_PRECISION_8_BIT (1 << 31) + +#define R700_TCP_CNTL 0x9610 + +#define R600_SMX_DC_CTL0 0xa020 +# define R700_USE_HASH_FUNCTION (1 << 0) +# define R700_CACHE_DEPTH(x) ((x) << 1) +# define R700_FLUSH_ALL_ON_EVENT (1 << 10) +# define R700_STALL_ON_EVENT (1 << 11) +#define R700_SMX_EVENT_CTL 0xa02c +# define R700_ES_FLUSH_CTL(x) ((x) << 0) +# define R700_GS_FLUSH_CTL(x) ((x) << 3) +# define R700_ACK_FLUSH_CTL(x) ((x) << 6) +# define R700_SYNC_FLUSH_CTL (1 << 8) + +#define R600_SQ_CONFIG 0x8c00 +# define R600_VC_ENABLE (1 << 0) +# define R600_EXPORT_SRC_C (1 << 1) +# define R600_DX9_CONSTS (1 << 2) +# define R600_ALU_INST_PREFER_VECTOR (1 << 3) +# define R600_DX10_CLAMP (1 << 4) +# define R600_CLAUSE_SEQ_PRIO(x) ((x) << 8) +# define R600_PS_PRIO(x) ((x) << 24) +# define R600_VS_PRIO(x) ((x) << 26) +# define R600_GS_PRIO(x) ((x) << 28) +# define R600_ES_PRIO(x) ((x) << 30) +#define R600_SQ_GPR_RESOURCE_MGMT_1 0x8c04 +# define R600_NUM_PS_GPRS(x) ((x) << 0) +# define R600_NUM_VS_GPRS(x) ((x) << 16) +# define R700_DYN_GPR_ENABLE (1 << 27) +# define R600_NUM_CLAUSE_TEMP_GPRS(x) ((x) << 28) +#define R600_SQ_GPR_RESOURCE_MGMT_2 0x8c08 +# define R600_NUM_GS_GPRS(x) ((x) << 0) +# define R600_NUM_ES_GPRS(x) ((x) << 16) +#define R600_SQ_THREAD_RESOURCE_MGMT 0x8c0c +# define R600_NUM_PS_THREADS(x) ((x) << 0) +# define R600_NUM_VS_THREADS(x) ((x) << 8) +# define R600_NUM_GS_THREADS(x) ((x) << 16) +# define R600_NUM_ES_THREADS(x) ((x) << 24) +#define R600_SQ_STACK_RESOURCE_MGMT_1 0x8c10 +# define R600_NUM_PS_STACK_ENTRIES(x) ((x) << 0) +# define R600_NUM_VS_STACK_ENTRIES(x) ((x) << 16) +#define R600_SQ_STACK_RESOURCE_MGMT_2 0x8c14 +# define R600_NUM_GS_STACK_ENTRIES(x) ((x) << 0) +# define R600_NUM_ES_STACK_ENTRIES(x) ((x) << 16) +#define R600_SQ_MS_FIFO_SIZES 0x8cf0 +# define R600_CACHE_FIFO_SIZE(x) ((x) << 0) +# define R600_FETCH_FIFO_HIWATER(x) ((x) << 8) +# define R600_DONE_FIFO_HIWATER(x) ((x) << 16) +# define R600_ALU_UPDATE_FIFO_HIWATER(x) ((x) << 24) +#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_0 0x8db0 +# define R700_SIMDA_RING0(x) ((x) << 0) +# define R700_SIMDA_RING1(x) ((x) << 8) +# define R700_SIMDB_RING0(x) ((x) << 16) +# define R700_SIMDB_RING1(x) ((x) << 24) +#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_1 0x8db4 +#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_2 0x8db8 +#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_3 0x8dbc +#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_4 0x8dc0 +#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_5 0x8dc4 +#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_6 0x8dc8 +#define R700_SQ_DYN_GPR_SIZE_SIMD_AB_7 0x8dcc + +#define R600_SPI_PS_IN_CONTROL_0 0x286cc +# define R600_NUM_INTERP(x) ((x) << 0) +# define R600_POSITION_ENA (1 << 8) +# define R600_POSITION_CENTROID (1 << 9) +# define R600_POSITION_ADDR(x) ((x) << 10) +# define R600_PARAM_GEN(x) ((x) << 15) +# define R600_PARAM_GEN_ADDR(x) ((x) << 19) +# define R600_BARYC_SAMPLE_CNTL(x) ((x) << 26) +# define R600_PERSP_GRADIENT_ENA (1 << 28) +# define R600_LINEAR_GRADIENT_ENA (1 << 29) +# define R600_POSITION_SAMPLE (1 << 30) +# define R600_BARYC_AT_SAMPLE_ENA (1 << 31) +#define R600_SPI_PS_IN_CONTROL_1 0x286d0 +# define R600_GEN_INDEX_PIX (1 << 0) +# define R600_GEN_INDEX_PIX_ADDR(x) ((x) << 1) +# define R600_FRONT_FACE_ENA (1 << 8) +# define R600_FRONT_FACE_CHAN(x) ((x) << 9) +# define R600_FRONT_FACE_ALL_BITS (1 << 11) +# define R600_FRONT_FACE_ADDR(x) ((x) << 12) +# define R600_FOG_ADDR(x) ((x) << 17) +# define R600_FIXED_PT_POSITION_ENA (1 << 24) +# define R600_FIXED_PT_POSITION_ADDR(x) ((x) << 25) +# define R700_POSITION_ULC (1 << 30) +#define R600_SPI_INPUT_Z 0x286d8 + +#define R600_SPI_CONFIG_CNTL 0x9100 +# define R600_GPR_WRITE_PRIORITY(x) ((x) << 0) +# define R600_DISABLE_INTERP_1 (1 << 5) +#define R600_SPI_CONFIG_CNTL_1 0x913c +# define R600_VTX_DONE_DELAY(x) ((x) << 0) +# define R600_INTERP_ONE_PRIM_PER_ROW (1 << 4) + +#define R600_GB_TILING_CONFIG 0x98f0 +# define R600_PIPE_TILING(x) ((x) << 1) +# define R600_BANK_TILING(x) ((x) << 4) +# define R600_GROUP_SIZE(x) ((x) << 6) +# define R600_ROW_TILING(x) ((x) << 8) +# define R600_BANK_SWAPS(x) ((x) << 11) +# define R600_SAMPLE_SPLIT(x) ((x) << 14) +# define R600_BACKEND_MAP(x) ((x) << 16) +#define R600_DCP_TILING_CONFIG 0x6ca0 +#define R600_HDP_TILING_CONFIG 0x2f3c + +#define R600_CC_RB_BACKEND_DISABLE 0x98f4 +#define R700_CC_SYS_RB_BACKEND_DISABLE 0x3f88 +# define R600_BACKEND_DISABLE(x) ((x) << 16) + +#define R600_CC_GC_SHADER_PIPE_CONFIG 0x8950 +#define R600_GC_USER_SHADER_PIPE_CONFIG 0x8954 +# define R600_INACTIVE_QD_PIPES(x) ((x) << 8) +# define R600_INACTIVE_QD_PIPES_MASK (0xff << 8) +# define R600_INACTIVE_SIMDS(x) ((x) << 16) +# define R600_INACTIVE_SIMDS_MASK (0xff << 16) + +#define R700_CGTS_SYS_TCC_DISABLE 0x3f90 +#define R700_CGTS_USER_SYS_TCC_DISABLE 0x3f94 +#define R700_CGTS_TCC_DISABLE 0x9148 +#define R700_CGTS_USER_TCC_DISABLE 0x914c + +/* Constants */ +#define RADEON_MAX_USEC_TIMEOUT 100000 /* 100 ms */ + +#define RADEON_LAST_FRAME_REG RADEON_SCRATCH_REG0 +#define RADEON_LAST_DISPATCH_REG RADEON_SCRATCH_REG1 +#define RADEON_LAST_CLEAR_REG RADEON_SCRATCH_REG2 +#define RADEON_LAST_SWI_REG RADEON_SCRATCH_REG3 +#define RADEON_LAST_DISPATCH 1 + +#define R600_LAST_FRAME_REG R600_SCRATCH_REG0 +#define R600_LAST_DISPATCH_REG R600_SCRATCH_REG1 +#define R600_LAST_CLEAR_REG R600_SCRATCH_REG2 +#define R600_LAST_SWI_REG R600_SCRATCH_REG3 + +#define RADEON_MAX_VB_AGE 0x7fffffff +#define RADEON_MAX_VB_VERTS (0xffff) + +#define RADEON_RING_HIGH_MARK 128 + +#define RADEON_PCIGART_TABLE_SIZE (32*1024) + +#define RADEON_READ(reg) DRM_READ32( dev_priv->mmio, (reg) ) +#define RADEON_WRITE(reg, val) \ +do { \ + if (reg < 0x10000) { \ + DRM_WRITE32(dev_priv->mmio, (reg), (val)); \ + } else { \ + DRM_WRITE32(dev_priv->mmio, RADEON_MM_INDEX, (reg)); \ + DRM_WRITE32(dev_priv->mmio, RADEON_MM_DATA, (val)); \ + } \ +} while (0) +#define RADEON_READ8(reg) DRM_READ8( dev_priv->mmio, (reg) ) +#define RADEON_WRITE8(reg,val) DRM_WRITE8( dev_priv->mmio, (reg), (val) ) + +#define RADEON_WRITE_PLL(addr, val) \ +do { \ + RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, \ + ((addr) & 0x1f) | RADEON_PLL_WR_EN ); \ + RADEON_WRITE(RADEON_CLOCK_CNTL_DATA, (val)); \ +} while (0) + +#define RADEON_WRITE_PCIE(addr, val) \ +do { \ + RADEON_WRITE8(RADEON_PCIE_INDEX, \ + ((addr) & 0xff)); \ + RADEON_WRITE(RADEON_PCIE_DATA, (val)); \ +} while (0) + +#define R500_WRITE_MCIND(addr, val) \ +do { \ + RADEON_WRITE(R520_MC_IND_INDEX, 0xff0000 | ((addr) & 0xff)); \ + RADEON_WRITE(R520_MC_IND_DATA, (val)); \ + RADEON_WRITE(R520_MC_IND_INDEX, 0); \ +} while (0) + +#define RS480_WRITE_MCIND(addr, val) \ +do { \ + RADEON_WRITE(RS480_NB_MC_INDEX, \ + ((addr) & 0xff) | RS480_NB_MC_IND_WR_EN); \ + RADEON_WRITE(RS480_NB_MC_DATA, (val)); \ + RADEON_WRITE(RS480_NB_MC_INDEX, 0xff); \ +} while (0) + +#define RS690_WRITE_MCIND(addr, val) \ +do { \ + RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_WR_EN | ((addr) & RS690_MC_INDEX_MASK)); \ + RADEON_WRITE(RS690_MC_DATA, val); \ + RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_WR_ACK); \ +} while (0) + +#define RS600_WRITE_MCIND(addr, val) \ +do { \ + RADEON_WRITE(RS600_MC_INDEX, RS600_MC_IND_WR_EN | RS600_MC_IND_CITF_ARB0 | ((addr) & RS600_MC_ADDR_MASK)); \ + RADEON_WRITE(RS600_MC_DATA, val); \ +} while (0) + +#define IGP_WRITE_MCIND(addr, val) \ +do { \ + if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) || \ + ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) \ + RS690_WRITE_MCIND(addr, val); \ + else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) \ + RS600_WRITE_MCIND(addr, val); \ + else \ + RS480_WRITE_MCIND(addr, val); \ +} while (0) + +#define CP_PACKET0( reg, n ) \ + (RADEON_CP_PACKET0 | ((n) << 16) | ((reg) >> 2)) +#define CP_PACKET0_TABLE( reg, n ) \ + (RADEON_CP_PACKET0 | RADEON_ONE_REG_WR | ((n) << 16) | ((reg) >> 2)) +#define CP_PACKET1( reg0, reg1 ) \ + (RADEON_CP_PACKET1 | (((reg1) >> 2) << 15) | ((reg0) >> 2)) +#define CP_PACKET2() \ + (RADEON_CP_PACKET2) +#define CP_PACKET3( pkt, n ) \ + (RADEON_CP_PACKET3 | (pkt) | ((n) << 16)) + +/* ================================================================ + * Engine control helper macros + */ + +#define RADEON_WAIT_UNTIL_2D_IDLE() do { \ + OUT_RING( CP_PACKET0( RADEON_WAIT_UNTIL, 0 ) ); \ + OUT_RING( (RADEON_WAIT_2D_IDLECLEAN | \ + RADEON_WAIT_HOST_IDLECLEAN) ); \ +} while (0) + +#define RADEON_WAIT_UNTIL_3D_IDLE() do { \ + OUT_RING( CP_PACKET0( RADEON_WAIT_UNTIL, 0 ) ); \ + OUT_RING( (RADEON_WAIT_3D_IDLECLEAN | \ + RADEON_WAIT_HOST_IDLECLEAN) ); \ +} while (0) + +#define RADEON_WAIT_UNTIL_IDLE() do { \ + OUT_RING( CP_PACKET0( RADEON_WAIT_UNTIL, 0 ) ); \ + OUT_RING( (RADEON_WAIT_2D_IDLECLEAN | \ + RADEON_WAIT_3D_IDLECLEAN | \ + RADEON_WAIT_HOST_IDLECLEAN) ); \ +} while (0) + +#define RADEON_WAIT_UNTIL_PAGE_FLIPPED() do { \ + OUT_RING( CP_PACKET0( RADEON_WAIT_UNTIL, 0 ) ); \ + OUT_RING( RADEON_WAIT_CRTC_PFLIP ); \ +} while (0) + +#define RADEON_FLUSH_CACHE() do { \ + if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) { \ + OUT_RING(CP_PACKET0(RADEON_RB3D_DSTCACHE_CTLSTAT, 0)); \ + OUT_RING(RADEON_RB3D_DC_FLUSH); \ + } else { \ + OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); \ + OUT_RING(R300_RB3D_DC_FLUSH); \ + } \ +} while (0) + +#define RADEON_PURGE_CACHE() do { \ + if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) { \ + OUT_RING(CP_PACKET0(RADEON_RB3D_DSTCACHE_CTLSTAT, 0)); \ + OUT_RING(RADEON_RB3D_DC_FLUSH | RADEON_RB3D_DC_FREE); \ + } else { \ + OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); \ + OUT_RING(R300_RB3D_DC_FLUSH | R300_RB3D_DC_FREE); \ + } \ +} while (0) + +#define RADEON_FLUSH_ZCACHE() do { \ + if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) { \ + OUT_RING(CP_PACKET0(RADEON_RB3D_ZCACHE_CTLSTAT, 0)); \ + OUT_RING(RADEON_RB3D_ZC_FLUSH); \ + } else { \ + OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0)); \ + OUT_RING(R300_ZC_FLUSH); \ + } \ +} while (0) + +#define RADEON_PURGE_ZCACHE() do { \ + if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) { \ + OUT_RING(CP_PACKET0(RADEON_RB3D_ZCACHE_CTLSTAT, 0)); \ + OUT_RING(RADEON_RB3D_ZC_FLUSH | RADEON_RB3D_ZC_FREE); \ + } else { \ + OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0)); \ + OUT_RING(R300_ZC_FLUSH | R300_ZC_FREE); \ + } \ +} while (0) + +/* ================================================================ + * Misc helper macros + */ + +/* Perfbox functionality only. + */ +#define RING_SPACE_TEST_WITH_RETURN( dev_priv ) \ +do { \ + if (!(dev_priv->stats.boxes & RADEON_BOX_DMA_IDLE)) { \ + u32 head = GET_RING_HEAD( dev_priv ); \ + if (head == dev_priv->ring.tail) \ + dev_priv->stats.boxes |= RADEON_BOX_DMA_IDLE; \ + } \ +} while (0) + +#define VB_AGE_TEST_WITH_RETURN( dev_priv ) \ +do { \ + struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv; \ + drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv; \ + if ( sarea_priv->last_dispatch >= RADEON_MAX_VB_AGE ) { \ + int __ret; \ + if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) \ + __ret = r600_do_cp_idle(dev_priv); \ + else \ + __ret = radeon_do_cp_idle(dev_priv); \ + if ( __ret ) return __ret; \ + sarea_priv->last_dispatch = 0; \ + radeon_freelist_reset( dev ); \ + } \ +} while (0) + +#define RADEON_DISPATCH_AGE( age ) do { \ + OUT_RING( CP_PACKET0( RADEON_LAST_DISPATCH_REG, 0 ) ); \ + OUT_RING( age ); \ +} while (0) + +#define RADEON_FRAME_AGE( age ) do { \ + OUT_RING( CP_PACKET0( RADEON_LAST_FRAME_REG, 0 ) ); \ + OUT_RING( age ); \ +} while (0) + +#define RADEON_CLEAR_AGE( age ) do { \ + OUT_RING( CP_PACKET0( RADEON_LAST_CLEAR_REG, 0 ) ); \ + OUT_RING( age ); \ +} while (0) + +#define R600_DISPATCH_AGE(age) do { \ + OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1)); \ + OUT_RING((R600_LAST_DISPATCH_REG - R600_SET_CONFIG_REG_OFFSET) >> 2); \ + OUT_RING(age); \ +} while (0) + +#define R600_FRAME_AGE(age) do { \ + OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1)); \ + OUT_RING((R600_LAST_FRAME_REG - R600_SET_CONFIG_REG_OFFSET) >> 2); \ + OUT_RING(age); \ +} while (0) + +#define R600_CLEAR_AGE(age) do { \ + OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1)); \ + OUT_RING((R600_LAST_CLEAR_REG - R600_SET_CONFIG_REG_OFFSET) >> 2); \ + OUT_RING(age); \ +} while (0) + +/* ================================================================ + * Ring control + */ + +#define RADEON_VERBOSE 0 + +#define RING_LOCALS int write, _nr, _align_nr; unsigned int mask; u32 *ring; + +#define RADEON_RING_ALIGN 16 + +#define BEGIN_RING( n ) do { \ + if ( RADEON_VERBOSE ) { \ + DRM_INFO( "BEGIN_RING( %d )\n", (n)); \ + } \ + _align_nr = RADEON_RING_ALIGN - ((dev_priv->ring.tail + n) & (RADEON_RING_ALIGN-1)); \ + _align_nr += n; \ + if (dev_priv->ring.space <= (_align_nr * sizeof(u32))) { \ + COMMIT_RING(); \ + radeon_wait_ring( dev_priv, _align_nr * sizeof(u32)); \ + } \ + _nr = n; dev_priv->ring.space -= (n) * sizeof(u32); \ + ring = dev_priv->ring.start; \ + write = dev_priv->ring.tail; \ + mask = dev_priv->ring.tail_mask; \ +} while (0) + +#define ADVANCE_RING() do { \ + if ( RADEON_VERBOSE ) { \ + DRM_INFO( "ADVANCE_RING() wr=0x%06x tail=0x%06x\n", \ + write, dev_priv->ring.tail ); \ + } \ + if (((dev_priv->ring.tail + _nr) & mask) != write) { \ + DRM_ERROR( \ + "ADVANCE_RING(): mismatch: nr: %x write: %x line: %d\n", \ + ((dev_priv->ring.tail + _nr) & mask), \ + write, __LINE__); \ + } else \ + dev_priv->ring.tail = write; \ +} while (0) + +extern void radeon_commit_ring(drm_radeon_private_t *dev_priv); + +#define COMMIT_RING() do { \ + radeon_commit_ring(dev_priv); \ + } while(0) + +#define OUT_RING( x ) do { \ + if ( RADEON_VERBOSE ) { \ + DRM_INFO( " OUT_RING( 0x%08x ) at 0x%x\n", \ + (unsigned int)(x), write ); \ + } \ + ring[write++] = (x); \ + write &= mask; \ +} while (0) + +#define OUT_RING_REG( reg, val ) do { \ + OUT_RING( CP_PACKET0( reg, 0 ) ); \ + OUT_RING( val ); \ +} while (0) + +#define OUT_RING_TABLE( tab, sz ) do { \ + int _size = (sz); \ + int *_tab = (int *)(tab); \ + \ + if (write + _size > mask) { \ + int _i = (mask+1) - write; \ + _size -= _i; \ + while (_i > 0 ) { \ + *(int *)(ring + write) = *_tab++; \ + write++; \ + _i--; \ + } \ + write = 0; \ + _tab += _i; \ + } \ + while (_size > 0) { \ + *(ring + write) = *_tab++; \ + write++; \ + _size--; \ + } \ + write &= mask; \ +} while (0) + +/** + * Copy given number of dwords from drm buffer to the ring buffer. + */ +#define OUT_RING_DRM_BUFFER(buf, sz) do { \ + int _size = (sz) * 4; \ + struct drm_buffer *_buf = (buf); \ + int _part_size; \ + while (_size > 0) { \ + _part_size = _size; \ + \ + if (write + _part_size/4 > mask) \ + _part_size = ((mask + 1) - write)*4; \ + \ + if (drm_buffer_index(_buf) + _part_size > PAGE_SIZE) \ + _part_size = PAGE_SIZE - drm_buffer_index(_buf);\ + \ + \ + \ + memcpy(ring + write, &_buf->data[drm_buffer_page(_buf)] \ + [drm_buffer_index(_buf)], _part_size); \ + \ + _size -= _part_size; \ + write = (write + _part_size/4) & mask; \ + drm_buffer_advance(_buf, _part_size); \ + } \ +} while (0) + + +#endif /* CONFIG_DRM_RADEON_UMS */ + +#endif /* __RADEON_DRV_H__ */ diff --git a/sys/dev/pci/drm/radeon/radeon_encoders.c b/sys/dev/pci/drm/radeon/radeon_encoders.c index c3f9b6e0c0d..b1fb98e1e93 100644 --- a/sys/dev/pci/drm/radeon/radeon_encoders.c +++ b/sys/dev/pci/drm/radeon/radeon_encoders.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_encoders.c,v 1.4 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2007-8 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -159,10 +158,48 @@ radeon_get_encoder_enum(struct drm_device *dev, uint32_t supported_device, uint8 return ret; } +static void radeon_encoder_add_backlight(struct radeon_encoder *radeon_encoder, + struct drm_connector *connector) +{ + struct drm_device *dev = radeon_encoder->base.dev; + struct radeon_device *rdev = dev->dev_private; + bool use_bl = false; + + if (!(radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))) + return; + + if (radeon_backlight == 0) { + return; + } else if (radeon_backlight == 1) { + use_bl = true; + } else if (radeon_backlight == -1) { + /* Quirks */ + /* Amilo Xi 2550 only works with acpi bl */ + if ((rdev->pdev->device == 0x9583) && + (rdev->pdev->subsystem_vendor == 0x1734) && + (rdev->pdev->subsystem_device == 0x1107)) + use_bl = false; +/* Older PPC macs use on-GPU backlight controller */ +#ifndef CONFIG_PPC_PMAC + /* disable native backlight control on older asics */ + else if (rdev->family < CHIP_R600) + use_bl = false; +#endif + else + use_bl = true; + } + + if (use_bl) { + if (rdev->is_atom_bios) + radeon_atom_backlight_init(radeon_encoder, connector); + else + radeon_legacy_backlight_init(radeon_encoder, connector); + } +} + void radeon_link_encoder_connector(struct drm_device *dev) { - struct radeon_device *rdev = dev->dev_private; struct drm_connector *connector; struct radeon_connector *radeon_connector; struct drm_encoder *encoder; @@ -175,13 +212,8 @@ radeon_link_encoder_connector(struct drm_device *dev) radeon_encoder = to_radeon_encoder(encoder); if (radeon_encoder->devices & radeon_connector->devices) { drm_mode_connector_attach_encoder(connector, encoder); - if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { - if (rdev->is_atom_bios) - radeon_atom_backlight_init(radeon_encoder, connector); - else - radeon_legacy_backlight_init(radeon_encoder, connector); - rdev->mode_info.bl_encoder = radeon_encoder; - } + if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) + radeon_encoder_add_backlight(radeon_encoder, connector); } } } @@ -214,7 +246,16 @@ radeon_get_connector_for_encoder(struct drm_encoder *encoder) list_for_each_entry(connector, &dev->mode_config.connector_list, head) { radeon_connector = to_radeon_connector(connector); - if (radeon_encoder->active_device & radeon_connector->devices) + if (radeon_encoder->is_mst_encoder) { + struct radeon_encoder_mst *mst_enc; + + if (!radeon_connector->is_mst_connector) + continue; + + mst_enc = radeon_encoder->enc_priv; + if (mst_enc->connector == radeon_connector->mst_port) + return connector; + } else if (radeon_encoder->active_device & radeon_connector->devices) return connector; } return NULL; @@ -344,7 +385,7 @@ bool radeon_dig_monitor_is_duallink(struct drm_encoder *encoder, case DRM_MODE_CONNECTOR_HDMIB: if (radeon_connector->use_digital) { /* HDMI 1.3 supports up to 340 Mhz over single link */ - if (ASIC_IS_DCE6(rdev) && drm_detect_hdmi_monitor(radeon_connector->edid)) { + if (ASIC_IS_DCE6(rdev) && drm_detect_hdmi_monitor(radeon_connector_edid(connector))) { if (pixel_clock > 340000) return true; else @@ -360,13 +401,16 @@ bool radeon_dig_monitor_is_duallink(struct drm_encoder *encoder, case DRM_MODE_CONNECTOR_DVID: case DRM_MODE_CONNECTOR_HDMIA: case DRM_MODE_CONNECTOR_DisplayPort: + if (radeon_connector->is_mst_connector) + return false; + dig_connector = radeon_connector->con_priv; if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) return false; else { /* HDMI 1.3 supports up to 340 Mhz over single link */ - if (ASIC_IS_DCE6(rdev) && drm_detect_hdmi_monitor(radeon_connector->edid)) { + if (ASIC_IS_DCE6(rdev) && drm_detect_hdmi_monitor(radeon_connector_edid(connector))) { if (pixel_clock > 340000) return true; else @@ -383,3 +427,24 @@ bool radeon_dig_monitor_is_duallink(struct drm_encoder *encoder, } } +bool radeon_encoder_is_digital(struct drm_encoder *encoder) +{ + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + switch (radeon_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_LVDS: + case ENCODER_OBJECT_ID_INTERNAL_TMDS1: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1: + case ENCODER_OBJECT_ID_INTERNAL_LVTM1: + case ENCODER_OBJECT_ID_INTERNAL_DVO1: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: + case ENCODER_OBJECT_ID_INTERNAL_DDI: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + return true; + default: + return false; + } +} diff --git a/sys/dev/pci/drm/radeon/radeon_family.h b/sys/dev/pci/drm/radeon/radeon_family.h index a56e9759ed4..4b7b87f71a6 100644 --- a/sys/dev/pci/drm/radeon/radeon_family.h +++ b/sys/dev/pci/drm/radeon/radeon_family.h @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_family.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -92,6 +91,13 @@ enum radeon_family { CHIP_TAHITI, CHIP_PITCAIRN, CHIP_VERDE, + CHIP_OLAND, + CHIP_HAINAN, + CHIP_BONAIRE, + CHIP_KAVERI, + CHIP_KABINI, + CHIP_HAWAII, + CHIP_MULLINS, CHIP_LAST, }; @@ -110,6 +116,7 @@ enum radeon_chip_flags { RADEON_NEW_MEMMAP = 0x00400000UL, RADEON_IS_PCI = 0x00800000UL, RADEON_IS_IGPGART = 0x01000000UL, + RADEON_IS_PX = 0x02000000UL, }; #endif diff --git a/sys/dev/pci/drm/radeon/radeon_fb.c b/sys/dev/pci/drm/radeon/radeon_fb.c index e94dddbd651..2d4f221c0fc 100644 --- a/sys/dev/pci/drm/radeon/radeon_fb.c +++ b/sys/dev/pci/drm/radeon/radeon_fb.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_fb.c,v 1.12 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright © 2007 David Airlie * @@ -33,6 +32,7 @@ #include + /* object hierarchy - this contains a helper + a radeon fb the helper contains a pointer to radeon framebuffer baseclass. @@ -49,9 +49,9 @@ static struct fb_ops radeonfb_ops = { .owner = THIS_MODULE, .fb_check_var = drm_fb_helper_check_var, .fb_set_par = drm_fb_helper_set_par, - .fb_fillrect = cfb_fillrect, - .fb_copyarea = cfb_copyarea, - .fb_imageblit = cfb_imageblit, + .fb_fillrect = drm_fb_helper_cfb_fillrect, + .fb_copyarea = drm_fb_helper_cfb_copyarea, + .fb_imageblit = drm_fb_helper_cfb_imageblit, .fb_pan_display = drm_fb_helper_pan_display, .fb_blank = drm_fb_helper_blank, .fb_setcmap = drm_fb_helper_setcmap, @@ -60,7 +60,6 @@ static struct fb_ops radeonfb_ops = { }; #endif -void radeondrm_burner_cb(void *); int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled) { @@ -123,11 +122,10 @@ static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev, if (rdev->family >= CHIP_R600) height = roundup2(mode_cmd->height, 8); size = mode_cmd->pitches[0] * height; - aligned_size = PAGE_ALIGN(size); + aligned_size = roundup2(size, PAGE_SIZE); ret = radeon_gem_object_create(rdev, aligned_size, 0, RADEON_GEM_DOMAIN_VRAM, - false, true, - &gobj); + 0, true, &gobj); if (ret) { printk(KERN_ERR "failed to allocate framebuffer (%d)\n", aligned_size); @@ -186,9 +184,11 @@ out_unref: return ret; } -static int radeonfb_create(struct radeon_fbdev *rfbdev, +static int radeonfb_create(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { + struct radeon_fbdev *rfbdev = + container_of(helper, struct radeon_fbdev, helper); struct radeon_device *rdev = rfbdev->rdev; struct fb_info *info; struct rasops_info *ri = &rdev->ro; @@ -196,13 +196,8 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev, struct drm_mode_fb_cmd2 mode_cmd; struct drm_gem_object *gobj = NULL; struct radeon_bo *rbo = NULL; -#if 0 - struct device *device = &rdev->pdev->dev; -#endif int ret; -#if 0 unsigned long tmp; -#endif mode_cmd.width = sizes->surface_width; mode_cmd.height = sizes->surface_height; @@ -223,9 +218,9 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev, rbo = gem_to_radeon_bo(gobj); /* okay we have an object now allocate the framebuffer */ - info = framebuffer_alloc(0, device); - if (info == NULL) { - ret = -ENOMEM; + info = drm_fb_helper_alloc_fbi(helper); + if (IS_ERR(info)) { + ret = PTR_ERR(info); goto out_unref; } @@ -233,27 +228,28 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev, ret = radeon_framebuffer_init(rdev->ddev, &rfbdev->rfb, &mode_cmd, gobj); if (ret) { - DRM_ERROR("failed to initalise framebuffer %d\n", ret); - goto out_unref; + DRM_ERROR("failed to initialize framebuffer %d\n", ret); + goto out_destroy_fbi; } fb = &rfbdev->rfb.base; /* setup helper */ rfbdev->helper.fb = fb; - rfbdev->helper.fbdev = info; -#ifdef notyet memset_io(rbo->kptr, 0x0, radeon_bo_size(rbo)); +#ifdef __linux__ strcpy(info->fix.id, "radeondrmfb"); drm_fb_helper_fill_fix(info, fb->pitches[0], fb->depth); info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT; info->fbops = &radeonfb_ops; +#endif tmp = radeon_bo_gpu_offset(rbo) - rdev->mc.vram_start; +#ifdef __linux__ info->fix.smem_start = rdev->mc.aper_base + tmp; info->fix.smem_len = radeon_bo_size(rbo); info->screen_base = rbo->kptr; @@ -262,37 +258,24 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev, drm_fb_helper_fill_var(info, &rfbdev->helper, sizes->fb_width, sizes->fb_height); /* setup aperture base/size for vesafb takeover */ - info->apertures = alloc_apertures(1); - if (!info->apertures) { - ret = -ENOMEM; - goto out_unref; - } info->apertures->ranges[0].base = rdev->ddev->mode_config.fb_base; info->apertures->ranges[0].size = rdev->mc.aper_size; +#endif /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ +#ifdef __linux__ if (info->screen_base == NULL) { ret = -ENOSPC; - goto out_unref; - } - - ret = fb_alloc_cmap(&info->cmap, 256, 0); - if (ret) { - ret = -ENOMEM; - goto out_unref; + goto out_destroy_fbi; } DRM_INFO("fb mappable at 0x%lX\n", info->fix.smem_start); #endif -#ifdef DRMDEBUG DRM_INFO("vram apper at 0x%lX\n", (unsigned long)rdev->mc.aper_base); DRM_INFO("size %lu\n", (unsigned long)radeon_bo_size(rbo)); - DRM_INFO("fb depth is %d bpp is %d\n", fb->depth, fb->bits_per_pixel); + DRM_INFO("fb depth is %d\n", fb->depth); DRM_INFO(" pitch is %d\n", fb->pitches[0]); -#endif - - memset(rbo->kptr, 0x0, radeon_bo_size(rbo)); ri->ri_bits = rbo->kptr; ri->ri_depth = fb->bits_per_pixel; @@ -319,76 +302,57 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev, break; } -#if 0 +#ifdef __linux__ vga_switcheroo_client_fb_set(rdev->ddev->pdev, info); #endif return 0; +out_destroy_fbi: +#ifdef __linux__ + drm_fb_helper_release_fbi(helper); +#endif out_unref: if (rbo) { } if (fb && ret) { drm_gem_object_unreference(gobj); + drm_framebuffer_unregister_private(fb); drm_framebuffer_cleanup(fb); kfree(fb); } return ret; } -static int radeon_fb_find_or_create_single(struct drm_fb_helper *helper, - struct drm_fb_helper_surface_size *sizes) -{ - struct radeon_fbdev *rfbdev = (struct radeon_fbdev *)helper; - int new_fb = 0; - int ret; - - if (!helper->fb) { - ret = radeonfb_create(rfbdev, sizes); - if (ret) - return ret; - new_fb = 1; - } - return new_fb; -} - void radeon_fb_output_poll_changed(struct radeon_device *rdev) { drm_fb_helper_hotplug_event(&rdev->mode_info.rfbdev->helper); } +#ifdef notyet static int radeon_fbdev_destroy(struct drm_device *dev, struct radeon_fbdev *rfbdev) { - printf("%s stub\n", __func__); - return -ENOSYS; -#ifdef notyet - struct fb_info *info; struct radeon_framebuffer *rfb = &rfbdev->rfb; - if (rfbdev->helper.fbdev) { - info = rfbdev->helper.fbdev; - - unregister_framebuffer(info); - if (info->cmap.len) - fb_dealloc_cmap(&info->cmap); - framebuffer_release(info); - } + drm_fb_helper_unregister_fbi(&rfbdev->helper); + drm_fb_helper_release_fbi(&rfbdev->helper); if (rfb->obj) { radeonfb_destroy_pinned_object(rfb->obj); rfb->obj = NULL; } drm_fb_helper_fini(&rfbdev->helper); + drm_framebuffer_unregister_private(&rfb->base); drm_framebuffer_cleanup(&rfb->base); return 0; -#endif } +#endif -static struct drm_fb_helper_funcs radeon_fb_helper_funcs = { +static const struct drm_fb_helper_funcs radeon_fb_helper_funcs = { .gamma_set = radeon_crtc_fb_gamma_set, .gamma_get = radeon_crtc_fb_gamma_get, - .fb_probe = radeon_fb_find_or_create_single, + .fb_probe = radeonfb_create, }; int radeon_fbdev_init(struct radeon_device *rdev) @@ -414,38 +378,27 @@ int radeon_fbdev_init(struct radeon_device *rdev) ret = drm_fb_helper_init(rdev->ddev, &rfbdev->helper, rdev->num_crtc, RADEONFB_CONN_LIMIT); - if (ret) { - kfree(rfbdev); - return ret; - } + if (ret) + goto free; - task_set(&rdev->burner_task, radeondrm_burner_cb, rdev); + ret = drm_fb_helper_single_add_all_connectors(&rfbdev->helper); + if (ret) + goto fini; + + /* disable all the possible outputs/crtcs before entering KMS mode */ + drm_helper_disable_unused_functions(rdev->ddev); + + ret = drm_fb_helper_initial_config(&rfbdev->helper, bpp_sel); + if (ret) + goto fini; - drm_fb_helper_single_add_all_connectors(&rfbdev->helper); -#ifdef __sparc64__ -{ - struct drm_fb_helper *fb_helper = &rfbdev->helper; - struct drm_fb_helper_connector *fb_helper_conn; - int i; - - for (i = 0; i < fb_helper->connector_count; i++) { - struct drm_cmdline_mode *mode; - struct drm_connector *connector; - - fb_helper_conn = fb_helper->connector_info[i]; - connector = fb_helper_conn->connector; - mode = &connector->cmdline_mode; - - mode->specified = true; - mode->xres = rdev->sf.sf_width; - mode->yres = rdev->sf.sf_height; - mode->bpp_specified = true; - mode->bpp = rdev->sf.sf_depth; - } -} -#endif - drm_fb_helper_initial_config(&rfbdev->helper, bpp_sel); return 0; + +fini: + drm_fb_helper_fini(&rfbdev->helper); +free: + kfree(rfbdev); + return ret; } void radeon_fbdev_fini(struct radeon_device *rdev) @@ -453,9 +406,9 @@ void radeon_fbdev_fini(struct radeon_device *rdev) if (!rdev->mode_info.rfbdev) return; - task_del(systq, &rdev->burner_task); - +#ifdef notyet radeon_fbdev_destroy(rdev->ddev, rdev->mode_info.rfbdev); +#endif kfree(rdev->mode_info.rfbdev); rdev->mode_info.rfbdev = NULL; } @@ -467,16 +420,6 @@ void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state) #endif } -int radeon_fbdev_total_size(struct radeon_device *rdev) -{ - struct radeon_bo *robj; - int size = 0; - - robj = gem_to_radeon_bo(rdev->mode_info.rfbdev->rfb.obj); - size += radeon_bo_size(robj); - return size; -} - bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj) { if (robj == gem_to_radeon_bo(rdev->mode_info.rfbdev->rfb.obj)) @@ -484,6 +427,32 @@ bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj) return false; } +void radeon_fb_add_connector(struct radeon_device *rdev, struct drm_connector *connector) +{ + drm_fb_helper_add_one_connector(&rdev->mode_info.rfbdev->helper, connector); +} + +void radeon_fb_remove_connector(struct radeon_device *rdev, struct drm_connector *connector) +{ + drm_fb_helper_remove_one_connector(&rdev->mode_info.rfbdev->helper, connector); +} + +void radeon_fbdev_restore_mode(struct radeon_device *rdev) +{ + struct radeon_fbdev *rfbdev = rdev->mode_info.rfbdev; + struct drm_fb_helper *fb_helper; + int ret; + + if (!rfbdev) + return; + + fb_helper = &rfbdev->helper; + + ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper); + if (ret) + DRM_DEBUG("failed to restore crtc mode\n"); +} + void radeondrm_burner(void *v, u_int on, u_int flags) { diff --git a/sys/dev/pci/drm/radeon/radeon_fence.c b/sys/dev/pci/drm/radeon/radeon_fence.c index 5c2cae52499..6fd9cf59975 100644 --- a/sys/dev/pci/drm/radeon/radeon_fence.c +++ b/sys/dev/pci/drm/radeon/radeon_fence.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_fence.c,v 1.12 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2009 Jerome Glisse. * All Rights Reserved. @@ -29,6 +28,7 @@ * Jerome Glisse * Dave Airlie */ +#include #include #include "radeon_reg.h" #include "radeon.h" @@ -58,7 +58,9 @@ static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) { struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; if (likely(rdev->wb.enabled || !drv->scratch_reg)) { - *drv->cpu_addr = cpu_to_le32(seq); + if (drv->cpu_addr) { + *drv->cpu_addr = cpu_to_le32(seq); + } } else { WREG32(drv->scratch_reg, seq); } @@ -79,13 +81,36 @@ static u32 radeon_fence_read(struct radeon_device *rdev, int ring) u32 seq = 0; if (likely(rdev->wb.enabled || !drv->scratch_reg)) { - seq = le32_to_cpu(*drv->cpu_addr); + if (drv->cpu_addr) { + seq = le32_to_cpu(*drv->cpu_addr); + } else { + seq = lower_32_bits(atomic64_read(&drv->last_seq)); + } } else { seq = RREG32(drv->scratch_reg); } return seq; } +/** + * radeon_fence_schedule_check - schedule lockup check + * + * @rdev: radeon_device pointer + * @ring: ring index we should work with + * + * Queues a delayed work item to check for lockups. + */ +static void radeon_fence_schedule_check(struct radeon_device *rdev, int ring) +{ + /* + * Do not reset the timer here with mod_delayed_work, + * this can livelock in an interaction with TTM delayed destroy. + */ + queue_delayed_work(system_power_efficient_wq, + &rdev->fence_drv[ring].lockup_work, + RADEON_FENCE_JIFFIES_TIMEOUT); +} + /** * radeon_fence_emit - emit a fence on the requested ring * @@ -100,30 +125,71 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence **fence, int ring) { + u64 seq = ++rdev->fence_drv[ring].sync_seq[ring]; + /* we are protected by the ring emission mutex */ *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL); if ((*fence) == NULL) { return -ENOMEM; } - kref_init(&((*fence)->kref)); (*fence)->rdev = rdev; - (*fence)->seq = ++rdev->fence_drv[ring].sync_seq[ring]; + (*fence)->seq = seq; (*fence)->ring = ring; + (*fence)->is_vm_update = false; + fence_init(&(*fence)->base, &radeon_fence_ops, + &rdev->fence_queue.lock, rdev->fence_context + ring, seq); radeon_fence_ring_emit(rdev, ring, *fence); - trace_radeon_fence_emit(rdev->ddev, (*fence)->seq); + trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); + radeon_fence_schedule_check(rdev, ring); return 0; } /** - * radeon_fence_process - process a fence + * radeon_fence_check_signaled - callback from fence_queue + * + * this function is called with fence_queue lock held, which is also used + * for the fence locking itself, so unlocked variants are used for + * fence_signal, and remove_wait_queue. + */ +static int radeon_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key) +{ + struct radeon_fence *fence; + u64 seq; + + fence = container_of(wait, struct radeon_fence, fence_wake); + + /* + * We cannot use radeon_fence_process here because we're already + * in the waitqueue, in a call from wake_up_all. + */ + seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq); + if (seq >= fence->seq) { + int ret = fence_signal_locked(&fence->base); + + if (!ret) + FENCE_TRACE(&fence->base, "signaled from irq context\n"); + else + FENCE_TRACE(&fence->base, "was already signaled\n"); + + radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring); + __remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake); + fence_put(&fence->base); + } else + FENCE_TRACE(&fence->base, "pending\n"); + return 0; +} + +/** + * radeon_fence_activity - check for fence activity * * @rdev: radeon_device pointer * @ring: ring index the fence is associated with * - * Checks the current fence value and wakes the fence queue - * if the sequence number has increased (all asics). + * Checks the current fence value and calculates the last + * signalled fence value. Returns true if activity occured + * on the ring, and the fence_queue should be waken up. */ -void radeon_fence_process(struct radeon_device *rdev, int ring) +static bool radeon_fence_activity(struct radeon_device *rdev, int ring) { uint64_t seq, last_seq, last_emitted; unsigned count_loop = 0; @@ -179,35 +245,87 @@ void radeon_fence_process(struct radeon_device *rdev, int ring) } } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); - if (wake) { - rdev->fence_drv[ring].last_activity = jiffies; + if (seq < last_emitted) + radeon_fence_schedule_check(rdev, ring); + + return wake; +} + +/** + * radeon_fence_check_lockup - check for hardware lockup + * + * @work: delayed work item + * + * Checks for fence activity and if there is none probe + * the hardware if a lockup occured. + */ +static void radeon_fence_check_lockup(struct work_struct *work) +{ + struct radeon_fence_driver *fence_drv; + struct radeon_device *rdev; + int ring; + + fence_drv = container_of(work, struct radeon_fence_driver, + lockup_work.work); + rdev = fence_drv->rdev; + ring = fence_drv - &rdev->fence_drv[0]; + + if (!down_read_trylock(&rdev->exclusive_lock)) { + /* just reschedule the check if a reset is going on */ + radeon_fence_schedule_check(rdev, ring); + return; + } + + if (fence_drv->delayed_irq && rdev->ddev->irq_enabled) { + unsigned long irqflags; + + fence_drv->delayed_irq = false; + spin_lock_irqsave(&rdev->irq.lock, irqflags); + radeon_irq_set(rdev); + spin_unlock_irqrestore(&rdev->irq.lock, irqflags); + } + + if (radeon_fence_activity(rdev, ring)) + wake_up_all(&rdev->fence_queue); + + else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { + + /* good news we believe it's a lockup */ + dev_warn(rdev->dev, "GPU lockup (current fence id " + "0x%016llx last fence id 0x%016llx on ring %d)\n", + (uint64_t)atomic64_read(&fence_drv->last_seq), + fence_drv->sync_seq[ring], ring); + + /* remember that we need an reset */ + rdev->needs_reset = true; wake_up_all(&rdev->fence_queue); } + up_read(&rdev->exclusive_lock); } /** - * radeon_fence_destroy - destroy a fence + * radeon_fence_process - process a fence * - * @kref: fence kref + * @rdev: radeon_device pointer + * @ring: ring index the fence is associated with * - * Frees the fence object (all asics). + * Checks the current fence value and wakes the fence queue + * if the sequence number has increased (all asics). */ -static void radeon_fence_destroy(struct kref *kref) +void radeon_fence_process(struct radeon_device *rdev, int ring) { - struct radeon_fence *fence; - - fence = container_of(kref, struct radeon_fence, kref); - kfree(fence); + if (radeon_fence_activity(rdev, ring)) + wake_up_all(&rdev->fence_queue); } /** - * radeon_fence_seq_signaled - check if a fence sequeuce number has signaled + * radeon_fence_seq_signaled - check if a fence sequence number has signaled * * @rdev: radeon device pointer * @seq: sequence number * @ring: ring index the fence is associated with * - * Check if the last singled fence sequnce number is >= the requested + * Check if the last signaled fence sequnce number is >= the requested * sequence number (all asics). * Returns true if the fence has signaled (current fence value * is >= requested value) or false if it has not (current fence @@ -228,311 +346,215 @@ static bool radeon_fence_seq_signaled(struct radeon_device *rdev, return false; } -/** - * radeon_fence_signaled - check if a fence has signaled - * - * @fence: radeon fence object - * - * Check if the requested fence has signaled (all asics). - * Returns true if the fence has signaled or false if it has not. - */ -bool radeon_fence_signaled(struct radeon_fence *fence) +static bool radeon_fence_is_signaled(struct fence *f) { - if (!fence) { - return true; - } - if (fence->seq == RADEON_FENCE_SIGNALED_SEQ) { + struct radeon_fence *fence = to_radeon_fence(f); + struct radeon_device *rdev = fence->rdev; + unsigned ring = fence->ring; + u64 seq = fence->seq; + + if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { return true; } - if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { - fence->seq = RADEON_FENCE_SIGNALED_SEQ; - return true; + + if (down_read_trylock(&rdev->exclusive_lock)) { + radeon_fence_process(rdev, ring); + up_read(&rdev->exclusive_lock); + + if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { + return true; + } } return false; } /** - * radeon_fence_wait_seq - wait for a specific sequence number - * - * @rdev: radeon device pointer - * @target_seq: sequence number we want to wait for - * @ring: ring index the fence is associated with - * @intr: use interruptable sleep - * @lock_ring: whether the ring should be locked or not + * radeon_fence_enable_signaling - enable signalling on fence + * @fence: fence * - * Wait for the requested sequence number to be written (all asics). - * @intr selects whether to use interruptable (true) or non-interruptable - * (false) sleep when waiting for the sequence number. Helper function - * for radeon_fence_wait(), et al. - * Returns 0 if the sequence number has passed, error for all other cases. - * -EDEADLK is returned when a GPU lockup has been detected and the ring is - * marked as not ready so no further jobs get scheduled until a successful - * reset. + * This function is called with fence_queue lock held, and adds a callback + * to fence_queue that checks if this fence is signaled, and if so it + * signals the fence and removes itself. */ -static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq, - unsigned ring, bool intr, bool lock_ring) +static bool radeon_fence_enable_signaling(struct fence *f) { - unsigned long timeout, last_activity; - uint64_t seq; - unsigned i; - bool signaled; - int r, error; + struct radeon_fence *fence = to_radeon_fence(f); + struct radeon_device *rdev = fence->rdev; - while (target_seq > atomic64_read(&rdev->fence_drv[ring].last_seq)) { - if (!rdev->ring[ring].ready) { - return -EBUSY; - } + if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) + return false; - timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT; - if (time_after(rdev->fence_drv[ring].last_activity, timeout)) { - /* the normal case, timeout is somewhere before last_activity */ - timeout = rdev->fence_drv[ring].last_activity - timeout; - } else { - /* either jiffies wrapped around, or no fence was signaled in the last 500ms - * anyway we will just wait for the minimum amount and then check for a lockup - */ - timeout = 1; - } - seq = atomic64_read(&rdev->fence_drv[ring].last_seq); - /* Save current last activity valuee, used to check for GPU lockups */ - last_activity = rdev->fence_drv[ring].last_activity; - - trace_radeon_fence_wait_begin(rdev->ddev, seq); - radeon_irq_kms_sw_irq_get(rdev, ring); - r = timeout; - while (r > 0) { - signaled = radeon_fence_seq_signaled(rdev, target_seq, ring); - if (signaled) - break; - error = tsleep(&rdev->fence_queue, - PZERO | (intr ? PCATCH : 0), "rfnwt", timeout); - if (error == ERESTART) - error = EINTR; /* XXX */ - if (error == EWOULDBLOCK) - error = 0; - r = -error; - } - radeon_irq_kms_sw_irq_put(rdev, ring); - if (unlikely(r < 0)) { - return r; - } - trace_radeon_fence_wait_end(rdev->ddev, seq); - - if (unlikely(!signaled)) { - /* we were interrupted for some reason and fence - * isn't signaled yet, resume waiting */ - if (r) { - continue; - } - - /* check if sequence value has changed since last_activity */ - if (seq != atomic64_read(&rdev->fence_drv[ring].last_seq)) { - continue; - } - - if (lock_ring) { - mutex_lock(&rdev->ring_lock); - } - - /* test if somebody else has already decided that this is a lockup */ - if (last_activity != rdev->fence_drv[ring].last_activity) { - if (lock_ring) { - mutex_unlock(&rdev->ring_lock); - } - continue; - } - - if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { - /* good news we believe it's a lockup */ - dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n", - target_seq, seq); - - /* change last activity so nobody else think there is a lockup */ - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - rdev->fence_drv[i].last_activity = jiffies; - } - - /* mark the ring as not ready any more */ - rdev->ring[ring].ready = false; - if (lock_ring) { - mutex_unlock(&rdev->ring_lock); - } - return -EDEADLK; - } - - if (lock_ring) { - mutex_unlock(&rdev->ring_lock); - } + if (down_read_trylock(&rdev->exclusive_lock)) { + radeon_irq_kms_sw_irq_get(rdev, fence->ring); + + if (radeon_fence_activity(rdev, fence->ring)) + wake_up_all_locked(&rdev->fence_queue); + + /* did fence get signaled after we enabled the sw irq? */ + if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) { + radeon_irq_kms_sw_irq_put(rdev, fence->ring); + up_read(&rdev->exclusive_lock); + return false; } + + up_read(&rdev->exclusive_lock); + } else { + /* we're probably in a lockup, lets not fiddle too much */ + if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring)) + rdev->fence_drv[fence->ring].delayed_irq = true; + radeon_fence_schedule_check(rdev, fence->ring); } - return 0; + + fence->fence_wake.flags = 0; + fence->fence_wake.private = NULL; + fence->fence_wake.func = radeon_fence_check_signaled; + __add_wait_queue(&rdev->fence_queue, &fence->fence_wake); + fence_get(f); + + FENCE_TRACE(&fence->base, "armed on ring %i!\n", fence->ring); + return true; } /** - * radeon_fence_wait - wait for a fence to signal + * radeon_fence_signaled - check if a fence has signaled * * @fence: radeon fence object - * @intr: use interruptable sleep * - * Wait for the requested fence to signal (all asics). - * @intr selects whether to use interruptable (true) or non-interruptable - * (false) sleep when waiting for the fence. - * Returns 0 if the fence has passed, error for all other cases. + * Check if the requested fence has signaled (all asics). + * Returns true if the fence has signaled or false if it has not. */ -int radeon_fence_wait(struct radeon_fence *fence, bool intr) +bool radeon_fence_signaled(struct radeon_fence *fence) { - int r; + if (!fence) + return true; - if (fence == NULL) { - WARN(1, "Querying an invalid fence : %p !\n", fence); - return -EINVAL; - } + if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { + int ret; - r = radeon_fence_wait_seq(fence->rdev, fence->seq, - fence->ring, intr, true); - if (r) { - return r; + ret = fence_signal(&fence->base); + if (!ret) + FENCE_TRACE(&fence->base, "signaled from radeon_fence_signaled\n"); + return true; } - fence->seq = RADEON_FENCE_SIGNALED_SEQ; - return 0; + return false; } +/** + * radeon_fence_any_seq_signaled - check if any sequence number is signaled + * + * @rdev: radeon device pointer + * @seq: sequence numbers + * + * Check if the last signaled fence sequnce number is >= the requested + * sequence number (all asics). + * Returns true if any has signaled (current value is >= requested value) + * or false if it has not. Helper function for radeon_fence_wait_seq. + */ static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq) { unsigned i; for (i = 0; i < RADEON_NUM_RINGS; ++i) { - if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) { + if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) return true; - } } return false; } /** - * radeon_fence_wait_any_seq - wait for a sequence number on any ring + * radeon_fence_wait_seq_timeout - wait for a specific sequence numbers * * @rdev: radeon device pointer * @target_seq: sequence number(s) we want to wait for * @intr: use interruptable sleep + * @timeout: maximum time to wait, or MAX_SCHEDULE_TIMEOUT for infinite wait * * Wait for the requested sequence number(s) to be written by any ring * (all asics). Sequnce number array is indexed by ring id. * @intr selects whether to use interruptable (true) or non-interruptable * (false) sleep when waiting for the sequence number. Helper function - * for radeon_fence_wait_any(), et al. - * Returns 0 if the sequence number has passed, error for all other cases. + * for radeon_fence_wait_*(). + * Returns remaining time if the sequence number has passed, 0 when + * the wait timeout, or an error for all other cases. + * -EDEADLK is returned when a GPU lockup has been detected. */ -static int radeon_fence_wait_any_seq(struct radeon_device *rdev, - u64 *target_seq, bool intr) +static long radeon_fence_wait_seq_timeout(struct radeon_device *rdev, + u64 *target_seq, bool intr, + long timeout) { - unsigned long timeout, last_activity, tmp; - unsigned i, ring = RADEON_NUM_RINGS; - bool signaled; - int r, error; + long r; + int i; + + if (radeon_fence_any_seq_signaled(rdev, target_seq)) + return timeout; - for (i = 0, last_activity = 0; i < RADEON_NUM_RINGS; ++i) { - if (!target_seq[i]) { + /* enable IRQs and tracing */ + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + if (!target_seq[i]) continue; - } - /* use the most recent one as indicator */ - if (time_after(rdev->fence_drv[i].last_activity, last_activity)) { - last_activity = rdev->fence_drv[i].last_activity; - } + trace_radeon_fence_wait_begin(rdev->ddev, i, target_seq[i]); + radeon_irq_kms_sw_irq_get(rdev, i); + } - /* For lockup detection just pick the lowest ring we are - * actively waiting for - */ - if (i < ring) { - ring = i; - } + if (intr) { + r = wait_event_interruptible_timeout(rdev->fence_queue, ( + radeon_fence_any_seq_signaled(rdev, target_seq) + || rdev->needs_reset), timeout); + } else { + r = wait_event_timeout(rdev->fence_queue, ( + radeon_fence_any_seq_signaled(rdev, target_seq) + || rdev->needs_reset), timeout); } - /* nothing to wait for ? */ - if (ring == RADEON_NUM_RINGS) { - return -ENOENT; + if (rdev->needs_reset) + r = -EDEADLK; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + if (!target_seq[i]) + continue; + + radeon_irq_kms_sw_irq_put(rdev, i); + trace_radeon_fence_wait_end(rdev->ddev, i, target_seq[i]); } - while (!radeon_fence_any_seq_signaled(rdev, target_seq)) { - timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT; - if (time_after(last_activity, timeout)) { - /* the normal case, timeout is somewhere before last_activity */ - timeout = last_activity - timeout; - } else { - /* either jiffies wrapped around, or no fence was signaled in the last 500ms - * anyway we will just wait for the minimum amount and then check for a lockup - */ - timeout = 1; - } + return r; +} - trace_radeon_fence_wait_begin(rdev->ddev, target_seq[ring]); - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - if (target_seq[i]) { - radeon_irq_kms_sw_irq_get(rdev, i); - } - } - r = timeout; - while (r > 0) { - signaled = radeon_fence_any_seq_signaled(rdev, target_seq); - if (signaled) - break; - error = tsleep(&rdev->fence_queue, - PZERO | (intr ? PCATCH : 0), "rfwa", timeout); - if (error == ERESTART) - error = EINTR; /* XXX */ - if (error == EWOULDBLOCK) - error = 0; - r = -error; - } - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - if (target_seq[i]) { - radeon_irq_kms_sw_irq_put(rdev, i); - } - } - if (unlikely(r < 0)) { - return r; - } - trace_radeon_fence_wait_end(rdev->ddev, target_seq[ring]); - - if (unlikely(!signaled)) { - /* we were interrupted for some reason and fence - * isn't signaled yet, resume waiting */ - if (r) { - continue; - } - - mutex_lock(&rdev->ring_lock); - for (i = 0, tmp = 0; i < RADEON_NUM_RINGS; ++i) { - if (time_after(rdev->fence_drv[i].last_activity, tmp)) { - tmp = rdev->fence_drv[i].last_activity; - } - } - /* test if somebody else has already decided that this is a lockup */ - if (last_activity != tmp) { - last_activity = tmp; - mutex_unlock(&rdev->ring_lock); - continue; - } - - if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { - /* good news we believe it's a lockup */ - dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx)\n", - target_seq[ring]); - - /* change last activity so nobody else think there is a lockup */ - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - rdev->fence_drv[i].last_activity = jiffies; - } - - /* mark the ring as not ready any more */ - rdev->ring[ring].ready = false; - mutex_unlock(&rdev->ring_lock); - return -EDEADLK; - } - mutex_unlock(&rdev->ring_lock); - } +/** + * radeon_fence_wait - wait for a fence to signal + * + * @fence: radeon fence object + * @intr: use interruptible sleep + * + * Wait for the requested fence to signal (all asics). + * @intr selects whether to use interruptable (true) or non-interruptable + * (false) sleep when waiting for the fence. + * Returns 0 if the fence has passed, error for all other cases. + */ +int radeon_fence_wait(struct radeon_fence *fence, bool intr) +{ + uint64_t seq[RADEON_NUM_RINGS] = {}; + long r; + + /* + * This function should not be called on !radeon fences. + * If this is the case, it would mean this function can + * also be called on radeon fences belonging to another card. + * exclusive_lock is not held in that case. + */ + if (WARN_ON_ONCE(!to_radeon_fence(&fence->base))) + return fence_wait(&fence->base, intr); + + seq[fence->ring] = fence->seq; + r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, MAX_SCHEDULE_TIMEOUT); + if (r < 0) { + return r; } + + r = fence_signal(&fence->base); + if (!r) + FENCE_TRACE(&fence->base, "signaled from fence_wait\n"); return 0; } @@ -554,8 +576,8 @@ int radeon_fence_wait_any(struct radeon_device *rdev, bool intr) { uint64_t seq[RADEON_NUM_RINGS]; - unsigned i; - int r; + unsigned i, num_rings = 0; + long r; for (i = 0; i < RADEON_NUM_RINGS; ++i) { seq[i] = 0; @@ -564,23 +586,23 @@ int radeon_fence_wait_any(struct radeon_device *rdev, continue; } - if (fences[i]->seq == RADEON_FENCE_SIGNALED_SEQ) { - /* something was allready signaled */ - return 0; - } - seq[i] = fences[i]->seq; + ++num_rings; } - r = radeon_fence_wait_any_seq(rdev, seq, intr); - if (r) { + /* nothing to wait for ? */ + if (num_rings == 0) + return -ENOENT; + + r = radeon_fence_wait_seq_timeout(rdev, seq, intr, MAX_SCHEDULE_TIMEOUT); + if (r < 0) { return r; } return 0; } /** - * radeon_fence_wait_next_locked - wait for the next fence to signal + * radeon_fence_wait_next - wait for the next fence to signal * * @rdev: radeon device pointer * @ring: ring index the fence is associated with @@ -589,21 +611,25 @@ int radeon_fence_wait_any(struct radeon_device *rdev, * Returns 0 if the next fence has passed, error for all other cases. * Caller must hold ring lock. */ -int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring) +int radeon_fence_wait_next(struct radeon_device *rdev, int ring) { - uint64_t seq; + uint64_t seq[RADEON_NUM_RINGS] = {}; + long r; - seq = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; - if (seq >= rdev->fence_drv[ring].sync_seq[ring]) { + seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; + if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { /* nothing to wait for, last_seq is already the last emited fence */ return -ENOENT; } - return radeon_fence_wait_seq(rdev, seq, ring, false, false); + r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); + if (r < 0) + return r; + return 0; } /** - * radeon_fence_wait_empty_locked - wait for all fences to signal + * radeon_fence_wait_empty - wait for all fences to signal * * @rdev: radeon device pointer * @ring: ring index the fence is associated with @@ -612,17 +638,21 @@ int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring) * Returns 0 if the fences have passed, error for all other cases. * Caller must hold ring lock. */ -int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring) +int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) { - uint64_t seq = rdev->fence_drv[ring].sync_seq[ring]; - int r; + uint64_t seq[RADEON_NUM_RINGS] = {}; + long r; - r = radeon_fence_wait_seq(rdev, seq, ring, false, false); - if (r) { - if (r == -EDEADLK) { + seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; + if (!seq[ring]) + return 0; + + r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); + if (r < 0) { + if (r == -EDEADLK) return -EDEADLK; - } - dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%d)\n", + + dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%ld)\n", ring, r); } return 0; @@ -638,7 +668,7 @@ int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring) */ struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) { - kref_get(&fence->kref); + fence_get(&fence->base); return fence; } @@ -655,7 +685,7 @@ void radeon_fence_unref(struct radeon_fence **fence) *fence = NULL; if (tmp) { - kref_put(&tmp->kref, radeon_fence_destroy); + fence_put(&tmp->base); } } @@ -771,7 +801,19 @@ int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); if (rdev->wb.use_event || !radeon_ring_supports_scratch_reg(rdev, &rdev->ring[ring])) { rdev->fence_drv[ring].scratch_reg = 0; - index = R600_WB_EVENT_OFFSET + ring * 4; + if (ring != R600_RING_TYPE_UVD_INDEX) { + index = R600_WB_EVENT_OFFSET + ring * 4; + rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; + rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + + index; + + } else { + /* put fence directly behind firmware */ + index = roundup2(rdev->uvd_fw->size, 8); + rdev->fence_drv[ring].cpu_addr = rdev->uvd.cpu_addr + index; + rdev->fence_drv[ring].gpu_addr = rdev->uvd.gpu_addr + index; + } + } else { r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); if (r) { @@ -781,15 +823,13 @@ int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; + rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; + rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; } - rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; - rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring); rdev->fence_drv[ring].initialized = true; -#ifdef DRMDEBUG dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx and cpu addr 0x%p\n", ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); -#endif return 0; } @@ -813,8 +853,10 @@ static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) for (i = 0; i < RADEON_NUM_RINGS; ++i) rdev->fence_drv[ring].sync_seq[i] = 0; atomic64_set(&rdev->fence_drv[ring].last_seq, 0); - rdev->fence_drv[ring].last_activity = jiffies; rdev->fence_drv[ring].initialized = false; + INIT_DELAYED_WORK(&rdev->fence_drv[ring].lockup_work, + radeon_fence_check_lockup); + rdev->fence_drv[ring].rdev = rdev; } /** @@ -859,11 +901,12 @@ void radeon_fence_driver_fini(struct radeon_device *rdev) for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { if (!rdev->fence_drv[ring].initialized) continue; - r = radeon_fence_wait_empty_locked(rdev, ring); + r = radeon_fence_wait_empty(rdev, ring); if (r) { /* no need to trigger GPU reset as we are unloading */ - radeon_fence_driver_force_completion(rdev); + radeon_fence_driver_force_completion(rdev, ring); } + cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); wake_up_all(&rdev->fence_queue); radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); rdev->fence_drv[ring].initialized = false; @@ -875,18 +918,16 @@ void radeon_fence_driver_fini(struct radeon_device *rdev) * radeon_fence_driver_force_completion - force all fence waiter to complete * * @rdev: radeon device pointer + * @ring: the ring to complete * * In case of GPU reset failure make sure no process keep waiting on fence * that will never complete. */ -void radeon_fence_driver_force_completion(struct radeon_device *rdev) +void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring) { - int ring; - - for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { - if (!rdev->fence_drv[ring].initialized) - continue; + if (rdev->fence_drv[ring].initialized) { radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring); + cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); } } @@ -906,6 +947,8 @@ static int radeon_debugfs_fence_info(struct seq_file *m, void *data) if (!rdev->fence_drv[i].initialized) continue; + radeon_fence_process(rdev, i); + seq_printf(m, "--- ring %d ---\n", i); seq_printf(m, "Last signaled fence 0x%016llx\n", (unsigned long long)atomic64_read(&rdev->fence_drv[i].last_seq)); @@ -921,16 +964,128 @@ static int radeon_debugfs_fence_info(struct seq_file *m, void *data) return 0; } +/** + * radeon_debugfs_gpu_reset - manually trigger a gpu reset + * + * Manually trigger a gpu reset at the next fence wait. + */ +static int radeon_debugfs_gpu_reset(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct radeon_device *rdev = dev->dev_private; + + down_read(&rdev->exclusive_lock); + seq_printf(m, "%d\n", rdev->needs_reset); + rdev->needs_reset = true; + wake_up_all(&rdev->fence_queue); + up_read(&rdev->exclusive_lock); + + return 0; +} + static struct drm_info_list radeon_debugfs_fence_list[] = { {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL}, + {"radeon_gpu_reset", &radeon_debugfs_gpu_reset, 0, NULL} }; #endif int radeon_debugfs_fence_init(struct radeon_device *rdev) { #if defined(CONFIG_DEBUG_FS) - return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1); + return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 2); #else return 0; #endif } + +static const char *radeon_fence_get_driver_name(struct fence *fence) +{ + return "radeon"; +} + +static const char *radeon_fence_get_timeline_name(struct fence *f) +{ + struct radeon_fence *fence = to_radeon_fence(f); + switch (fence->ring) { + case RADEON_RING_TYPE_GFX_INDEX: return "radeon.gfx"; + case CAYMAN_RING_TYPE_CP1_INDEX: return "radeon.cp1"; + case CAYMAN_RING_TYPE_CP2_INDEX: return "radeon.cp2"; + case R600_RING_TYPE_DMA_INDEX: return "radeon.dma"; + case CAYMAN_RING_TYPE_DMA1_INDEX: return "radeon.dma1"; + case R600_RING_TYPE_UVD_INDEX: return "radeon.uvd"; + case TN_RING_TYPE_VCE1_INDEX: return "radeon.vce1"; + case TN_RING_TYPE_VCE2_INDEX: return "radeon.vce2"; + default: WARN_ON_ONCE(1); return "radeon.unk"; + } +} + +static inline bool radeon_test_signaled(struct radeon_fence *fence) +{ + return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags); +} + +struct radeon_wait_cb { + struct fence_cb base; + void *task; +}; + +static void +radeon_fence_wait_cb(struct fence *fence, struct fence_cb *cb) +{ + struct radeon_wait_cb *wait = + container_of(cb, struct radeon_wait_cb, base); + wake_up_process(wait->task); +} + +static signed long radeon_fence_default_wait(struct fence *f, bool intr, + signed long t) +{ + struct radeon_fence *fence = to_radeon_fence(f); + struct radeon_device *rdev = fence->rdev; + struct radeon_wait_cb cb; + + cb.task = curproc; + + if (fence_add_callback(f, &cb.base, radeon_fence_wait_cb)) + return t; + + while (t > 0) { + if (intr) + set_current_state(TASK_INTERRUPTIBLE); + else + set_current_state(TASK_UNINTERRUPTIBLE); + + /* + * radeon_test_signaled must be called after + * set_current_state to prevent a race with wake_up_process + */ + if (radeon_test_signaled(fence)) + break; + + if (rdev->needs_reset) { + t = -EDEADLK; + break; + } + + KASSERT(sch_ident != NULL); + t = schedule_timeout(t); + + if (t > 0 && intr && signal_pending(current)) + t = -ERESTARTSYS; + } + + __set_current_state(TASK_RUNNING); + fence_remove_callback(f, &cb.base); + + return t; +} + +const struct fence_ops radeon_fence_ops = { + .get_driver_name = radeon_fence_get_driver_name, + .get_timeline_name = radeon_fence_get_timeline_name, + .enable_signaling = radeon_fence_enable_signaling, + .signaled = radeon_fence_is_signaled, + .wait = radeon_fence_default_wait, + .release = NULL, +}; diff --git a/sys/dev/pci/drm/radeon/radeon_gart.c b/sys/dev/pci/drm/radeon/radeon_gart.c index f77ac75f490..4ab155bd0c9 100644 --- a/sys/dev/pci/drm/radeon/radeon_gart.c +++ b/sys/dev/pci/drm/radeon/radeon_gart.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_gart.c,v 1.10 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -29,7 +28,6 @@ #include #include #include "radeon.h" -#include "radeon_reg.h" /* * GART @@ -80,7 +78,6 @@ int radeon_gart_table_ram_alloc(struct radeon_device *rdev) if (dmah == NULL) { return -ENOMEM; } - rdev->gart.dmah = dmah; rdev->gart.table_addr = dmah->map->dm_segs[0].ds_addr; rdev->gart.ptr = dmah->kva; @@ -102,6 +99,13 @@ void radeon_gart_table_ram_free(struct radeon_device *rdev) if (rdev->gart.ptr == NULL) { return; } +#ifdef CONFIG_X86 + if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || + rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { + set_memory_wb((unsigned long)rdev->gart.ptr, + rdev->gart.table_size >> PAGE_SHIFT); + } +#endif drm_dmamem_free(rdev->dmat, rdev->gart.dmah); rdev->gart.ptr = NULL; rdev->gart.table_addr = 0; @@ -124,7 +128,7 @@ int radeon_gart_table_vram_alloc(struct radeon_device *rdev) if (rdev->gart.robj == NULL) { r = radeon_bo_create(rdev, rdev->gart.table_size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM, - NULL, &rdev->gart.robj); + 0, NULL, NULL, &rdev->gart.robj); if (r) { return r; } @@ -161,6 +165,19 @@ int radeon_gart_table_vram_pin(struct radeon_device *rdev) radeon_bo_unpin(rdev->gart.robj); radeon_bo_unreserve(rdev->gart.robj); rdev->gart.table_addr = gpu_addr; + + if (!r) { + int i; + + /* We might have dropped some GART table updates while it wasn't + * mapped, restore all entries + */ + for (i = 0; i < rdev->gart.num_gpu_pages; i++) + radeon_gart_set_page(rdev, i, rdev->gart.pages_entry[i]); + mb(); + radeon_gart_tlb_flush(rdev); + } + return r; } @@ -202,7 +219,6 @@ void radeon_gart_table_vram_free(struct radeon_device *rdev) if (rdev->gart.robj == NULL) { return; } - radeon_gart_table_vram_unpin(rdev); radeon_bo_unref(&rdev->gart.robj); } @@ -225,7 +241,6 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, unsigned t; unsigned p; int i, j; - u64 page_base; if (!rdev->gart.ready) { WARN(1, "trying to unbind memory from uninitialized GART !\n"); @@ -236,18 +251,19 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, for (i = 0; i < pages; i++, p++) { if (rdev->gart.pages[p]) { rdev->gart.pages[p] = NULL; - rdev->gart.pages_addr[p] = rdev->dummy_page.addr; - page_base = rdev->gart.pages_addr[p]; for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { + rdev->gart.pages_entry[t] = rdev->dummy_page.entry; if (rdev->gart.ptr) { - radeon_gart_set_page(rdev, t, page_base); + radeon_gart_set_page(rdev, t, + rdev->dummy_page.entry); } - page_base += RADEON_GPU_PAGE_SIZE; } } } - mb(); - radeon_gart_tlb_flush(rdev); + if (rdev->gart.ptr) { + mb(); + radeon_gart_tlb_flush(rdev); + } } /** @@ -258,17 +274,19 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, * @pages: number of pages to bind * @pagelist: pages to bind * @dma_addr: DMA addresses of pages + * @flags: RADEON_GART_PAGE_* flags * * Binds the requested pages to the gart page table * (all asics). * Returns 0 for success, -EINVAL for failure. */ int radeon_gart_bind(struct radeon_device *rdev, unsigned offset, - int pages, struct vm_page **pagelist, bus_addr_t *dma_addr) + int pages, struct vm_page **pagelist, dma_addr_t *dma_addr, + uint32_t flags) { unsigned t; unsigned p; - uint64_t page_base; + uint64_t page_base, page_entry; int i, j; if (!rdev->gart.ready) { @@ -279,46 +297,22 @@ int radeon_gart_bind(struct radeon_device *rdev, unsigned offset, p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); for (i = 0; i < pages; i++, p++) { - rdev->gart.pages_addr[p] = dma_addr[i]; rdev->gart.pages[p] = pagelist[i]; - if (rdev->gart.ptr) { - page_base = rdev->gart.pages_addr[p]; - for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { - radeon_gart_set_page(rdev, t, page_base); - page_base += RADEON_GPU_PAGE_SIZE; - } - } - } - mb(); - radeon_gart_tlb_flush(rdev); - return 0; -} - -/** - * radeon_gart_restore - bind all pages in the gart page table - * - * @rdev: radeon_device pointer - * - * Binds all pages in the gart page table (all asics). - * Used to rebuild the gart table on device startup or resume. - */ -void radeon_gart_restore(struct radeon_device *rdev) -{ - int i, j, t; - u64 page_base; - - if (!rdev->gart.ptr) { - return; - } - for (i = 0, t = 0; i < rdev->gart.num_cpu_pages; i++) { - page_base = rdev->gart.pages_addr[i]; + page_base = dma_addr[i]; for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { - radeon_gart_set_page(rdev, t, page_base); + page_entry = radeon_gart_get_page_entry(page_base, flags); + rdev->gart.pages_entry[t] = page_entry; + if (rdev->gart.ptr) { + radeon_gart_set_page(rdev, t, page_entry); + } page_base += RADEON_GPU_PAGE_SIZE; } } - mb(); - radeon_gart_tlb_flush(rdev); + if (rdev->gart.ptr) { + mb(); + radeon_gart_tlb_flush(rdev); + } + return 0; } /** @@ -347,26 +341,23 @@ int radeon_gart_init(struct radeon_device *rdev) /* Compute table size */ rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE; rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE; -#ifdef DRMDEBUG DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n", rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages); -#endif /* Allocate pages table */ rdev->gart.pages = vzalloc(sizeof(void *) * rdev->gart.num_cpu_pages); if (rdev->gart.pages == NULL) { radeon_gart_fini(rdev); return -ENOMEM; } - rdev->gart.pages_addr = vzalloc(sizeof(bus_addr_t) * - rdev->gart.num_cpu_pages); - if (rdev->gart.pages_addr == NULL) { + rdev->gart.pages_entry = vmalloc(sizeof(uint64_t) * + rdev->gart.num_gpu_pages); + if (rdev->gart.pages_entry == NULL) { radeon_gart_fini(rdev); return -ENOMEM; } /* set GART entry to point to the dummy page by default */ - for (i = 0; i < rdev->gart.num_cpu_pages; i++) { - rdev->gart.pages_addr[i] = rdev->dummy_page.addr; - } + for (i = 0; i < rdev->gart.num_gpu_pages; i++) + rdev->gart.pages_entry[i] = rdev->dummy_page.entry; return 0; } @@ -379,918 +370,15 @@ int radeon_gart_init(struct radeon_device *rdev) */ void radeon_gart_fini(struct radeon_device *rdev) { - if (rdev->gart.pages && rdev->gart.pages_addr && rdev->gart.ready) { + if (rdev->gart.ready) { /* unbind pages */ radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages); } rdev->gart.ready = false; vfree(rdev->gart.pages); - vfree(rdev->gart.pages_addr); + vfree(rdev->gart.pages_entry); rdev->gart.pages = NULL; - rdev->gart.pages_addr = NULL; + rdev->gart.pages_entry = NULL; radeon_dummy_page_fini(rdev); } - -/* - * GPUVM - * GPUVM is similar to the legacy gart on older asics, however - * rather than there being a single global gart table - * for the entire GPU, there are multiple VM page tables active - * at any given time. The VM page tables can contain a mix - * vram pages and system memory pages and system memory pages - * can be mapped as snooped (cached system pages) or unsnooped - * (uncached system pages). - * Each VM has an ID associated with it and there is a page table - * associated with each VMID. When execting a command buffer, - * the kernel tells the the ring what VMID to use for that command - * buffer. VMIDs are allocated dynamically as commands are submitted. - * The userspace drivers maintain their own address space and the kernel - * sets up their pages tables accordingly when they submit their - * command buffers and a VMID is assigned. - * Cayman/Trinity support up to 8 active VMs at any given time; - * SI supports 16. - */ - -/* - * vm helpers - * - * TODO bind a default page at vm initialization for default address - */ - -/** - * radeon_vm_num_pde - return the number of page directory entries - * - * @rdev: radeon_device pointer - * - * Calculate the number of page directory entries (cayman+). - */ -static unsigned radeon_vm_num_pdes(struct radeon_device *rdev) -{ - return rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE; -} - -/** - * radeon_vm_directory_size - returns the size of the page directory in bytes - * - * @rdev: radeon_device pointer - * - * Calculate the size of the page directory in bytes (cayman+). - */ -static unsigned radeon_vm_directory_size(struct radeon_device *rdev) -{ - return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8); -} - -/** - * radeon_vm_manager_init - init the vm manager - * - * @rdev: radeon_device pointer - * - * Init the vm manager (cayman+). - * Returns 0 for success, error for failure. - */ -int radeon_vm_manager_init(struct radeon_device *rdev) -{ - struct radeon_vm *vm; - struct radeon_bo_va *bo_va; - int r; - unsigned size; - - if (!rdev->vm_manager.enabled) { - /* allocate enough for 2 full VM pts */ - size = radeon_vm_directory_size(rdev); - size += rdev->vm_manager.max_pfn * 8; - size *= 2; - r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager, - RADEON_GPU_PAGE_ALIGN(size), - RADEON_GPU_PAGE_SIZE, - RADEON_GEM_DOMAIN_VRAM); - if (r) { - dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n", - (rdev->vm_manager.max_pfn * 8) >> 10); - return r; - } - - r = radeon_asic_vm_init(rdev); - if (r) - return r; - - rdev->vm_manager.enabled = true; - - r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager); - if (r) - return r; - } - - /* restore page table */ - list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) { - if (vm->page_directory == NULL) - continue; - - list_for_each_entry(bo_va, &vm->va, vm_list) { - bo_va->valid = false; - } - } - return 0; -} - -/** - * radeon_vm_free_pt - free the page table for a specific vm - * - * @rdev: radeon_device pointer - * @vm: vm to unbind - * - * Free the page table of a specific vm (cayman+). - * - * Global and local mutex must be lock! - */ -static void radeon_vm_free_pt(struct radeon_device *rdev, - struct radeon_vm *vm) -{ - struct radeon_bo_va *bo_va; - int i; - - if (!vm->page_directory) - return; - - list_del_init(&vm->list); - radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); - - list_for_each_entry(bo_va, &vm->va, vm_list) { - bo_va->valid = false; - } - - if (vm->page_tables == NULL) - return; - - for (i = 0; i < radeon_vm_num_pdes(rdev); i++) - radeon_sa_bo_free(rdev, &vm->page_tables[i], vm->fence); - - kfree(vm->page_tables); -} - -/** - * radeon_vm_manager_fini - tear down the vm manager - * - * @rdev: radeon_device pointer - * - * Tear down the VM manager (cayman+). - */ -void radeon_vm_manager_fini(struct radeon_device *rdev) -{ - struct radeon_vm *vm, *tmp; - int i; - - if (!rdev->vm_manager.enabled) - return; - - mutex_lock(&rdev->vm_manager.lock); - /* free all allocated page tables */ - list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) { - mutex_lock(&vm->mutex); - radeon_vm_free_pt(rdev, vm); - mutex_unlock(&vm->mutex); - } - for (i = 0; i < RADEON_NUM_VM; ++i) { - radeon_fence_unref(&rdev->vm_manager.active[i]); - } - radeon_asic_vm_fini(rdev); - mutex_unlock(&rdev->vm_manager.lock); - - radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager); - radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager); - rdev->vm_manager.enabled = false; -} - -/** - * radeon_vm_evict - evict page table to make room for new one - * - * @rdev: radeon_device pointer - * @vm: VM we want to allocate something for - * - * Evict a VM from the lru, making sure that it isn't @vm. (cayman+). - * Returns 0 for success, -ENOMEM for failure. - * - * Global and local mutex must be locked! - */ -static int radeon_vm_evict(struct radeon_device *rdev, struct radeon_vm *vm) -{ - struct radeon_vm *vm_evict; - - if (list_empty(&rdev->vm_manager.lru_vm)) - return -ENOMEM; - - vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, - struct radeon_vm, list); - if (vm_evict == vm) - return -ENOMEM; - - mutex_lock(&vm_evict->mutex); - radeon_vm_free_pt(rdev, vm_evict); - mutex_unlock(&vm_evict->mutex); - return 0; -} - -/** - * radeon_vm_alloc_pt - allocates a page table for a VM - * - * @rdev: radeon_device pointer - * @vm: vm to bind - * - * Allocate a page table for the requested vm (cayman+). - * Returns 0 for success, error for failure. - * - * Global and local mutex must be locked! - */ -int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm) -{ - unsigned pd_size, pts_size; - u64 *pd_addr; - int r; - - if (vm == NULL) { - return -EINVAL; - } - - if (vm->page_directory != NULL) { - return 0; - } - -retry: - pd_size = RADEON_GPU_PAGE_ALIGN(radeon_vm_directory_size(rdev)); - r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, - &vm->page_directory, pd_size, - RADEON_GPU_PAGE_SIZE, false); - if (r == -ENOMEM) { - r = radeon_vm_evict(rdev, vm); - if (r) - return r; - goto retry; - - } else if (r) { - return r; - } - - vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->page_directory); - - /* Initially clear the page directory */ - pd_addr = radeon_sa_bo_cpu_addr(vm->page_directory); - memset(pd_addr, 0, pd_size); - - pts_size = radeon_vm_num_pdes(rdev) * sizeof(struct radeon_sa_bo *); - vm->page_tables = kzalloc(pts_size, GFP_KERNEL); - - if (vm->page_tables == NULL) { - DRM_ERROR("Cannot allocate memory for page table array\n"); - radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); - return -ENOMEM; - } - - return 0; -} - -/** - * radeon_vm_add_to_lru - add VMs page table to LRU list - * - * @rdev: radeon_device pointer - * @vm: vm to add to LRU - * - * Add the allocated page table to the LRU list (cayman+). - * - * Global mutex must be locked! - */ -void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm) -{ - list_del_init(&vm->list); - list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); -} - -/** - * radeon_vm_grab_id - allocate the next free VMID - * - * @rdev: radeon_device pointer - * @vm: vm to allocate id for - * @ring: ring we want to submit job to - * - * Allocate an id for the vm (cayman+). - * Returns the fence we need to sync to (if any). - * - * Global and local mutex must be locked! - */ -struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, - struct radeon_vm *vm, int ring) -{ - struct radeon_fence *best[RADEON_NUM_RINGS] = {}; - unsigned choices[2] = {}; - unsigned i; - - /* check if the id is still valid */ - if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id]) - return NULL; - - /* we definately need to flush */ - radeon_fence_unref(&vm->last_flush); - - /* skip over VMID 0, since it is the system VM */ - for (i = 1; i < rdev->vm_manager.nvm; ++i) { - struct radeon_fence *fence = rdev->vm_manager.active[i]; - - if (fence == NULL) { - /* found a free one */ - vm->id = i; - return NULL; - } - - if (radeon_fence_is_earlier(fence, best[fence->ring])) { - best[fence->ring] = fence; - choices[fence->ring == ring ? 0 : 1] = i; - } - } - - for (i = 0; i < 2; ++i) { - if (choices[i]) { - vm->id = choices[i]; - return rdev->vm_manager.active[choices[i]]; - } - } - - /* should never happen */ - BUG(); - return NULL; -} - -/** - * radeon_vm_fence - remember fence for vm - * - * @rdev: radeon_device pointer - * @vm: vm we want to fence - * @fence: fence to remember - * - * Fence the vm (cayman+). - * Set the fence used to protect page table and id. - * - * Global and local mutex must be locked! - */ -void radeon_vm_fence(struct radeon_device *rdev, - struct radeon_vm *vm, - struct radeon_fence *fence) -{ - radeon_fence_unref(&rdev->vm_manager.active[vm->id]); - rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence); - - radeon_fence_unref(&vm->fence); - vm->fence = radeon_fence_ref(fence); -} - -/** - * radeon_vm_bo_find - find the bo_va for a specific vm & bo - * - * @vm: requested vm - * @bo: requested buffer object - * - * Find @bo inside the requested vm (cayman+). - * Search inside the @bos vm list for the requested vm - * Returns the found bo_va or NULL if none is found - * - * Object has to be reserved! - */ -struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm, - struct radeon_bo *bo) -{ - struct radeon_bo_va *bo_va; - - list_for_each_entry(bo_va, &bo->va, bo_list) { - if (bo_va->vm == vm) { - return bo_va; - } - } - return NULL; -} - -/** - * radeon_vm_bo_add - add a bo to a specific vm - * - * @rdev: radeon_device pointer - * @vm: requested vm - * @bo: radeon buffer object - * - * Add @bo into the requested vm (cayman+). - * Add @bo to the list of bos associated with the vm - * Returns newly added bo_va or NULL for failure - * - * Object has to be reserved! - */ -struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev, - struct radeon_vm *vm, - struct radeon_bo *bo) -{ - struct radeon_bo_va *bo_va; - - bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL); - if (bo_va == NULL) { - return NULL; - } - bo_va->vm = vm; - bo_va->bo = bo; - bo_va->soffset = 0; - bo_va->eoffset = 0; - bo_va->flags = 0; - bo_va->valid = false; - bo_va->ref_count = 1; - INIT_LIST_HEAD(&bo_va->bo_list); - INIT_LIST_HEAD(&bo_va->vm_list); - - mutex_lock(&vm->mutex); - list_add(&bo_va->vm_list, &vm->va); - list_add_tail(&bo_va->bo_list, &bo->va); - mutex_unlock(&vm->mutex); - - return bo_va; -} - -/** - * radeon_vm_bo_set_addr - set bos virtual address inside a vm - * - * @rdev: radeon_device pointer - * @bo_va: bo_va to store the address - * @soffset: requested offset of the buffer in the VM address space - * @flags: attributes of pages (read/write/valid/etc.) - * - * Set offset of @bo_va (cayman+). - * Validate and set the offset requested within the vm address space. - * Returns 0 for success, error for failure. - * - * Object has to be reserved! - */ -int radeon_vm_bo_set_addr(struct radeon_device *rdev, - struct radeon_bo_va *bo_va, - uint64_t soffset, - uint32_t flags) -{ - uint64_t size = radeon_bo_size(bo_va->bo); - uint64_t eoffset, last_offset = 0; - struct radeon_vm *vm = bo_va->vm; - struct radeon_bo_va *tmp; - struct list_head *head; - unsigned last_pfn; - - if (soffset) { - /* make sure object fit at this offset */ - eoffset = soffset + size; - if (soffset >= eoffset) { - return -EINVAL; - } - - last_pfn = eoffset / RADEON_GPU_PAGE_SIZE; - if (last_pfn > rdev->vm_manager.max_pfn) { - dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", - last_pfn, rdev->vm_manager.max_pfn); - return -EINVAL; - } - - } else { - eoffset = last_pfn = 0; - } - - mutex_lock(&vm->mutex); - head = &vm->va; - last_offset = 0; - list_for_each_entry(tmp, &vm->va, vm_list) { - if (bo_va == tmp) { - /* skip over currently modified bo */ - continue; - } - - if (soffset >= last_offset && eoffset <= tmp->soffset) { - /* bo can be added before this one */ - break; - } - if (eoffset > tmp->soffset && soffset < tmp->eoffset) { - /* bo and tmp overlap, invalid offset */ - dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n", - bo_va->bo, (unsigned)bo_va->soffset, tmp->bo, - (unsigned)tmp->soffset, (unsigned)tmp->eoffset); - mutex_unlock(&vm->mutex); - return -EINVAL; - } - last_offset = tmp->eoffset; - head = &tmp->vm_list; - } - - bo_va->soffset = soffset; - bo_va->eoffset = eoffset; - bo_va->flags = flags; - bo_va->valid = false; - list_move(&bo_va->vm_list, head); - - mutex_unlock(&vm->mutex); - return 0; -} - -/** - * radeon_vm_map_gart - get the physical address of a gart page - * - * @rdev: radeon_device pointer - * @addr: the unmapped addr - * - * Look up the physical address of the page that the pte resolves - * to (cayman+). - * Returns the physical address of the page. - */ -uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr) -{ - uint64_t result; - - /* page table offset */ - result = rdev->gart.pages_addr[addr >> PAGE_SHIFT]; - - /* in case cpu page size != gpu page size*/ - result |= addr & (PAGE_MASK); - - return result; -} - -/** - * radeon_vm_update_pdes - make sure that page directory is valid - * - * @rdev: radeon_device pointer - * @vm: requested vm - * @start: start of GPU address range - * @end: end of GPU address range - * - * Allocates new page tables if necessary - * and updates the page directory (cayman+). - * Returns 0 for success, error for failure. - * - * Global and local mutex must be locked! - */ -static int radeon_vm_update_pdes(struct radeon_device *rdev, - struct radeon_vm *vm, - uint64_t start, uint64_t end) -{ - static const uint32_t incr = RADEON_VM_PTE_COUNT * 8; - - uint64_t last_pde = ~0, last_pt = ~0; - unsigned count = 0; - uint64_t pt_idx; - int r; - - start = (start / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; - end = (end / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; - - /* walk over the address space and update the page directory */ - for (pt_idx = start; pt_idx <= end; ++pt_idx) { - uint64_t pde, pt; - - if (vm->page_tables[pt_idx]) - continue; - -retry: - r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, - &vm->page_tables[pt_idx], - RADEON_VM_PTE_COUNT * 8, - RADEON_GPU_PAGE_SIZE, false); - - if (r == -ENOMEM) { - r = radeon_vm_evict(rdev, vm); - if (r) - return r; - goto retry; - } else if (r) { - return r; - } - - pde = vm->pd_gpu_addr + pt_idx * 8; - - pt = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); - - if (((last_pde + 8 * count) != pde) || - ((last_pt + incr * count) != pt)) { - - if (count) { - radeon_asic_vm_set_page(rdev, last_pde, - last_pt, count, incr, - RADEON_VM_PAGE_VALID); - } - - count = 1; - last_pde = pde; - last_pt = pt; - } else { - ++count; - } - } - - if (count) { - radeon_asic_vm_set_page(rdev, last_pde, last_pt, count, - incr, RADEON_VM_PAGE_VALID); - - } - - return 0; -} - -/** - * radeon_vm_update_ptes - make sure that page tables are valid - * - * @rdev: radeon_device pointer - * @vm: requested vm - * @start: start of GPU address range - * @end: end of GPU address range - * @dst: destination address to map to - * @flags: mapping flags - * - * Update the page tables in the range @start - @end (cayman+). - * - * Global and local mutex must be locked! - */ -static void radeon_vm_update_ptes(struct radeon_device *rdev, - struct radeon_vm *vm, - uint64_t start, uint64_t end, - uint64_t dst, uint32_t flags) -{ - static const uint64_t mask = RADEON_VM_PTE_COUNT - 1; - - uint64_t last_pte = ~0, last_dst = ~0; - unsigned count = 0; - uint64_t addr; - - start = start / RADEON_GPU_PAGE_SIZE; - end = end / RADEON_GPU_PAGE_SIZE; - - /* walk over the address space and update the page tables */ - for (addr = start; addr < end; ) { - uint64_t pt_idx = addr >> RADEON_VM_BLOCK_SIZE; - unsigned nptes; - uint64_t pte; - - if ((addr & ~mask) == (end & ~mask)) - nptes = end - addr; - else - nptes = RADEON_VM_PTE_COUNT - (addr & mask); - - pte = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); - pte += (addr & mask) * 8; - - if ((last_pte + 8 * count) != pte) { - - if (count) { - radeon_asic_vm_set_page(rdev, last_pte, - last_dst, count, - RADEON_GPU_PAGE_SIZE, - flags); - } - - count = nptes; - last_pte = pte; - last_dst = dst; - } else { - count += nptes; - } - - addr += nptes; - dst += nptes * RADEON_GPU_PAGE_SIZE; - } - - if (count) { - radeon_asic_vm_set_page(rdev, last_pte, last_dst, count, - RADEON_GPU_PAGE_SIZE, flags); - } -} - -/** - * radeon_vm_bo_update_pte - map a bo into the vm page table - * - * @rdev: radeon_device pointer - * @vm: requested vm - * @bo: radeon buffer object - * @mem: ttm mem - * - * Fill in the page table entries for @bo (cayman+). - * Returns 0 for success, -EINVAL for failure. - * - * Object have to be reserved & global and local mutex must be locked! - */ -int radeon_vm_bo_update_pte(struct radeon_device *rdev, - struct radeon_vm *vm, - struct radeon_bo *bo, - struct ttm_mem_reg *mem) -{ - unsigned ridx = rdev->asic->vm.pt_ring_index; - struct radeon_ring *ring = &rdev->ring[ridx]; - struct radeon_semaphore *sem = NULL; - struct radeon_bo_va *bo_va; - unsigned nptes, npdes, ndw; - uint64_t addr; - int r; - - /* nothing to do if vm isn't bound */ - if (vm->page_directory == NULL) - return 0; - - bo_va = radeon_vm_bo_find(vm, bo); - if (bo_va == NULL) { - dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm); - return -EINVAL; - } - - if (!bo_va->soffset) { - dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n", - bo, vm); - return -EINVAL; - } - - if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL)) - return 0; - - bo_va->flags &= ~RADEON_VM_PAGE_VALID; - bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM; - if (mem) { - addr = mem->start << PAGE_SHIFT; - if (mem->mem_type != TTM_PL_SYSTEM) { - bo_va->flags |= RADEON_VM_PAGE_VALID; - bo_va->valid = true; - } - if (mem->mem_type == TTM_PL_TT) { - bo_va->flags |= RADEON_VM_PAGE_SYSTEM; - } else { - addr += rdev->vm_manager.vram_base_offset; - } - } else { - addr = 0; - bo_va->valid = false; - } - - if (vm->fence && radeon_fence_signaled(vm->fence)) { - radeon_fence_unref(&vm->fence); - } - - if (vm->fence && vm->fence->ring != ridx) { - r = radeon_semaphore_create(rdev, &sem); - if (r) { - return r; - } - } - - nptes = radeon_bo_ngpu_pages(bo); - - /* assume two extra pdes in case the mapping overlaps the borders */ - npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2; - - /* estimate number of dw needed */ - /* semaphore, fence and padding */ - ndw = 32; - - if (RADEON_VM_BLOCK_SIZE > 11) - /* reserve space for one header for every 2k dwords */ - ndw += (nptes >> 11) * 4; - else - /* reserve space for one header for - every (1 << BLOCK_SIZE) entries */ - ndw += (nptes >> RADEON_VM_BLOCK_SIZE) * 4; - - /* reserve space for pte addresses */ - ndw += nptes * 2; - - /* reserve space for one header for every 2k dwords */ - ndw += (npdes >> 11) * 4; - - /* reserve space for pde addresses */ - ndw += npdes * 2; - - r = radeon_ring_lock(rdev, ring, ndw); - if (r) { - return r; - } - - if (sem && radeon_fence_need_sync(vm->fence, ridx)) { - radeon_semaphore_sync_rings(rdev, sem, vm->fence->ring, ridx); - radeon_fence_note_sync(vm->fence, ridx); - } - - r = radeon_vm_update_pdes(rdev, vm, bo_va->soffset, bo_va->eoffset); - if (r) { - radeon_ring_unlock_undo(rdev, ring); - return r; - } - - radeon_vm_update_ptes(rdev, vm, bo_va->soffset, bo_va->eoffset, - addr, bo_va->flags); - - radeon_fence_unref(&vm->fence); - r = radeon_fence_emit(rdev, &vm->fence, ridx); - if (r) { - radeon_ring_unlock_undo(rdev, ring); - return r; - } - radeon_ring_unlock_commit(rdev, ring); - radeon_semaphore_free(rdev, &sem, vm->fence); - radeon_fence_unref(&vm->last_flush); - - return 0; -} - -/** - * radeon_vm_bo_rmv - remove a bo to a specific vm - * - * @rdev: radeon_device pointer - * @bo_va: requested bo_va - * - * Remove @bo_va->bo from the requested vm (cayman+). - * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and - * remove the ptes for @bo_va in the page table. - * Returns 0 for success. - * - * Object have to be reserved! - */ -int radeon_vm_bo_rmv(struct radeon_device *rdev, - struct radeon_bo_va *bo_va) -{ - int r = 0; - - mutex_lock(&rdev->vm_manager.lock); - mutex_lock(&bo_va->vm->mutex); - if (bo_va->soffset) { - r = radeon_vm_bo_update_pte(rdev, bo_va->vm, bo_va->bo, NULL); - } - mutex_unlock(&rdev->vm_manager.lock); - list_del(&bo_va->vm_list); - mutex_unlock(&bo_va->vm->mutex); - list_del(&bo_va->bo_list); - - kfree(bo_va); - return r; -} - -/** - * radeon_vm_bo_invalidate - mark the bo as invalid - * - * @rdev: radeon_device pointer - * @vm: requested vm - * @bo: radeon buffer object - * - * Mark @bo as invalid (cayman+). - */ -void radeon_vm_bo_invalidate(struct radeon_device *rdev, - struct radeon_bo *bo) -{ - struct radeon_bo_va *bo_va; - - list_for_each_entry(bo_va, &bo->va, bo_list) { - bo_va->valid = false; - } -} - -/** - * radeon_vm_init - initialize a vm instance - * - * @rdev: radeon_device pointer - * @vm: requested vm - * - * Init @vm fields (cayman+). - */ -void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) -{ - vm->id = 0; - vm->fence = NULL; - rw_init(&vm->mutex, "vmlk"); - INIT_LIST_HEAD(&vm->list); - INIT_LIST_HEAD(&vm->va); -} - -/** - * radeon_vm_fini - tear down a vm instance - * - * @rdev: radeon_device pointer - * @vm: requested vm - * - * Tear down @vm (cayman+). - * Unbind the VM and remove all bos from the vm bo list - */ -void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) -{ - struct radeon_bo_va *bo_va, *tmp; - int r; - - mutex_lock(&rdev->vm_manager.lock); - mutex_lock(&vm->mutex); - radeon_vm_free_pt(rdev, vm); - mutex_unlock(&rdev->vm_manager.lock); - - if (!list_empty(&vm->va)) { - dev_err(rdev->dev, "still active bo inside vm\n"); - } - list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) { - list_del_init(&bo_va->vm_list); - r = radeon_bo_reserve(bo_va->bo, false); - if (!r) { - list_del_init(&bo_va->bo_list); - radeon_bo_unreserve(bo_va->bo); - kfree(bo_va); - } - } - radeon_fence_unref(&vm->fence); - radeon_fence_unref(&vm->last_flush); - mutex_unlock(&vm->mutex); -} diff --git a/sys/dev/pci/drm/radeon/radeon_gem.c b/sys/dev/pci/drm/radeon/radeon_gem.c index 338fdc2a1ca..c9a4478b254 100644 --- a/sys/dev/pci/drm/radeon/radeon_gem.c +++ b/sys/dev/pci/drm/radeon/radeon_gem.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_gem.c,v 1.10 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -39,13 +38,14 @@ void radeon_gem_object_free(struct drm_gem_object *gobj) if (robj->gem_base.import_attach) drm_prime_gem_destroy(&robj->gem_base, robj->tbo.sg); #endif + radeon_mn_unregister(robj); radeon_bo_unref(&robj); } } -int radeon_gem_object_create(struct radeon_device *rdev, int size, +int radeon_gem_object_create(struct radeon_device *rdev, unsigned long size, int alignment, int initial_domain, - bool discardable, bool kernel, + u32 flags, bool kernel, struct drm_gem_object **obj) { struct radeon_bo *robj; @@ -58,28 +58,32 @@ int radeon_gem_object_create(struct radeon_device *rdev, int size, alignment = PAGE_SIZE; } - /* maximun bo size is the minimun btw visible vram and gtt size */ - max_size = min(rdev->mc.visible_vram_size, rdev->mc.gtt_size); + /* Maximum bo size is the unpinned gtt size since we use the gtt to + * handle vram to system pool migrations. + */ + max_size = rdev->mc.gtt_size - rdev->gart_pin_size; if (size > max_size) { - printk(KERN_WARNING "%s:%d alloc size %dMb bigger than %ldMb limit\n", - __func__, __LINE__, size >> 20, max_size >> 20); + DRM_DEBUG("Allocation size %ldMb bigger than %ldMb limit\n", + size >> 20, max_size >> 20); return -ENOMEM; } retry: - r = radeon_bo_create(rdev, size, alignment, kernel, initial_domain, NULL, &robj); + r = radeon_bo_create(rdev, size, alignment, kernel, initial_domain, + flags, NULL, NULL, &robj); if (r) { if (r != -ERESTARTSYS) { if (initial_domain == RADEON_GEM_DOMAIN_VRAM) { initial_domain |= RADEON_GEM_DOMAIN_GTT; goto retry; } - DRM_ERROR("Failed to allocate GEM object (%d, %d, %u, %d)\n", + DRM_ERROR("Failed to allocate GEM object (%ld, %d, %u, %d)\n", size, initial_domain, alignment, r); } return r; } *obj = &robj->gem_base; + robj->pid = curproc->p_p->ps_pid; mutex_lock(&rdev->gem.mutex); list_add_tail(&robj->list, &rdev->gem.objects); @@ -88,12 +92,12 @@ retry: return 0; } -int radeon_gem_set_domain(struct drm_gem_object *gobj, +static int radeon_gem_set_domain(struct drm_gem_object *gobj, uint32_t rdomain, uint32_t wdomain) { struct radeon_bo *robj; uint32_t domain; - int r; + long r; /* FIXME: reeimplement */ robj = gem_to_radeon_bo(gobj); @@ -109,9 +113,12 @@ int radeon_gem_set_domain(struct drm_gem_object *gobj, } if (domain == RADEON_GEM_DOMAIN_CPU) { /* Asking for cpu access wait for object idle */ - r = radeon_bo_wait(robj, NULL, false); - if (r) { - printk(KERN_ERR "Failed to wait for object !\n"); + r = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true, 30 * HZ); + if (!r) + r = -EBUSY; + + if (r < 0 && r != -EINTR) { + printk(KERN_ERR "Failed to wait for object: %li\n", r); return r; } } @@ -142,7 +149,8 @@ int radeon_gem_object_open(struct drm_gem_object *obj, struct drm_file *file_pri struct radeon_bo_va *bo_va; int r; - if (rdev->family < CHIP_CAYMAN) { + if ((rdev->family < CHIP_CAYMAN) || + (!rdev->accel_working)) { return 0; } @@ -172,7 +180,8 @@ void radeon_gem_object_close(struct drm_gem_object *obj, struct radeon_bo_va *bo_va; int r; - if (rdev->family < CHIP_CAYMAN) { + if ((rdev->family < CHIP_CAYMAN) || + (!rdev->accel_working)) { return; } @@ -210,18 +219,15 @@ int radeon_gem_info_ioctl(struct drm_device *dev, void *data, struct radeon_device *rdev = dev->dev_private; struct drm_radeon_gem_info *args = data; struct ttm_mem_type_manager *man; - unsigned i; man = &rdev->mman.bdev.man[TTM_PL_VRAM]; args->vram_size = rdev->mc.real_vram_size; args->vram_visible = (u64)man->size << PAGE_SHIFT; - if (rdev->stollen_vga_memory) - args->vram_visible -= radeon_bo_size(rdev->stollen_vga_memory); - args->vram_visible -= radeon_fbdev_total_size(rdev); - args->gart_size = rdev->mc.gtt_size - 4096 - RADEON_IB_POOL_SIZE*64*1024; - for(i = 0; i < RADEON_NUM_RINGS; ++i) - args->gart_size -= rdev->ring[i].ring_size; + args->vram_visible -= rdev->vram_pin_size; + args->gart_size = rdev->mc.gtt_size; + args->gart_size -= rdev->gart_pin_size; + return 0; } @@ -254,8 +260,8 @@ int radeon_gem_create_ioctl(struct drm_device *dev, void *data, /* create a gem object to contain this object in */ args->size = roundup(args->size, PAGE_SIZE); r = radeon_gem_object_create(rdev, args->size, args->alignment, - args->initial_domain, false, - false, &gobj); + args->initial_domain, args->flags, + false, &gobj); if (r) { up_read(&rdev->exclusive_lock); r = radeon_gem_handle_lockup(rdev, r); @@ -274,6 +280,97 @@ int radeon_gem_create_ioctl(struct drm_device *dev, void *data, return 0; } +int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + return -ENOSYS; +#ifdef notyet + struct radeon_device *rdev = dev->dev_private; + struct drm_radeon_gem_userptr *args = data; + struct drm_gem_object *gobj; + struct radeon_bo *bo; + uint32_t handle; + int r; + + if (offset_in_page(args->addr | args->size)) + return -EINVAL; + + /* reject unknown flag values */ + if (args->flags & ~(RADEON_GEM_USERPTR_READONLY | + RADEON_GEM_USERPTR_ANONONLY | RADEON_GEM_USERPTR_VALIDATE | + RADEON_GEM_USERPTR_REGISTER)) + return -EINVAL; + + if (args->flags & RADEON_GEM_USERPTR_READONLY) { + /* readonly pages not tested on older hardware */ + if (rdev->family < CHIP_R600) + return -EINVAL; + + } else if (!(args->flags & RADEON_GEM_USERPTR_ANONONLY) || + !(args->flags & RADEON_GEM_USERPTR_REGISTER)) { + + /* if we want to write to it we must require anonymous + memory and install a MMU notifier */ + return -EACCES; + } + + down_read(&rdev->exclusive_lock); + + /* create a gem object to contain this object in */ + r = radeon_gem_object_create(rdev, args->size, 0, + RADEON_GEM_DOMAIN_CPU, 0, + false, &gobj); + if (r) + goto handle_lockup; + + bo = gem_to_radeon_bo(gobj); + r = radeon_ttm_tt_set_userptr(bo->tbo.ttm, args->addr, args->flags); + if (r) + goto release_object; + + if (args->flags & RADEON_GEM_USERPTR_REGISTER) { + r = radeon_mn_register(bo, args->addr); + if (r) + goto release_object; + } + + if (args->flags & RADEON_GEM_USERPTR_VALIDATE) { + down_read(¤t->mm->mmap_sem); + r = radeon_bo_reserve(bo, true); + if (r) { + up_read(¤t->mm->mmap_sem); + goto release_object; + } + + radeon_ttm_placement_from_domain(bo, RADEON_GEM_DOMAIN_GTT); + r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); + radeon_bo_unreserve(bo); + up_read(¤t->mm->mmap_sem); + if (r) + goto release_object; + } + + r = drm_gem_handle_create(filp, gobj, &handle); + /* drop reference from allocate - handle holds it now */ + drm_gem_object_unreference_unlocked(gobj); + if (r) + goto handle_lockup; + + args->handle = handle; + up_read(&rdev->exclusive_lock); + return 0; + +release_object: + drm_gem_object_unreference_unlocked(gobj); + +handle_lockup: + up_read(&rdev->exclusive_lock); + r = radeon_gem_handle_lockup(rdev, r); + + return r; +#endif +} + int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { @@ -317,6 +414,10 @@ int radeon_mode_dumb_mmap(struct drm_file *filp, return -ENOENT; } robj = gem_to_radeon_bo(gobj); + if (radeon_ttm_tt_has_userptr(robj->tbo.ttm)) { + drm_gem_object_unreference_unlocked(gobj); + return -EPERM; + } *offset_p = radeon_bo_mmap_offset(robj); drm_gem_object_unreference_unlocked(gobj); return 0; @@ -333,7 +434,6 @@ int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data, int radeon_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { - struct radeon_device *rdev = dev->dev_private; struct drm_radeon_gem_busy *args = data; struct drm_gem_object *gobj; struct radeon_bo *robj; @@ -345,21 +445,16 @@ int radeon_gem_busy_ioctl(struct drm_device *dev, void *data, return -ENOENT; } robj = gem_to_radeon_bo(gobj); - r = radeon_bo_wait(robj, &cur_placement, true); - switch (cur_placement) { - case TTM_PL_VRAM: - args->domain = RADEON_GEM_DOMAIN_VRAM; - break; - case TTM_PL_TT: - args->domain = RADEON_GEM_DOMAIN_GTT; - break; - case TTM_PL_SYSTEM: - args->domain = RADEON_GEM_DOMAIN_CPU; - default: - break; - } + + r = reservation_object_test_signaled_rcu(robj->tbo.resv, true); + if (r == 0) + r = -EBUSY; + else + r = 0; + + cur_placement = ACCESS_ONCE(robj->tbo.mem.mem_type); + args->domain = radeon_mem_type_to_domain(cur_placement); drm_gem_object_unreference_unlocked(gobj); - r = radeon_gem_handle_lockup(rdev, r); return r; } @@ -370,17 +465,27 @@ int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data, struct drm_radeon_gem_wait_idle *args = data; struct drm_gem_object *gobj; struct radeon_bo *robj; - int r; + int r = 0; + uint32_t cur_placement = 0; + long ret; gobj = drm_gem_object_lookup(dev, filp, args->handle); if (gobj == NULL) { return -ENOENT; } robj = gem_to_radeon_bo(gobj); - r = radeon_bo_wait(robj, NULL, false); - /* callback hw specific functions if any */ - if (rdev->asic->ioctl_wait_idle) - robj->rdev->asic->ioctl_wait_idle(rdev, robj); + + ret = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true, 30 * HZ); + if (ret == 0) + r = -EBUSY; + else if (ret < 0) + r = ret; + + /* Flush HDP cache via MMIO if necessary */ + cur_placement = ACCESS_ONCE(robj->tbo.mem.mem_type); + if (rdev->asic->mmio_hdp_flush && + radeon_mem_type_to_domain(cur_placement) == RADEON_GEM_DOMAIN_VRAM) + robj->rdev->asic->mmio_hdp_flush(rdev); drm_gem_object_unreference_unlocked(gobj); r = radeon_gem_handle_lockup(rdev, r); return r; @@ -427,6 +532,68 @@ out: return r; } +/** + * radeon_gem_va_update_vm -update the bo_va in its VM + * + * @rdev: radeon_device pointer + * @bo_va: bo_va to update + * + * Update the bo_va directly after setting it's address. Errors are not + * vital here, so they are not reported back to userspace. + */ +static void radeon_gem_va_update_vm(struct radeon_device *rdev, + struct radeon_bo_va *bo_va) +{ + struct ttm_validate_buffer tv, *entry; + struct radeon_bo_list *vm_bos; + struct ww_acquire_ctx ticket; + struct list_head list; + unsigned domain; + int r; + + INIT_LIST_HEAD(&list); + + tv.bo = &bo_va->bo->tbo; + tv.shared = true; + list_add(&tv.head, &list); + + vm_bos = radeon_vm_get_bos(rdev, bo_va->vm, &list); + if (!vm_bos) + return; + + r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL); + if (r) + goto error_free; + + list_for_each_entry(entry, &list, head) { + domain = radeon_mem_type_to_domain(entry->bo->mem.mem_type); + /* if anything is swapped out don't swap it in here, + just abort and wait for the next CS */ + if (domain == RADEON_GEM_DOMAIN_CPU) + goto error_unreserve; + } + + mutex_lock(&bo_va->vm->mutex); + r = radeon_vm_clear_freed(rdev, bo_va->vm); + if (r) + goto error_unlock; + + if (bo_va->it.start) + r = radeon_vm_bo_update(rdev, bo_va, &bo_va->bo->tbo.mem); + +error_unlock: + mutex_unlock(&bo_va->vm->mutex); + +error_unreserve: + ttm_eu_backoff_reservation(&ticket, &list); + +error_free: + drm_free_large(vm_bos); + + if (r && r != -ERESTARTSYS) + DRM_ERROR("Couldn't update BO_VA (%d)\n", r); +} + int radeon_gem_va_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { @@ -474,11 +641,6 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data, args->operation = RADEON_VA_RESULT_ERROR; return -EINVAL; } - if (!(args->flags & RADEON_VM_PAGE_SNOOPED)) { - dev_err(&dev->pdev->dev, "only supported snooped mapping for now\n"); - args->operation = RADEON_VA_RESULT_ERROR; - return -EINVAL; - } switch (args->operation) { case RADEON_VA_MAP: @@ -512,9 +674,10 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data, switch (args->operation) { case RADEON_VA_MAP: - if (bo_va->soffset) { + if (bo_va->it.start) { args->operation = RADEON_VA_RESULT_VA_EXIST; - args->offset = bo_va->soffset; + args->offset = bo_va->it.start * RADEON_GPU_PAGE_SIZE; + radeon_bo_unreserve(rbo); goto out; } r = radeon_vm_bo_set_addr(rdev, bo_va, args->offset, args->flags); @@ -525,12 +688,54 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data, default: break; } + if (!r) + radeon_gem_va_update_vm(rdev, bo_va); args->operation = RADEON_VA_RESULT_OK; if (r) { args->operation = RADEON_VA_RESULT_ERROR; } out: - radeon_bo_unreserve(rbo); + drm_gem_object_unreference_unlocked(gobj); + return r; +} + +int radeon_gem_op_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + struct drm_radeon_gem_op *args = data; + struct drm_gem_object *gobj; + struct radeon_bo *robj; + int r; + + gobj = drm_gem_object_lookup(dev, filp, args->handle); + if (gobj == NULL) { + return -ENOENT; + } + robj = gem_to_radeon_bo(gobj); + + r = -EPERM; + if (radeon_ttm_tt_has_userptr(robj->tbo.ttm)) + goto out; + + r = radeon_bo_reserve(robj, false); + if (unlikely(r)) + goto out; + + switch (args->op) { + case RADEON_GEM_OP_GET_INITIAL_DOMAIN: + args->value = robj->initial_domain; + break; + case RADEON_GEM_OP_SET_INITIAL_DOMAIN: + robj->initial_domain = args->value & (RADEON_GEM_DOMAIN_VRAM | + RADEON_GEM_DOMAIN_GTT | + RADEON_GEM_DOMAIN_CPU); + break; + default: + r = -EINVAL; + } + + radeon_bo_unreserve(robj); +out: drm_gem_object_unreference_unlocked(gobj); return r; } @@ -546,12 +751,11 @@ int radeon_mode_dumb_create(struct drm_file *file_priv, args->pitch = radeon_align_pitch(rdev, args->width, args->bpp, 0) * ((args->bpp + 1) / 8); args->size = args->pitch * args->height; - args->size = PAGE_ALIGN(args->size); + args->size = roundup2(args->size, PAGE_SIZE); r = radeon_gem_object_create(rdev, args->size, 0, - RADEON_GEM_DOMAIN_VRAM, - false, ttm_bo_type_device, - &gobj); + RADEON_GEM_DOMAIN_VRAM, 0, + false, &gobj); if (r) return -ENOMEM; @@ -565,9 +769,51 @@ int radeon_mode_dumb_create(struct drm_file *file_priv, return 0; } -int radeon_mode_dumb_destroy(struct drm_file *file_priv, - struct drm_device *dev, - uint32_t handle) +#if defined(CONFIG_DEBUG_FS) +static int radeon_debugfs_gem_info(struct seq_file *m, void *data) { - return drm_gem_handle_delete(file_priv, handle); + struct drm_info_node *node = (struct drm_info_node *)m->private; + struct drm_device *dev = node->minor->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_bo *rbo; + unsigned i = 0; + + mutex_lock(&rdev->gem.mutex); + list_for_each_entry(rbo, &rdev->gem.objects, list) { + unsigned domain; + const char *placement; + + domain = radeon_mem_type_to_domain(rbo->tbo.mem.mem_type); + switch (domain) { + case RADEON_GEM_DOMAIN_VRAM: + placement = "VRAM"; + break; + case RADEON_GEM_DOMAIN_GTT: + placement = " GTT"; + break; + case RADEON_GEM_DOMAIN_CPU: + default: + placement = " CPU"; + break; + } + seq_printf(m, "bo[0x%08x] %8ldkB %8ldMB %s pid %8ld\n", + i, radeon_bo_size(rbo) >> 10, radeon_bo_size(rbo) >> 20, + placement, (unsigned long)rbo->pid); + i++; + } + mutex_unlock(&rdev->gem.mutex); + return 0; +} + +static struct drm_info_list radeon_debugfs_gem_list[] = { + {"radeon_gem_info", &radeon_debugfs_gem_info, 0, NULL}, +}; +#endif + +int radeon_gem_debugfs_init(struct radeon_device *rdev) +{ +#if defined(CONFIG_DEBUG_FS) + return radeon_debugfs_add_files(rdev, radeon_debugfs_gem_list, 1); +#endif + return 0; } diff --git a/sys/dev/pci/drm/radeon/radeon_i2c.c b/sys/dev/pci/drm/radeon/radeon_i2c.c index eba4d3a6553..2794f28b9d0 100644 --- a/sys/dev/pci/drm/radeon/radeon_i2c.c +++ b/sys/dev/pci/drm/radeon/radeon_i2c.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_i2c.c,v 1.9 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2007-8 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -90,13 +89,15 @@ bool radeon_ddc_probe(struct radeon_connector *radeon_connector, bool use_aux) /* bit banging i2c */ -static int pre_xfer(void *cookie) +static int pre_xfer(struct i2c_adapter *i2c_adap) { - struct radeon_i2c_chan *i2c = cookie; + struct radeon_i2c_chan *i2c = i2c_get_adapdata(i2c_adap); struct radeon_device *rdev = i2c->dev->dev_private; struct radeon_i2c_bus_rec *rec = &i2c->rec; uint32_t temp; + mutex_lock(&i2c->mutex); + /* RV410 appears to have a bug where the hw i2c in reset * holds the i2c port in a bad state - switch hw i2c away before * doing DDC - do this for all r200s/r300s/r400s for safety sake @@ -158,9 +159,9 @@ static int pre_xfer(void *cookie) return 0; } -static void post_xfer(void *cookie) +static void post_xfer(struct i2c_adapter *i2c_adap) { - struct radeon_i2c_chan *i2c = cookie; + struct radeon_i2c_chan *i2c = i2c_get_adapdata(i2c_adap); struct radeon_device *rdev = i2c->dev->dev_private; struct radeon_i2c_bus_rec *rec = &i2c->rec; uint32_t temp; @@ -173,6 +174,8 @@ static void post_xfer(void *cookie) temp = RREG32(rec->mask_data_reg) & ~rec->mask_data_mask; WREG32(rec->mask_data_reg, temp); temp = RREG32(rec->mask_data_reg); + + mutex_unlock(&i2c->mutex); } static int get_clock(void *i2c_priv) @@ -280,14 +283,16 @@ radeon_bb_read_bits(void *cookie) int radeon_acquire_bus(void *cookie, int flags) { - pre_xfer(cookie); + struct radeon_i2c_chan *i2c = cookie; + pre_xfer(&i2c->adapter); return (0); } void radeon_release_bus(void *cookie, int flags) { - post_xfer(cookie); + struct radeon_i2c_chan *i2c = cookie; + post_xfer(&i2c->adapter); } int @@ -320,6 +325,7 @@ radeon_write_byte(void *cookie, u_int8_t byte, int flags) return (i2c_bitbang_write_byte(cookie, byte, flags, &radeon_bbops)); } + /* hw i2c */ static u32 radeon_get_i2c_prescale(struct radeon_device *rdev) @@ -906,6 +912,8 @@ static int radeon_hw_i2c_xfer(struct i2c_adapter *i2c_adap, struct radeon_i2c_bus_rec *rec = &i2c->rec; int ret = 0; + mutex_lock(&i2c->mutex); + switch (rdev->family) { case CHIP_R100: case CHIP_RV100: @@ -972,6 +980,8 @@ static int radeon_hw_i2c_xfer(struct i2c_adapter *i2c_adap, break; } + mutex_unlock(&i2c->mutex); + return ret; } @@ -996,7 +1006,7 @@ struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev, { struct radeon_device *rdev = dev->dev_private; struct radeon_i2c_chan *i2c; - int ret = 0; + int ret; /* don't add the mm_i2c bus unless hw_i2c is enabled */ if (rec->mm_i2c && (radeon_hw_i2c == 0)) @@ -1007,13 +1017,14 @@ struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev, return NULL; i2c->rec = *rec; -#ifdef notyet +#ifdef __linux__ i2c->adapter.owner = THIS_MODULE; i2c->adapter.class = I2C_CLASS_DDC; i2c->adapter.dev.parent = &dev->pdev->dev; #endif i2c->dev = dev; i2c_set_adapdata(&i2c->adapter, i2c); + rw_init(&i2c->mutex, "riic"); if (rec->mm_i2c || (rec->hw_capable && radeon_hw_i2c && @@ -1044,28 +1055,28 @@ struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev, /* set the radeon bit adapter */ snprintf(i2c->adapter.name, sizeof(i2c->adapter.name), "Radeon i2c bit bus %s", name); + i2c->adapter.algo_data = &i2c->bit; #ifdef notyet - i2c->adapter.algo_data = &i2c->algo.bit; - i2c->algo.bit.pre_xfer = pre_xfer; - i2c->algo.bit.post_xfer = post_xfer; - i2c->algo.bit.setsda = set_data; - i2c->algo.bit.setscl = set_clock; - i2c->algo.bit.getsda = get_data; - i2c->algo.bit.getscl = get_clock; - i2c->algo.bit.udelay = 10; - i2c->algo.bit.timeout = usecs_to_jiffies(2200); /* from VESA */ - i2c->algo.bit.data = i2c; - ret = i2c_bit_add_bus(&i2c->adapter); + i2c->bit.pre_xfer = pre_xfer; + i2c->bit.post_xfer = post_xfer; + i2c->bit.setsda = set_data; + i2c->bit.setscl = set_clock; + i2c->bit.getsda = get_data; + i2c->bit.getscl = get_clock; + i2c->bit.udelay = 10; + i2c->bit.timeout = usecs_to_jiffies(2200); /* from VESA */ + i2c->bit.data = i2c; #else - i2c->adapter.ic.ic_cookie = i2c; - i2c->adapter.ic.ic_acquire_bus = radeon_acquire_bus; - i2c->adapter.ic.ic_release_bus = radeon_release_bus; - i2c->adapter.ic.ic_send_start = radeon_send_start; - i2c->adapter.ic.ic_send_stop = radeon_send_stop; - i2c->adapter.ic.ic_initiate_xfer = radeon_initiate_xfer; - i2c->adapter.ic.ic_read_byte = radeon_read_byte; - i2c->adapter.ic.ic_write_byte = radeon_write_byte; + i2c->bit.ic.ic_cookie = i2c; + i2c->bit.ic.ic_acquire_bus = radeon_acquire_bus; + i2c->bit.ic.ic_release_bus = radeon_release_bus; + i2c->bit.ic.ic_send_start = radeon_send_start; + i2c->bit.ic.ic_send_stop = radeon_send_stop; + i2c->bit.ic.ic_initiate_xfer = radeon_initiate_xfer; + i2c->bit.ic.ic_read_byte = radeon_read_byte; + i2c->bit.ic.ic_write_byte = radeon_write_byte; #endif + ret = i2c_bit_add_bus(&i2c->adapter); if (ret) { DRM_ERROR("Failed to register bit i2c %s\n", name); goto out_free; @@ -1076,6 +1087,7 @@ struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev, out_free: kfree(i2c); return NULL; + } void radeon_i2c_destroy(struct radeon_i2c_chan *i2c) @@ -1144,11 +1156,6 @@ struct radeon_i2c_chan *radeon_i2c_lookup(struct radeon_device *rdev, return NULL; } -struct drm_encoder *radeon_best_encoder(struct drm_connector *connector) -{ - return NULL; -} - void radeon_i2c_get_byte(struct radeon_i2c_chan *i2c_bus, u8 slave_addr, u8 addr, diff --git a/sys/dev/pci/drm/radeon/radeon_ib.c b/sys/dev/pci/drm/radeon/radeon_ib.c new file mode 100644 index 00000000000..af089965ac1 --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_ib.c @@ -0,0 +1,319 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + * Christian König + */ +#include +#include "radeon.h" + +/* + * IB + * IBs (Indirect Buffers) and areas of GPU accessible memory where + * commands are stored. You can put a pointer to the IB in the + * command ring and the hw will fetch the commands from the IB + * and execute them. Generally userspace acceleration drivers + * produce command buffers which are send to the kernel and + * put in IBs for execution by the requested ring. + */ +static int radeon_debugfs_sa_init(struct radeon_device *rdev); + +/** + * radeon_ib_get - request an IB (Indirect Buffer) + * + * @rdev: radeon_device pointer + * @ring: ring index the IB is associated with + * @ib: IB object returned + * @size: requested IB size + * + * Request an IB (all asics). IBs are allocated using the + * suballocator. + * Returns 0 on success, error on failure. + */ +int radeon_ib_get(struct radeon_device *rdev, int ring, + struct radeon_ib *ib, struct radeon_vm *vm, + unsigned size) +{ + int r; + + r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256); + if (r) { + dev_err(rdev->dev, "failed to get a new IB (%d)\n", r); + return r; + } + + radeon_sync_create(&ib->sync); + + ib->ring = ring; + ib->fence = NULL; + ib->ptr = radeon_sa_bo_cpu_addr(ib->sa_bo); + ib->vm = vm; + if (vm) { + /* ib pool is bound at RADEON_VA_IB_OFFSET in virtual address + * space and soffset is the offset inside the pool bo + */ + ib->gpu_addr = ib->sa_bo->soffset + RADEON_VA_IB_OFFSET; + } else { + ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo); + } + ib->is_const_ib = false; + + return 0; +} + +/** + * radeon_ib_free - free an IB (Indirect Buffer) + * + * @rdev: radeon_device pointer + * @ib: IB object to free + * + * Free an IB (all asics). + */ +void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib) +{ + radeon_sync_free(rdev, &ib->sync, ib->fence); + radeon_sa_bo_free(rdev, &ib->sa_bo, ib->fence); + radeon_fence_unref(&ib->fence); +} + +/** + * radeon_ib_schedule - schedule an IB (Indirect Buffer) on the ring + * + * @rdev: radeon_device pointer + * @ib: IB object to schedule + * @const_ib: Const IB to schedule (SI only) + * @hdp_flush: Whether or not to perform an HDP cache flush + * + * Schedule an IB on the associated ring (all asics). + * Returns 0 on success, error on failure. + * + * On SI, there are two parallel engines fed from the primary ring, + * the CE (Constant Engine) and the DE (Drawing Engine). Since + * resource descriptors have moved to memory, the CE allows you to + * prime the caches while the DE is updating register state so that + * the resource descriptors will be already in cache when the draw is + * processed. To accomplish this, the userspace driver submits two + * IBs, one for the CE and one for the DE. If there is a CE IB (called + * a CONST_IB), it will be put on the ring prior to the DE IB. Prior + * to SI there was just a DE IB. + */ +int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib, + struct radeon_ib *const_ib, bool hdp_flush) +{ + struct radeon_ring *ring = &rdev->ring[ib->ring]; + int r = 0; + + if (!ib->length_dw || !ring->ready) { + /* TODO: Nothings in the ib we should report. */ + dev_err(rdev->dev, "couldn't schedule ib\n"); + return -EINVAL; + } + + /* 64 dwords should be enough for fence too */ + r = radeon_ring_lock(rdev, ring, 64 + RADEON_NUM_SYNCS * 8); + if (r) { + dev_err(rdev->dev, "scheduling IB failed (%d).\n", r); + return r; + } + + /* grab a vm id if necessary */ + if (ib->vm) { + struct radeon_fence *vm_id_fence; + vm_id_fence = radeon_vm_grab_id(rdev, ib->vm, ib->ring); + radeon_sync_fence(&ib->sync, vm_id_fence); + } + + /* sync with other rings */ + r = radeon_sync_rings(rdev, &ib->sync, ib->ring); + if (r) { + dev_err(rdev->dev, "failed to sync rings (%d)\n", r); + radeon_ring_unlock_undo(rdev, ring); + return r; + } + + if (ib->vm) + radeon_vm_flush(rdev, ib->vm, ib->ring, + ib->sync.last_vm_update); + + if (const_ib) { + radeon_ring_ib_execute(rdev, const_ib->ring, const_ib); + radeon_sync_free(rdev, &const_ib->sync, NULL); + } + radeon_ring_ib_execute(rdev, ib->ring, ib); + r = radeon_fence_emit(rdev, &ib->fence, ib->ring); + if (r) { + dev_err(rdev->dev, "failed to emit fence for new IB (%d)\n", r); + radeon_ring_unlock_undo(rdev, ring); + return r; + } + if (const_ib) { + const_ib->fence = radeon_fence_ref(ib->fence); + } + + if (ib->vm) + radeon_vm_fence(rdev, ib->vm, ib->fence); + + radeon_ring_unlock_commit(rdev, ring, hdp_flush); + return 0; +} + +/** + * radeon_ib_pool_init - Init the IB (Indirect Buffer) pool + * + * @rdev: radeon_device pointer + * + * Initialize the suballocator to manage a pool of memory + * for use as IBs (all asics). + * Returns 0 on success, error on failure. + */ +int radeon_ib_pool_init(struct radeon_device *rdev) +{ + int r; + + if (rdev->ib_pool_ready) { + return 0; + } + + if (rdev->family >= CHIP_BONAIRE) { + r = radeon_sa_bo_manager_init(rdev, &rdev->ring_tmp_bo, + RADEON_IB_POOL_SIZE*64*1024, + RADEON_GPU_PAGE_SIZE, + RADEON_GEM_DOMAIN_GTT, + RADEON_GEM_GTT_WC); + } else { + /* Before CIK, it's better to stick to cacheable GTT due + * to the command stream checking + */ + r = radeon_sa_bo_manager_init(rdev, &rdev->ring_tmp_bo, + RADEON_IB_POOL_SIZE*64*1024, + RADEON_GPU_PAGE_SIZE, + RADEON_GEM_DOMAIN_GTT, 0); + } + if (r) { + return r; + } + + r = radeon_sa_bo_manager_start(rdev, &rdev->ring_tmp_bo); + if (r) { + return r; + } + + rdev->ib_pool_ready = true; + if (radeon_debugfs_sa_init(rdev)) { + dev_err(rdev->dev, "failed to register debugfs file for SA\n"); + } + return 0; +} + +/** + * radeon_ib_pool_fini - Free the IB (Indirect Buffer) pool + * + * @rdev: radeon_device pointer + * + * Tear down the suballocator managing the pool of memory + * for use as IBs (all asics). + */ +void radeon_ib_pool_fini(struct radeon_device *rdev) +{ + if (rdev->ib_pool_ready) { + radeon_sa_bo_manager_suspend(rdev, &rdev->ring_tmp_bo); + radeon_sa_bo_manager_fini(rdev, &rdev->ring_tmp_bo); + rdev->ib_pool_ready = false; + } +} + +/** + * radeon_ib_ring_tests - test IBs on the rings + * + * @rdev: radeon_device pointer + * + * Test an IB (Indirect Buffer) on each ring. + * If the test fails, disable the ring. + * Returns 0 on success, error if the primary GFX ring + * IB test fails. + */ +int radeon_ib_ring_tests(struct radeon_device *rdev) +{ + unsigned i; + int r; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + struct radeon_ring *ring = &rdev->ring[i]; + + if (!ring->ready) + continue; + + r = radeon_ib_test(rdev, i, ring); + if (r) { + radeon_fence_driver_force_completion(rdev, i); + ring->ready = false; + rdev->needs_reset = false; + + if (i == RADEON_RING_TYPE_GFX_INDEX) { + /* oh, oh, that's really bad */ + DRM_ERROR("radeon: failed testing IB on GFX ring (%d).\n", r); + rdev->accel_working = false; + return r; + + } else { + /* still not good, but we can live with it */ + DRM_ERROR("radeon: failed testing IB on ring %d (%d).\n", i, r); + } + } + } + return 0; +} + +/* + * Debugfs info + */ +#if defined(CONFIG_DEBUG_FS) + +static int radeon_debugfs_sa_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct radeon_device *rdev = dev->dev_private; + + radeon_sa_bo_dump_debug_info(&rdev->ring_tmp_bo, m); + + return 0; + +} + +static struct drm_info_list radeon_debugfs_sa_list[] = { + {"radeon_sa_info", &radeon_debugfs_sa_info, 0, NULL}, +}; + +#endif + +static int radeon_debugfs_sa_init(struct radeon_device *rdev) +{ +#if defined(CONFIG_DEBUG_FS) + return radeon_debugfs_add_files(rdev, radeon_debugfs_sa_list, 1); +#else + return 0; +#endif +} diff --git a/sys/dev/pci/drm/radeon/radeon_ioc32.c b/sys/dev/pci/drm/radeon/radeon_ioc32.c new file mode 100644 index 00000000000..5c92eff554e --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_ioc32.c @@ -0,0 +1,423 @@ +/** + * \file radeon_ioc32.c + * + * 32-bit ioctl compatibility routines for the Radeon DRM. + * + * \author Paul Mackerras + * + * Copyright (C) Paul Mackerras 2005 + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include +#include +#include "radeon_drv.h" + +typedef struct drm_radeon_init32 { + int func; + u32 sarea_priv_offset; + int is_pci; + int cp_mode; + int gart_size; + int ring_size; + int usec_timeout; + + unsigned int fb_bpp; + unsigned int front_offset, front_pitch; + unsigned int back_offset, back_pitch; + unsigned int depth_bpp; + unsigned int depth_offset, depth_pitch; + + u32 fb_offset; + u32 mmio_offset; + u32 ring_offset; + u32 ring_rptr_offset; + u32 buffers_offset; + u32 gart_textures_offset; +} drm_radeon_init32_t; + +static int compat_radeon_cp_init(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_init32_t init32; + drm_radeon_init_t __user *init; + + if (copy_from_user(&init32, (void __user *)arg, sizeof(init32))) + return -EFAULT; + + init = compat_alloc_user_space(sizeof(*init)); + if (!access_ok(VERIFY_WRITE, init, sizeof(*init)) + || __put_user(init32.func, &init->func) + || __put_user(init32.sarea_priv_offset, &init->sarea_priv_offset) + || __put_user(init32.is_pci, &init->is_pci) + || __put_user(init32.cp_mode, &init->cp_mode) + || __put_user(init32.gart_size, &init->gart_size) + || __put_user(init32.ring_size, &init->ring_size) + || __put_user(init32.usec_timeout, &init->usec_timeout) + || __put_user(init32.fb_bpp, &init->fb_bpp) + || __put_user(init32.front_offset, &init->front_offset) + || __put_user(init32.front_pitch, &init->front_pitch) + || __put_user(init32.back_offset, &init->back_offset) + || __put_user(init32.back_pitch, &init->back_pitch) + || __put_user(init32.depth_bpp, &init->depth_bpp) + || __put_user(init32.depth_offset, &init->depth_offset) + || __put_user(init32.depth_pitch, &init->depth_pitch) + || __put_user(init32.fb_offset, &init->fb_offset) + || __put_user(init32.mmio_offset, &init->mmio_offset) + || __put_user(init32.ring_offset, &init->ring_offset) + || __put_user(init32.ring_rptr_offset, &init->ring_rptr_offset) + || __put_user(init32.buffers_offset, &init->buffers_offset) + || __put_user(init32.gart_textures_offset, + &init->gart_textures_offset)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_CP_INIT, (unsigned long)init); +} + +typedef struct drm_radeon_clear32 { + unsigned int flags; + unsigned int clear_color; + unsigned int clear_depth; + unsigned int color_mask; + unsigned int depth_mask; /* misnamed field: should be stencil */ + u32 depth_boxes; +} drm_radeon_clear32_t; + +static int compat_radeon_cp_clear(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_clear32_t clr32; + drm_radeon_clear_t __user *clr; + + if (copy_from_user(&clr32, (void __user *)arg, sizeof(clr32))) + return -EFAULT; + + clr = compat_alloc_user_space(sizeof(*clr)); + if (!access_ok(VERIFY_WRITE, clr, sizeof(*clr)) + || __put_user(clr32.flags, &clr->flags) + || __put_user(clr32.clear_color, &clr->clear_color) + || __put_user(clr32.clear_depth, &clr->clear_depth) + || __put_user(clr32.color_mask, &clr->color_mask) + || __put_user(clr32.depth_mask, &clr->depth_mask) + || __put_user((void __user *)(unsigned long)clr32.depth_boxes, + &clr->depth_boxes)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_CLEAR, (unsigned long)clr); +} + +typedef struct drm_radeon_stipple32 { + u32 mask; +} drm_radeon_stipple32_t; + +static int compat_radeon_cp_stipple(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_stipple32_t __user *argp = (void __user *)arg; + drm_radeon_stipple_t __user *request; + u32 mask; + + if (get_user(mask, &argp->mask)) + return -EFAULT; + + request = compat_alloc_user_space(sizeof(*request)); + if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + || __put_user((unsigned int __user *)(unsigned long)mask, + &request->mask)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_STIPPLE, (unsigned long)request); +} + +typedef struct drm_radeon_tex_image32 { + unsigned int x, y; /* Blit coordinates */ + unsigned int width, height; + u32 data; +} drm_radeon_tex_image32_t; + +typedef struct drm_radeon_texture32 { + unsigned int offset; + int pitch; + int format; + int width; /* Texture image coordinates */ + int height; + u32 image; +} drm_radeon_texture32_t; + +static int compat_radeon_cp_texture(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_texture32_t req32; + drm_radeon_texture_t __user *request; + drm_radeon_tex_image32_t img32; + drm_radeon_tex_image_t __user *image; + + if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) + return -EFAULT; + if (req32.image == 0) + return -EINVAL; + if (copy_from_user(&img32, (void __user *)(unsigned long)req32.image, + sizeof(img32))) + return -EFAULT; + + request = compat_alloc_user_space(sizeof(*request) + sizeof(*image)); + if (!access_ok(VERIFY_WRITE, request, + sizeof(*request) + sizeof(*image))) + return -EFAULT; + image = (drm_radeon_tex_image_t __user *) (request + 1); + + if (__put_user(req32.offset, &request->offset) + || __put_user(req32.pitch, &request->pitch) + || __put_user(req32.format, &request->format) + || __put_user(req32.width, &request->width) + || __put_user(req32.height, &request->height) + || __put_user(image, &request->image) + || __put_user(img32.x, &image->x) + || __put_user(img32.y, &image->y) + || __put_user(img32.width, &image->width) + || __put_user(img32.height, &image->height) + || __put_user((const void __user *)(unsigned long)img32.data, + &image->data)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_TEXTURE, (unsigned long)request); +} + +typedef struct drm_radeon_vertex2_32 { + int idx; /* Index of vertex buffer */ + int discard; /* Client finished with buffer? */ + int nr_states; + u32 state; + int nr_prims; + u32 prim; +} drm_radeon_vertex2_32_t; + +static int compat_radeon_cp_vertex2(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_vertex2_32_t req32; + drm_radeon_vertex2_t __user *request; + + if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) + return -EFAULT; + + request = compat_alloc_user_space(sizeof(*request)); + if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + || __put_user(req32.idx, &request->idx) + || __put_user(req32.discard, &request->discard) + || __put_user(req32.nr_states, &request->nr_states) + || __put_user((void __user *)(unsigned long)req32.state, + &request->state) + || __put_user(req32.nr_prims, &request->nr_prims) + || __put_user((void __user *)(unsigned long)req32.prim, + &request->prim)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_VERTEX2, (unsigned long)request); +} + +typedef struct drm_radeon_cmd_buffer32 { + int bufsz; + u32 buf; + int nbox; + u32 boxes; +} drm_radeon_cmd_buffer32_t; + +static int compat_radeon_cp_cmdbuf(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_cmd_buffer32_t req32; + drm_radeon_cmd_buffer_t __user *request; + + if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) + return -EFAULT; + + request = compat_alloc_user_space(sizeof(*request)); + if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + || __put_user(req32.bufsz, &request->bufsz) + || __put_user((void __user *)(unsigned long)req32.buf, + &request->buf) + || __put_user(req32.nbox, &request->nbox) + || __put_user((void __user *)(unsigned long)req32.boxes, + &request->boxes)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_CMDBUF, (unsigned long)request); +} + +typedef struct drm_radeon_getparam32 { + int param; + u32 value; +} drm_radeon_getparam32_t; + +static int compat_radeon_cp_getparam(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_getparam32_t req32; + drm_radeon_getparam_t __user *request; + + if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) + return -EFAULT; + + request = compat_alloc_user_space(sizeof(*request)); + if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + || __put_user(req32.param, &request->param) + || __put_user((void __user *)(unsigned long)req32.value, + &request->value)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_GETPARAM, (unsigned long)request); +} + +typedef struct drm_radeon_mem_alloc32 { + int region; + int alignment; + int size; + u32 region_offset; /* offset from start of fb or GART */ +} drm_radeon_mem_alloc32_t; + +static int compat_radeon_mem_alloc(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_mem_alloc32_t req32; + drm_radeon_mem_alloc_t __user *request; + + if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) + return -EFAULT; + + request = compat_alloc_user_space(sizeof(*request)); + if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + || __put_user(req32.region, &request->region) + || __put_user(req32.alignment, &request->alignment) + || __put_user(req32.size, &request->size) + || __put_user((int __user *)(unsigned long)req32.region_offset, + &request->region_offset)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_ALLOC, (unsigned long)request); +} + +typedef struct drm_radeon_irq_emit32 { + u32 irq_seq; +} drm_radeon_irq_emit32_t; + +static int compat_radeon_irq_emit(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_irq_emit32_t req32; + drm_radeon_irq_emit_t __user *request; + + if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) + return -EFAULT; + + request = compat_alloc_user_space(sizeof(*request)); + if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + || __put_user((int __user *)(unsigned long)req32.irq_seq, + &request->irq_seq)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_IRQ_EMIT, (unsigned long)request); +} + +/* The two 64-bit arches where alignof(u64)==4 in 32-bit code */ +#if defined (CONFIG_X86_64) || defined(CONFIG_IA64) +typedef struct drm_radeon_setparam32 { + int param; + u64 value; +} __attribute__((packed)) drm_radeon_setparam32_t; + +static int compat_radeon_cp_setparam(struct file *file, unsigned int cmd, + unsigned long arg) +{ + drm_radeon_setparam32_t req32; + drm_radeon_setparam_t __user *request; + + if (copy_from_user(&req32, (void __user *) arg, sizeof(req32))) + return -EFAULT; + + request = compat_alloc_user_space(sizeof(*request)); + if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + || __put_user(req32.param, &request->param) + || __put_user((void __user *)(unsigned long)req32.value, + &request->value)) + return -EFAULT; + + return drm_ioctl(file, DRM_IOCTL_RADEON_SETPARAM, (unsigned long) request); +} +#else +#define compat_radeon_cp_setparam NULL +#endif /* X86_64 || IA64 */ + +static drm_ioctl_compat_t *radeon_compat_ioctls[] = { + [DRM_RADEON_CP_INIT] = compat_radeon_cp_init, + [DRM_RADEON_CLEAR] = compat_radeon_cp_clear, + [DRM_RADEON_STIPPLE] = compat_radeon_cp_stipple, + [DRM_RADEON_TEXTURE] = compat_radeon_cp_texture, + [DRM_RADEON_VERTEX2] = compat_radeon_cp_vertex2, + [DRM_RADEON_CMDBUF] = compat_radeon_cp_cmdbuf, + [DRM_RADEON_GETPARAM] = compat_radeon_cp_getparam, + [DRM_RADEON_SETPARAM] = compat_radeon_cp_setparam, + [DRM_RADEON_ALLOC] = compat_radeon_mem_alloc, + [DRM_RADEON_IRQ_EMIT] = compat_radeon_irq_emit, +}; + +/** + * Called whenever a 32-bit process running under a 64-bit kernel + * performs an ioctl on /dev/dri/card. + * + * \param filp file pointer. + * \param cmd command. + * \param arg user argument. + * \return zero on success or negative number on failure. + */ +long radeon_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + unsigned int nr = DRM_IOCTL_NR(cmd); + drm_ioctl_compat_t *fn = NULL; + int ret; + + if (nr < DRM_COMMAND_BASE) + return drm_compat_ioctl(filp, cmd, arg); + + if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(radeon_compat_ioctls)) + fn = radeon_compat_ioctls[nr - DRM_COMMAND_BASE]; + + if (fn != NULL) + ret = (*fn) (filp, cmd, arg); + else + ret = drm_ioctl(filp, cmd, arg); + + return ret; +} + +long radeon_kms_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + unsigned int nr = DRM_IOCTL_NR(cmd); + int ret; + + if (nr < DRM_COMMAND_BASE) + return drm_compat_ioctl(filp, cmd, arg); + + ret = radeon_drm_ioctl(filp, cmd, arg); + + return ret; +} diff --git a/sys/dev/pci/drm/radeon/radeon_irq_kms.c b/sys/dev/pci/drm/radeon/radeon_irq_kms.c index 81c45bbca4a..f75204ddb25 100644 --- a/sys/dev/pci/drm/radeon/radeon_irq_kms.c +++ b/sys/dev/pci/drm/radeon/radeon_irq_kms.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_irq_kms.c,v 1.12 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -33,12 +32,13 @@ #include "radeon.h" #include "atom.h" + #define RADEON_WAIT_IDLE_TIMEOUT 200 /** * radeon_driver_irq_handler_kms - irq handler for KMS * - * @DRM_IRQ_ARGS: args + * @int irq, void *arg: args * * This is the irq handler for the radeon KMS driver (all asics). * radeon_irq_process is a macro that points to the per-asic @@ -46,13 +46,17 @@ */ irqreturn_t radeon_driver_irq_handler_kms(void *arg) { - struct drm_device *dev = arg; + struct drm_device *dev = (struct drm_device *) arg; struct radeon_device *rdev = dev->dev_private; + irqreturn_t ret; if (!rdev->irq.installed) - return (0); + return 0; - return radeon_irq_process(rdev); + ret = radeon_irq_process(rdev); + if (ret == IRQ_HANDLED) + pm_runtime_mark_last_busy(dev->dev); + return ret; } /* @@ -69,21 +73,43 @@ irqreturn_t radeon_driver_irq_handler_kms(void *arg) * and calls the hotplug handler for each one, then sends * a drm hotplug event to alert userspace. */ -static void radeon_hotplug_work_func(void *arg1) +static void radeon_hotplug_work_func(struct work_struct *work) { - struct radeon_device *rdev = arg1; + struct radeon_device *rdev = container_of(work, struct radeon_device, + hotplug_work.work); struct drm_device *dev = rdev->ddev; struct drm_mode_config *mode_config = &dev->mode_config; struct drm_connector *connector; + /* we can race here at startup, some boards seem to trigger + * hotplug irqs when they shouldn't. */ + if (!rdev->mode_info.mode_config_initialized) + return; + + mutex_lock(&mode_config->mutex); if (mode_config->num_connector) { list_for_each_entry(connector, &mode_config->connector_list, head) radeon_connector_hotplug(connector); } + mutex_unlock(&mode_config->mutex); /* Just fire off a uevent and let userspace tell us what to do */ drm_helper_hpd_irq_event(dev); } +static void radeon_dp_work_func(struct work_struct *work) +{ + struct radeon_device *rdev = container_of(work, struct radeon_device, + dp_work); + struct drm_device *dev = rdev->ddev; + struct drm_mode_config *mode_config = &dev->mode_config; + struct drm_connector *connector; + + /* this should take a mutex */ + if (mode_config->num_connector) { + list_for_each_entry(connector, &mode_config->connector_list, head) + radeon_connector_hotplug(connector); + } +} /** * radeon_driver_irq_preinstall_kms - drm irq preinstall callback * @@ -102,6 +128,7 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev) /* Disable *all* interrupts */ for (i = 0; i < RADEON_NUM_RINGS; i++) atomic_set(&rdev->irq.ring_int[i], 0); + rdev->irq.dpm_thermal = false; for (i = 0; i < RADEON_MAX_HPD_PINS; i++) rdev->irq.hpd[i] = false; for (i = 0; i < RADEON_MAX_CRTCS; i++) { @@ -125,7 +152,13 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev) */ int radeon_driver_irq_postinstall_kms(struct drm_device *dev) { - dev->max_vblank_count = 0x001fffff; + struct radeon_device *rdev = dev->dev_private; + + if (ASIC_IS_AVIVO(rdev)) + dev->max_vblank_count = 0x00ffffff; + else + dev->max_vblank_count = 0x001fffff; + return 0; } @@ -149,6 +182,7 @@ void radeon_driver_irq_uninstall_kms(struct drm_device *dev) /* Disable *all* interrupts */ for (i = 0; i < RADEON_NUM_RINGS; i++) atomic_set(&rdev->irq.ring_int[i], 0); + rdev->irq.dpm_thermal = false; for (i = 0; i < RADEON_MAX_HPD_PINS; i++) rdev->irq.hpd[i] = false; for (i = 0; i < RADEON_MAX_CRTCS; i++) { @@ -180,6 +214,18 @@ bool radeon_msi_ok(struct radeon_device *rdev) if (rdev->flags & RADEON_IS_AGP) return false; + /* + * Older chips have a HW limitation, they can only generate 40 bits + * of address for "64-bit" MSIs which breaks on some platforms, notably + * IBM POWER servers, so we limit them + */ +#ifdef notyet + if (rdev->family < CHIP_BONAIRE) { + dev_info(rdev->dev, "radeon: MSI limited to 32-bit\n"); + rdev->pdev->no_64bit_msi = 1; + } +#endif + /* force MSI on */ if (radeon_msi == 1) return true; @@ -244,9 +290,6 @@ int radeon_irq_kms_init(struct radeon_device *rdev) { int r = 0; - task_set(&rdev->hotplug_task, radeon_hotplug_work_func, rdev); - task_set(&rdev->audio_task, r600_audio_update_hdmi, rdev); - mtx_init(&rdev->irq.lock, IPL_TTY); r = drm_vblank_init(rdev->ddev, rdev->num_crtc); if (r) { @@ -264,18 +307,25 @@ int radeon_irq_kms_init(struct radeon_device *rdev) } } #endif + + INIT_DELAYED_WORK(&rdev->hotplug_work, radeon_hotplug_work_func); + INIT_WORK(&rdev->dp_work, radeon_dp_work_func); + INIT_WORK(&rdev->audio_work, r600_audio_update_hdmi); + rdev->irq.installed = true; r = drm_irq_install(rdev->ddev, rdev->ddev->pdev->irq); if (r) { rdev->irq.installed = false; + flush_delayed_work(&rdev->hotplug_work); return r; } + DRM_INFO("radeon: irq initialized.\n"); return 0; } /** - * radeon_irq_kms_fini - tear down driver interrrupt info + * radeon_irq_kms_fini - tear down driver interrupt info * * @rdev: radeon device pointer * @@ -287,14 +337,10 @@ void radeon_irq_kms_fini(struct radeon_device *rdev) if (rdev->irq.installed) { drm_irq_uninstall(rdev->ddev); rdev->irq.installed = false; -#ifdef notyet if (rdev->msi_enabled) pci_disable_msi(rdev->pdev); -#endif + flush_delayed_work(&rdev->hotplug_work); } -#ifdef notyet - flush_work(&rdev->hotplug_work); -#endif } /** @@ -321,6 +367,21 @@ void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring) } } +/** + * radeon_irq_kms_sw_irq_get_delayed - enable software interrupt + * + * @rdev: radeon device pointer + * @ring: ring whose interrupt you want to enable + * + * Enables the software interrupt for a specific ring (all asics). + * The software interrupt is generally used to signal a fence on + * a particular ring. + */ +bool radeon_irq_kms_sw_irq_get_delayed(struct radeon_device *rdev, int ring) +{ + return atomic_inc_return(&rdev->irq.ring_int[ring]) == 1; +} + /** * radeon_irq_kms_sw_irq_put - disable software interrupt * diff --git a/sys/dev/pci/drm/radeon/radeon_kfd.c b/sys/dev/pci/drm/radeon/radeon_kfd.c new file mode 100644 index 00000000000..101f7b3302e --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_kfd.c @@ -0,0 +1,859 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include "radeon.h" +#include "cikd.h" +#include "cik_reg.h" +#include "radeon_kfd.h" +#include "radeon_ucode.h" +#include "cik_structs.h" + +#define CIK_PIPE_PER_MEC (4) + +static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = { + TCP_WATCH0_ADDR_H, TCP_WATCH0_ADDR_L, TCP_WATCH0_CNTL, + TCP_WATCH1_ADDR_H, TCP_WATCH1_ADDR_L, TCP_WATCH1_CNTL, + TCP_WATCH2_ADDR_H, TCP_WATCH2_ADDR_L, TCP_WATCH2_CNTL, + TCP_WATCH3_ADDR_H, TCP_WATCH3_ADDR_L, TCP_WATCH3_CNTL +}; + +struct kgd_mem { + struct radeon_bo *bo; + uint64_t gpu_addr; + void *cpu_ptr; +}; + + +static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size, + void **mem_obj, uint64_t *gpu_addr, + void **cpu_ptr); + +static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj); + +static uint64_t get_vmem_size(struct kgd_dev *kgd); +static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd); + +static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd); +static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type); + +/* + * Register access functions + */ + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases); + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid); + +static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t hpd_size, uint64_t hpd_gpu_addr); +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id); +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr); +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd); +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id); + +static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, + unsigned int timeout, uint32_t pipe_id, + uint32_t queue_id); +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd); +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int timeout); +static int kgd_address_watch_disable(struct kgd_dev *kgd); +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo); +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd); +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset); + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid); +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid); +static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid); + +static const struct kfd2kgd_calls kfd2kgd = { + .init_gtt_mem_allocation = alloc_gtt_mem, + .free_gtt_mem = free_gtt_mem, + .get_vmem_size = get_vmem_size, + .get_gpu_clock_counter = get_gpu_clock_counter, + .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz, + .program_sh_mem_settings = kgd_program_sh_mem_settings, + .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, + .init_pipeline = kgd_init_pipeline, + .init_interrupts = kgd_init_interrupts, + .hqd_load = kgd_hqd_load, + .hqd_sdma_load = kgd_hqd_sdma_load, + .hqd_is_occupied = kgd_hqd_is_occupied, + .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, + .hqd_destroy = kgd_hqd_destroy, + .hqd_sdma_destroy = kgd_hqd_sdma_destroy, + .address_watch_disable = kgd_address_watch_disable, + .address_watch_execute = kgd_address_watch_execute, + .wave_control_execute = kgd_wave_control_execute, + .address_watch_get_offset = kgd_address_watch_get_offset, + .get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid, + .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid, + .write_vmid_invalidate_request = write_vmid_invalidate_request, + .get_fw_version = get_fw_version +}; + +static const struct kgd2kfd_calls *kgd2kfd; + +bool radeon_kfd_init(void) +{ +#if defined(CONFIG_HSA_AMD_MODULE) + bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**); + + kgd2kfd_init_p = symbol_request(kgd2kfd_init); + + if (kgd2kfd_init_p == NULL) + return false; + + if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd)) { + symbol_put(kgd2kfd_init); + kgd2kfd = NULL; + + return false; + } + + return true; +#elif defined(CONFIG_HSA_AMD) + if (!kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd)) { + kgd2kfd = NULL; + + return false; + } + + return true; +#else + return false; +#endif +} + +void radeon_kfd_fini(void) +{ + if (kgd2kfd) { + kgd2kfd->exit(); + symbol_put(kgd2kfd_init); + } +} + +void radeon_kfd_device_probe(struct radeon_device *rdev) +{ + if (kgd2kfd) + rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev, + rdev->pdev, &kfd2kgd); +} + +void radeon_kfd_device_init(struct radeon_device *rdev) +{ + if (rdev->kfd) { + struct kgd2kfd_shared_resources gpu_resources = { + .compute_vmid_bitmap = 0xFF00, + + .first_compute_pipe = 1, + .compute_pipe_count = 4 - 1, + }; + + radeon_doorbell_get_kfd_info(rdev, + &gpu_resources.doorbell_physical_address, + &gpu_resources.doorbell_aperture_size, + &gpu_resources.doorbell_start_offset); + + kgd2kfd->device_init(rdev->kfd, &gpu_resources); + } +} + +void radeon_kfd_device_fini(struct radeon_device *rdev) +{ + if (rdev->kfd) { + kgd2kfd->device_exit(rdev->kfd); + rdev->kfd = NULL; + } +} + +void radeon_kfd_interrupt(struct radeon_device *rdev, const void *ih_ring_entry) +{ + if (rdev->kfd) + kgd2kfd->interrupt(rdev->kfd, ih_ring_entry); +} + +void radeon_kfd_suspend(struct radeon_device *rdev) +{ + if (rdev->kfd) + kgd2kfd->suspend(rdev->kfd); +} + +int radeon_kfd_resume(struct radeon_device *rdev) +{ + int r = 0; + + if (rdev->kfd) + r = kgd2kfd->resume(rdev->kfd); + + return r; +} + +static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size, + void **mem_obj, uint64_t *gpu_addr, + void **cpu_ptr) +{ + struct radeon_device *rdev = (struct radeon_device *)kgd; + struct kgd_mem **mem = (struct kgd_mem **) mem_obj; + int r; + + BUG_ON(kgd == NULL); + BUG_ON(gpu_addr == NULL); + BUG_ON(cpu_ptr == NULL); + + *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL); + if ((*mem) == NULL) + return -ENOMEM; + + r = radeon_bo_create(rdev, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_GTT, + RADEON_GEM_GTT_WC, NULL, NULL, &(*mem)->bo); + if (r) { + dev_err(rdev->dev, + "failed to allocate BO for amdkfd (%d)\n", r); + return r; + } + + /* map the buffer */ + r = radeon_bo_reserve((*mem)->bo, true); + if (r) { + dev_err(rdev->dev, "(%d) failed to reserve bo for amdkfd\n", r); + goto allocate_mem_reserve_bo_failed; + } + + r = radeon_bo_pin((*mem)->bo, RADEON_GEM_DOMAIN_GTT, + &(*mem)->gpu_addr); + if (r) { + dev_err(rdev->dev, "(%d) failed to pin bo for amdkfd\n", r); + goto allocate_mem_pin_bo_failed; + } + *gpu_addr = (*mem)->gpu_addr; + + r = radeon_bo_kmap((*mem)->bo, &(*mem)->cpu_ptr); + if (r) { + dev_err(rdev->dev, + "(%d) failed to map bo to kernel for amdkfd\n", r); + goto allocate_mem_kmap_bo_failed; + } + *cpu_ptr = (*mem)->cpu_ptr; + + radeon_bo_unreserve((*mem)->bo); + + return 0; + +allocate_mem_kmap_bo_failed: + radeon_bo_unpin((*mem)->bo); +allocate_mem_pin_bo_failed: + radeon_bo_unreserve((*mem)->bo); +allocate_mem_reserve_bo_failed: + radeon_bo_unref(&(*mem)->bo); + + return r; +} + +static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj) +{ + struct kgd_mem *mem = (struct kgd_mem *) mem_obj; + + BUG_ON(mem == NULL); + + radeon_bo_reserve(mem->bo, true); + radeon_bo_kunmap(mem->bo); + radeon_bo_unpin(mem->bo); + radeon_bo_unreserve(mem->bo); + radeon_bo_unref(&(mem->bo)); + kfree(mem); +} + +static uint64_t get_vmem_size(struct kgd_dev *kgd) +{ + struct radeon_device *rdev = (struct radeon_device *)kgd; + + BUG_ON(kgd == NULL); + + return rdev->mc.real_vram_size; +} + +static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd) +{ + struct radeon_device *rdev = (struct radeon_device *)kgd; + + return rdev->asic->get_gpu_clock_counter(rdev); +} + +static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd) +{ + struct radeon_device *rdev = (struct radeon_device *)kgd; + + /* The sclk is in quantas of 10kHz */ + return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100; +} + +static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd) +{ + return (struct radeon_device *)kgd; +} + +static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value) +{ + struct radeon_device *rdev = get_radeon_device(kgd); + + writel(value, (void __iomem *)(rdev->rmmio + offset)); +} + +static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset) +{ + struct radeon_device *rdev = get_radeon_device(kgd); + + return readl((void __iomem *)(rdev->rmmio + offset)); +} + +static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, + uint32_t queue, uint32_t vmid) +{ + struct radeon_device *rdev = get_radeon_device(kgd); + uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue); + + mutex_lock(&rdev->srbm_mutex); + write_register(kgd, SRBM_GFX_CNTL, value); +} + +static void unlock_srbm(struct kgd_dev *kgd) +{ + struct radeon_device *rdev = get_radeon_device(kgd); + + write_register(kgd, SRBM_GFX_CNTL, 0); + mutex_unlock(&rdev->srbm_mutex); +} + +static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t queue_id) +{ + uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1; + uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); + + lock_srbm(kgd, mec, pipe, queue_id, 0); +} + +static void release_queue(struct kgd_dev *kgd) +{ + unlock_srbm(kgd); +} + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases) +{ + lock_srbm(kgd, 0, 0, 0, vmid); + + write_register(kgd, SH_MEM_CONFIG, sh_mem_config); + write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base); + write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit); + write_register(kgd, SH_MEM_BASES, sh_mem_bases); + + unlock_srbm(kgd); +} + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid) +{ + /* + * We have to assume that there is no outstanding mapping. + * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 + * because a mapping is in progress or because a mapping finished and + * the SW cleared it. + * So the protocol is to always wait & clear. + */ + uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | + ATC_VMID_PASID_MAPPING_VALID_MASK; + + write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t), + pasid_mapping); + + while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) & + (1U << vmid))) + cpu_relax(); + write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid); + + /* Mapping vmid to pasid also for IH block */ + write_register(kgd, IH_VMID_0_LUT + vmid * sizeof(uint32_t), + pasid_mapping); + + return 0; +} + +static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t hpd_size, uint64_t hpd_gpu_addr) +{ + uint32_t mec = (pipe_id / CIK_PIPE_PER_MEC) + 1; + uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); + + lock_srbm(kgd, mec, pipe, 0, 0); + write_register(kgd, CP_HPD_EOP_BASE_ADDR, + lower_32_bits(hpd_gpu_addr >> 8)); + write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI, + upper_32_bits(hpd_gpu_addr >> 8)); + write_register(kgd, CP_HPD_EOP_VMID, 0); + write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size); + unlock_srbm(kgd); + + return 0; +} + +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) +{ + uint32_t mec; + uint32_t pipe; + + mec = (pipe_id / CIK_PIPE_PER_MEC) + 1; + pipe = (pipe_id % CIK_PIPE_PER_MEC); + + lock_srbm(kgd, mec, pipe, 0, 0); + + write_register(kgd, CPC_INT_CNTL, + TIME_STAMP_INT_ENABLE | OPCODE_ERROR_INT_ENABLE); + + unlock_srbm(kgd); + + return 0; +} + +static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m) +{ + uint32_t retval; + + retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET + + m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET; + + pr_debug("kfd: sdma base address: 0x%x\n", retval); + + return retval; +} + +static inline struct cik_mqd *get_mqd(void *mqd) +{ + return (struct cik_mqd *)mqd; +} + +static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) +{ + return (struct cik_sdma_rlc_registers *)mqd; +} + +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr) +{ + uint32_t wptr_shadow, is_wptr_shadow_valid; + struct cik_mqd *m; + + m = get_mqd(mqd); + + is_wptr_shadow_valid = !get_user(wptr_shadow, wptr); + + acquire_queue(kgd, pipe_id, queue_id); + write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo); + write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi); + write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control); + + write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo); + write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi); + write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control); + + write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control); + write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo); + write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi); + + write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr); + + write_register(kgd, CP_HQD_PERSISTENT_STATE, + m->cp_hqd_persistent_state); + write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd); + write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type); + + write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO, + m->cp_hqd_atomic0_preop_lo); + + write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI, + m->cp_hqd_atomic0_preop_hi); + + write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO, + m->cp_hqd_atomic1_preop_lo); + + write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI, + m->cp_hqd_atomic1_preop_hi); + + write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR, + m->cp_hqd_pq_rptr_report_addr_lo); + + write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI, + m->cp_hqd_pq_rptr_report_addr_hi); + + write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr); + + write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR, + m->cp_hqd_pq_wptr_poll_addr_lo); + + write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI, + m->cp_hqd_pq_wptr_poll_addr_hi); + + write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, + m->cp_hqd_pq_doorbell_control); + + write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid); + + write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum); + + write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority); + write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority); + + write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr); + + if (is_wptr_shadow_valid) + write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow); + + write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active); + release_queue(kgd); + + return 0; +} + +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd) +{ + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_VIRTUAL_ADDR, + m->sdma_rlc_virtual_addr); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_BASE, + m->sdma_rlc_rb_base); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_BASE_HI, + m->sdma_rlc_rb_base_hi); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_LO, + m->sdma_rlc_rb_rptr_addr_lo); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_HI, + m->sdma_rlc_rb_rptr_addr_hi); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_DOORBELL, + m->sdma_rlc_doorbell); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_CNTL, + m->sdma_rlc_rb_cntl); + + return 0; +} + +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id) +{ + uint32_t act; + bool retval = false; + uint32_t low, high; + + acquire_queue(kgd, pipe_id, queue_id); + act = read_register(kgd, CP_HQD_ACTIVE); + if (act) { + low = lower_32_bits(queue_address >> 8); + high = upper_32_bits(queue_address >> 8); + + if (low == read_register(kgd, CP_HQD_PQ_BASE) && + high == read_register(kgd, CP_HQD_PQ_BASE_HI)) + retval = true; + } + release_queue(kgd); + return retval; +} + +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) +{ + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t sdma_rlc_rb_cntl; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + sdma_rlc_rb_cntl = read_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_CNTL); + + if (sdma_rlc_rb_cntl & SDMA_RB_ENABLE) + return true; + + return false; +} + +static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, + unsigned int timeout, uint32_t pipe_id, + uint32_t queue_id) +{ + uint32_t temp; + + acquire_queue(kgd, pipe_id, queue_id); + write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0); + + write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type); + + while (true) { + temp = read_register(kgd, CP_HQD_ACTIVE); + if (temp & 0x1) + break; + if (timeout == 0) { + pr_err("kfd: cp queue preemption time out (%dms)\n", + temp); + release_queue(kgd); + return -ETIME; + } + msleep(20); + timeout -= 20; + } + + release_queue(kgd); + return 0; +} + +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int timeout) +{ + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t temp; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + temp = read_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL); + temp = temp & ~SDMA_RB_ENABLE; + write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL, temp); + + while (true) { + temp = read_register(kgd, sdma_base_addr + + SDMA0_RLC0_CONTEXT_STATUS); + if (temp & SDMA_RLC_IDLE) + break; + if (timeout == 0) + return -ETIME; + msleep(20); + timeout -= 20; + } + + write_register(kgd, sdma_base_addr + SDMA0_RLC0_DOORBELL, 0); + write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_RPTR, 0); + write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_WPTR, 0); + write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_BASE, 0); + + return 0; +} + +static int kgd_address_watch_disable(struct kgd_dev *kgd) +{ + union TCP_WATCH_CNTL_BITS cntl; + unsigned int i; + + cntl.u32All = 0; + + cntl.bitfields.valid = 0; + cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK; + cntl.bitfields.atc = 1; + + /* Turning off this address until we set all the registers */ + for (i = 0; i < MAX_WATCH_ADDRESSES; i++) + write_register(kgd, + watchRegs[i * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_CNTL], + cntl.u32All); + + return 0; +} + +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo) +{ + union TCP_WATCH_CNTL_BITS cntl; + + cntl.u32All = cntl_val; + + /* Turning off this watch point until we set all the registers */ + cntl.bitfields.valid = 0; + write_register(kgd, + watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_CNTL], + cntl.u32All); + + write_register(kgd, + watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_ADDR_HI], + addr_hi); + + write_register(kgd, + watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_ADDR_LO], + addr_lo); + + /* Enable the watch point */ + cntl.bitfields.valid = 1; + + write_register(kgd, + watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_CNTL], + cntl.u32All); + + return 0; +} + +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd) +{ + struct radeon_device *rdev = get_radeon_device(kgd); + uint32_t data; + + mutex_lock(&rdev->grbm_idx_mutex); + + write_register(kgd, GRBM_GFX_INDEX, gfx_index_val); + write_register(kgd, SQ_CMD, sq_cmd); + + /* Restore the GRBM_GFX_INDEX register */ + + data = INSTANCE_BROADCAST_WRITES | SH_BROADCAST_WRITES | + SE_BROADCAST_WRITES; + + write_register(kgd, GRBM_GFX_INDEX, data); + + mutex_unlock(&rdev->grbm_idx_mutex); + + return 0; +} + +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset) +{ + return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset]; +} + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid) +{ + uint32_t reg; + struct radeon_device *rdev = (struct radeon_device *) kgd; + + reg = RREG32(ATC_VMID0_PASID_MAPPING + vmid*4); + return reg & ATC_VMID_PASID_MAPPING_VALID_MASK; +} + +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct radeon_device *rdev = (struct radeon_device *) kgd; + + reg = RREG32(ATC_VMID0_PASID_MAPPING + vmid*4); + return reg & ATC_VMID_PASID_MAPPING_PASID_MASK; +} + +static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid) +{ + struct radeon_device *rdev = (struct radeon_device *) kgd; + + return WREG32(VM_INVALIDATE_REQUEST, 1 << vmid); +} + +static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type) +{ + struct radeon_device *rdev = (struct radeon_device *) kgd; + const union radeon_firmware_header *hdr; + + BUG_ON(kgd == NULL || rdev->mec_fw == NULL); + + switch (type) { + case KGD_ENGINE_PFP: + hdr = (const union radeon_firmware_header *) rdev->pfp_fw->data; + break; + + case KGD_ENGINE_ME: + hdr = (const union radeon_firmware_header *) rdev->me_fw->data; + break; + + case KGD_ENGINE_CE: + hdr = (const union radeon_firmware_header *) rdev->ce_fw->data; + break; + + case KGD_ENGINE_MEC1: + hdr = (const union radeon_firmware_header *) rdev->mec_fw->data; + break; + + case KGD_ENGINE_MEC2: + hdr = (const union radeon_firmware_header *) + rdev->mec2_fw->data; + break; + + case KGD_ENGINE_RLC: + hdr = (const union radeon_firmware_header *) rdev->rlc_fw->data; + break; + + case KGD_ENGINE_SDMA1: + case KGD_ENGINE_SDMA2: + hdr = (const union radeon_firmware_header *) + rdev->sdma_fw->data; + break; + + default: + return 0; + } + + if (hdr == NULL) + return 0; + + /* Only 12 bit in use*/ + return hdr->common.ucode_version; +} diff --git a/sys/dev/pci/drm/radeon/radeon_kfd.h b/sys/dev/pci/drm/radeon/radeon_kfd.h new file mode 100644 index 00000000000..c05e124fe22 --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_kfd.h @@ -0,0 +1,48 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * radeon_kfd.h defines the private interface between the + * AMD kernel graphics drivers and the AMD KFD. + */ + +#ifndef RADEON_KFD_H_INCLUDED +#define RADEON_KFD_H_INCLUDED + +#ifdef notyet +#include "kgd_kfd_interface.h" +#endif + +struct radeon_device; + +bool radeon_kfd_init(void); +void radeon_kfd_fini(void); + +void radeon_kfd_suspend(struct radeon_device *rdev); +int radeon_kfd_resume(struct radeon_device *rdev); +void radeon_kfd_interrupt(struct radeon_device *rdev, + const void *ih_ring_entry); +void radeon_kfd_device_probe(struct radeon_device *rdev); +void radeon_kfd_device_init(struct radeon_device *rdev); +void radeon_kfd_device_fini(struct radeon_device *rdev); + +#endif /* RADEON_KFD_H_INCLUDED */ diff --git a/sys/dev/pci/drm/radeon/radeon_kms.c b/sys/dev/pci/drm/radeon/radeon_kms.c index 23ae01d75fb..ce2ac47fb84 100644 --- a/sys/dev/pci/drm/radeon/radeon_kms.c +++ b/sys/dev/pci/drm/radeon/radeon_kms.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_kms.c,v 1.54 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -31,9 +30,14 @@ #include "radeon.h" #include #include "radeon_asic.h" -#include -/* can't include radeon_drv.h due to duplicated defines in radeon_reg.h */ +#include "radeon_kfd.h" + +#if defined(CONFIG_VGA_SWITCHEROO) +bool radeon_has_atpx(void); +#else +static inline bool radeon_has_atpx(void) { return false; } +#endif #include "vga.h" @@ -43,73 +47,13 @@ extern int vga_console_attached; #ifdef __amd64__ #include "efifb.h" +#include #endif #if NEFIFB > 0 #include #endif -#define DRIVER_NAME "radeon" -#define DRIVER_DESC "ATI Radeon" -#define DRIVER_DATE "20080613" - -#define KMS_DRIVER_MAJOR 2 -#define KMS_DRIVER_MINOR 29 -#define KMS_DRIVER_PATCHLEVEL 0 - -int radeon_driver_irq_handler_kms(void *); -void radeon_driver_irq_preinstall_kms(struct drm_device *); -int radeon_driver_irq_postinstall_kms(struct drm_device *); -void radeon_driver_irq_uninstall_kms(struct drm_device *d); - -void radeon_gem_object_free(struct drm_gem_object *); -int radeon_gem_object_open(struct drm_gem_object *, struct drm_file *); -void radeon_gem_object_close(struct drm_gem_object *, struct drm_file *); - -int radeon_driver_unload_kms(struct drm_device *); -int radeon_driver_load_kms(struct drm_device *, unsigned long); -int radeon_info_ioctl(struct drm_device *, void *, struct drm_file *); -int radeon_driver_firstopen_kms(struct drm_device *); -void radeon_driver_lastclose_kms(struct drm_device *); -int radeon_driver_open_kms(struct drm_device *, struct drm_file *); -void radeon_driver_postclose_kms(struct drm_device *, struct drm_file *); -void radeon_driver_preclose_kms(struct drm_device *, struct drm_file *); -u32 radeon_get_vblank_counter_kms(struct drm_device *, unsigned int); -int radeon_enable_vblank_kms(struct drm_device *, unsigned int); -void radeon_disable_vblank_kms(struct drm_device *, unsigned int); -int radeon_get_vblank_timestamp_kms(struct drm_device *, unsigned int, - int *, struct timeval *, unsigned); - -int radeon_dma_ioctl_kms(struct drm_device *, struct drm_dma *, struct drm_file *); - -int radeon_cp_init_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_start_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_stop_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_reset_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_idle_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_resume_kms(struct drm_device *, void *, struct drm_file *); -int radeon_engine_reset_kms(struct drm_device *, void *, struct drm_file *); -int radeon_fullscreen_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_swap_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_clear_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_vertex_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_indices_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_texture_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_stipple_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_indirect_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_vertex2_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_cmdbuf_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_getparam_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_flip_kms(struct drm_device *, void *, struct drm_file *); -int radeon_mem_alloc_kms(struct drm_device *, void *, struct drm_file *); -int radeon_mem_free_kms(struct drm_device *, void *, struct drm_file *); -int radeon_mem_init_heap_kms(struct drm_device *, void *, struct drm_file *); -int radeon_irq_emit_kms(struct drm_device *, void *, struct drm_file *); -int radeon_irq_wait_kms(struct drm_device *, void *, struct drm_file *); -int radeon_cp_setparam_kms(struct drm_device *, void *, struct drm_file *); -int radeon_surface_alloc_kms(struct drm_device *, void *, struct drm_file *); -int radeon_surface_free_kms(struct drm_device *, void *, struct drm_file *); - int radeondrm_probe(struct device *, void *, void *); void radeondrm_attach_kms(struct device *, struct device *, void *); int radeondrm_detach_kms(struct device *, int); @@ -117,152 +61,27 @@ int radeondrm_activate_kms(struct device *, int); void radeondrm_attachhook(struct device *); int radeondrm_forcedetach(struct radeon_device *); -extern struct drm_ioctl_desc radeon_ioctls_kms[]; -extern int radeon_max_kms_ioctl; - -struct cfattach radeondrm_ca = { - sizeof (struct radeon_device), radeondrm_probe, radeondrm_attach_kms, - radeondrm_detach_kms, radeondrm_activate_kms -}; - -struct cfdriver radeondrm_cd = { - NULL, "radeondrm", DV_DULL -}; - -int radeon_no_wb; -int radeon_modeset = 1; -int radeon_dynclks = -1; -int radeon_r4xx_atom = 0; -int radeon_agpmode = 0; -int radeon_vram_limit = 0; -int radeon_gart_size = 512; /* default gart size */ -int radeon_benchmarking = 0; -int radeon_testing = 0; -int radeon_connector_table = 0; -int radeon_tv = 1; -int radeon_audio = 0; -int radeon_disp_priority = 0; -int radeon_hw_i2c = 0; -int radeon_pcie_gen2 = -1; -int radeon_msi = -1; -int radeon_lockup_timeout = 10000; -int radeon_auxch = -1; - -MODULE_PARM_DESC(no_wb, "Disable AGP writeback for scratch registers"); -module_param_named(no_wb, radeon_no_wb, int, 0444); +bool radeon_msi_ok(struct radeon_device *); +irqreturn_t radeon_driver_irq_handler_kms(void *); -MODULE_PARM_DESC(modeset, "Disable/Enable modesetting"); -module_param_named(modeset, radeon_modeset, int, 0400); - -MODULE_PARM_DESC(dynclks, "Disable/Enable dynamic clocks"); -module_param_named(dynclks, radeon_dynclks, int, 0444); - -MODULE_PARM_DESC(r4xx_atom, "Enable ATOMBIOS modesetting for R4xx"); -module_param_named(r4xx_atom, radeon_r4xx_atom, int, 0444); - -MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing"); -module_param_named(vramlimit, radeon_vram_limit, int, 0600); - -MODULE_PARM_DESC(agpmode, "AGP Mode (-1 == PCI)"); -module_param_named(agpmode, radeon_agpmode, int, 0444); - -MODULE_PARM_DESC(gartsize, "Size of PCIE/IGP gart to setup in megabytes (32, 64, etc)"); -module_param_named(gartsize, radeon_gart_size, int, 0600); - -MODULE_PARM_DESC(benchmark, "Run benchmark"); -module_param_named(benchmark, radeon_benchmarking, int, 0444); - -MODULE_PARM_DESC(test, "Run tests"); -module_param_named(test, radeon_testing, int, 0444); - -MODULE_PARM_DESC(connector_table, "Force connector table"); -module_param_named(connector_table, radeon_connector_table, int, 0444); - -MODULE_PARM_DESC(tv, "TV enable (0 = disable)"); -module_param_named(tv, radeon_tv, int, 0444); - -MODULE_PARM_DESC(audio, "Audio enable (1 = enable)"); -module_param_named(audio, radeon_audio, int, 0444); - -MODULE_PARM_DESC(disp_priority, "Display Priority (0 = auto, 1 = normal, 2 = high)"); -module_param_named(disp_priority, radeon_disp_priority, int, 0444); - -MODULE_PARM_DESC(hw_i2c, "hw i2c engine enable (0 = disable)"); -module_param_named(hw_i2c, radeon_hw_i2c, int, 0444); - -MODULE_PARM_DESC(pcie_gen2, "PCIE Gen2 mode (-1 = auto, 0 = disable, 1 = enable)"); -module_param_named(pcie_gen2, radeon_pcie_gen2, int, 0444); - -MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)"); -module_param_named(msi, radeon_msi, int, 0444); - -MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (defaul 10000 = 10 seconds, 0 = disable)"); -module_param_named(lockup_timeout, radeon_lockup_timeout, int, 0444); +extern const struct drm_pcidev radeondrm_pciidlist[]; +extern struct drm_driver kms_driver; +const struct drm_ioctl_desc radeon_ioctls_kms[]; +extern int radeon_max_kms_ioctl; /* * set if the mountroot hook has a fatal error * such as not being able to find the firmware on newer cards */ -int radeon_fatal_error = 0; +int radeon_fatal_error; -const struct drm_pcidev radeondrm_pciidlist[] = { - radeon_PCI_IDS +struct cfattach radeondrm_ca = { + sizeof (struct radeon_device), radeondrm_probe, radeondrm_attach_kms, + radeondrm_detach_kms, radeondrm_activate_kms }; -static struct drm_driver kms_driver = { - .driver_features = - DRIVER_USE_AGP | - DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | - DRIVER_MODESET, - .buf_priv_size = 0, - .firstopen = radeon_driver_firstopen_kms, - .open = radeon_driver_open_kms, - .mmap = radeon_mmap, -#ifdef notyet - .preclose = radeon_driver_preclose_kms, - .postclose = radeon_driver_postclose_kms, -#endif - .lastclose = radeon_driver_lastclose_kms, -#ifdef notyet - .suspend = radeon_suspend_kms, - .resume = radeon_resume_kms, -#endif - .get_vblank_counter = radeon_get_vblank_counter_kms, - .enable_vblank = radeon_enable_vblank_kms, - .disable_vblank = radeon_disable_vblank_kms, - .get_vblank_timestamp = radeon_get_vblank_timestamp_kms, - .get_scanout_position = radeon_get_crtc_scanoutpos, -#if defined(CONFIG_DEBUG_FS) - .debugfs_init = radeon_debugfs_init, - .debugfs_cleanup = radeon_debugfs_cleanup, -#endif - .irq_preinstall = radeon_driver_irq_preinstall_kms, - .irq_postinstall = radeon_driver_irq_postinstall_kms, - .irq_uninstall = radeon_driver_irq_uninstall_kms, - .ioctls = radeon_ioctls_kms, - .gem_free_object = radeon_gem_object_free, - .gem_open_object = radeon_gem_object_open, - .gem_close_object = radeon_gem_object_close, - .gem_size = sizeof(struct radeon_bo), - .dma_ioctl = radeon_dma_ioctl_kms, - .dumb_create = radeon_mode_dumb_create, - .dumb_map_offset = radeon_mode_dumb_mmap, - .dumb_destroy = radeon_mode_dumb_destroy, -#ifdef notyet - .fops = &radeon_driver_kms_fops, - - .prime_handle_to_fd = drm_gem_prime_handle_to_fd, - .prime_fd_to_handle = drm_gem_prime_fd_to_handle, - .gem_prime_export = radeon_gem_prime_export, - .gem_prime_import = radeon_gem_prime_import, -#endif - - .name = DRIVER_NAME, - .desc = DRIVER_DESC, - .date = DRIVER_DATE, - .major = KMS_DRIVER_MAJOR, - .minor = KMS_DRIVER_MINOR, - .patchlevel = KMS_DRIVER_PATCHLEVEL, +struct cfdriver radeondrm_cd = { + NULL, "radeondrm", DV_DULL }; int @@ -286,6 +105,32 @@ radeondrm_probe(struct device *parent, void *match, void *aux) * the rest of the device (CP, writeback, etc.). * Returns 0 on success. */ +#ifdef __linux__ +int radeon_driver_unload_kms(struct drm_device *dev) +{ + struct radeon_device *rdev = dev->dev_private; + + if (rdev == NULL) + return 0; + + if (rdev->rmmio == NULL) + goto done_free; + + pm_runtime_get_sync(dev->dev); + + radeon_kfd_device_fini(rdev); + + radeon_acpi_fini(rdev); + + radeon_modeset_fini(rdev); + radeon_device_fini(rdev); + +done_free: + kfree(rdev); + dev->dev_private = NULL; + return 0; +} +#else int radeondrm_detach_kms(struct device *self, int flags) { @@ -294,7 +139,16 @@ radeondrm_detach_kms(struct device *self, int flags) if (rdev == NULL) return 0; + pci_intr_disestablish(rdev->pc, rdev->irqh); + +#ifdef notyet + pm_runtime_get_sync(dev->dev); + + radeon_kfd_device_fini(rdev); +#endif + radeon_acpi_fini(rdev); + radeon_modeset_fini(rdev); radeon_device_fini(rdev); @@ -303,14 +157,11 @@ radeondrm_detach_kms(struct device *self, int flags) rdev->ddev = NULL; } - pci_intr_disestablish(rdev->pc, rdev->irqh); - - if (rdev->rmmio_size > 0) - bus_space_unmap(rdev->memt, rdev->rmmio, rdev->rmmio_size); - return 0; } +#endif +void radeondrm_burner(void *, u_int, u_int); int radeondrm_wsioctl(void *, u_long, caddr_t, int, struct proc *); paddr_t radeondrm_wsmmap(void *, off_t, int); int radeondrm_alloc_screen(void *, const struct wsscreen_descr *, @@ -480,6 +331,7 @@ radeondrm_setcolor(void *v, u_int index, u_int8_t r, u_int8_t g, u_int8_t b) } #endif +#ifdef __linux__ /** * radeon_driver_load_kms - Main load function for KMS. * @@ -493,6 +345,84 @@ radeondrm_setcolor(void *v, u_int index, u_int8_t r, u_int8_t g, u_int8_t b) * (crtcs, encoders, hotplug detect, etc.). * Returns 0 on success, error on failure. */ +int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags) +{ + struct radeon_device *rdev; + int r, acpi_status; + + rdev = kzalloc(sizeof(struct radeon_device), GFP_KERNEL); + if (rdev == NULL) { + return -ENOMEM; + } + dev->dev_private = (void *)rdev; + + /* update BUS flag */ + if (drm_pci_device_is_agp(dev)) { + flags |= RADEON_IS_AGP; + } else if (pci_is_pcie(dev->pdev)) { + flags |= RADEON_IS_PCIE; + } else { + flags |= RADEON_IS_PCI; + } + + if ((radeon_runtime_pm != 0) && + radeon_has_atpx() && + ((flags & RADEON_IS_IGP) == 0)) + flags |= RADEON_IS_PX; + + /* radeon_device_init should report only fatal error + * like memory allocation failure or iomapping failure, + * or memory manager initialization failure, it must + * properly initialize the GPU MC controller and permit + * VRAM allocation + */ + r = radeon_device_init(rdev, dev, dev->pdev, flags); + if (r) { + dev_err(&dev->pdev->dev, "Fatal error during GPU init\n"); + goto out; + } + + /* Again modeset_init should fail only on fatal error + * otherwise it should provide enough functionalities + * for shadowfb to run + */ + r = radeon_modeset_init(rdev); + if (r) + dev_err(&dev->pdev->dev, "Fatal error during modeset init\n"); + + /* Call ACPI methods: require modeset init + * but failure is not fatal + */ + if (!r) { + acpi_status = radeon_acpi_init(rdev); + if (acpi_status) + dev_dbg(&dev->pdev->dev, + "Error during ACPI methods call\n"); + } + +#ifdef notyet + radeon_kfd_device_probe(rdev); + radeon_kfd_device_init(rdev); +#endif + + if (radeon_is_px(dev)) { + pm_runtime_use_autosuspend(dev->dev); + pm_runtime_set_autosuspend_delay(dev->dev, 5000); + pm_runtime_set_active(dev->dev); + pm_runtime_allow(dev->dev); + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + } + +out: + if (r) + radeon_driver_unload_kms(dev); + + + return r; +} +#endif + void radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) { @@ -502,7 +432,8 @@ radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) const struct drm_pcidev *id_entry; int is_agp; pcireg_t type; - uint8_t iobar; + int i; + uint8_t rmmio_bar; #if !defined(__sparc64__) pcireg_t addr, mask; int s; @@ -515,6 +446,7 @@ radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) id_entry = drm_find_description(PCI_VENDOR(pa->pa_id), PCI_PRODUCT(pa->pa_id), radeondrm_pciidlist); rdev->flags = id_entry->driver_data; + rdev->family = rdev->flags & RADEON_FAMILY_MASK; rdev->pc = pa->pa_pc; rdev->pa_tag = pa->pa_tag; rdev->iot = pa->pa_iot; @@ -544,9 +476,6 @@ radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) #endif #define RADEON_PCI_MEM 0x10 -#define RADEON_PCI_IO 0x14 -#define RADEON_PCI_MMIO 0x18 -#define RADEON_PCI_IO2 0x20 type = pci_mapreg_type(pa->pa_pc, pa->pa_tag, RADEON_PCI_MEM); if (PCI_MAPREG_TYPE(type) != PCI_MAPREG_TYPE_MEM || @@ -556,22 +485,41 @@ radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) return; } - if (PCI_MAPREG_MEM_TYPE(type) != PCI_MAPREG_MEM_TYPE_64BIT) - iobar = RADEON_PCI_IO; - else - iobar = RADEON_PCI_IO2; - - if (pci_mapreg_map(pa, iobar, PCI_MAPREG_TYPE_IO, 0, - NULL, &rdev->rio_mem, NULL, &rdev->rio_mem_size, 0)) { - printf(": can't map IO space\n"); - return; + for (i = PCI_MAPREG_START; i < PCI_MAPREG_END ; i+= 4) { + type = pci_mapreg_type(pa->pa_pc, pa->pa_tag, i); + if (PCI_MAPREG_TYPE(type) != PCI_MAPREG_TYPE_IO) + continue; + if (pci_mapreg_map(pa, i, type, 0, NULL, + &rdev->rio_mem, NULL, &rdev->rio_mem_size, 0)) { + printf(": can't map rio space\n"); + return; + } + + if (type & PCI_MAPREG_MEM_TYPE_64BIT) + i += 4; + } + + if (rdev->family >= CHIP_BONAIRE) { + type = pci_mapreg_type(pa->pa_pc, pa->pa_tag, 0x18); + if (PCI_MAPREG_TYPE(type) != PCI_MAPREG_TYPE_MEM || + pci_mapreg_map(pa, 0x18, type, 0, NULL, + &rdev->doorbell.bsh, &rdev->doorbell.base, + &rdev->doorbell.size, 0)) { + printf(": can't map doorbell space\n"); + return; + } } - type = pci_mapreg_type(pa->pa_pc, pa->pa_tag, RADEON_PCI_MMIO); + if (rdev->family >= CHIP_BONAIRE) + rmmio_bar = 0x24; + else + rmmio_bar = 0x18; + + type = pci_mapreg_type(pa->pa_pc, pa->pa_tag, rmmio_bar); if (PCI_MAPREG_TYPE(type) != PCI_MAPREG_TYPE_MEM || - pci_mapreg_map(pa, RADEON_PCI_MMIO, type, 0, NULL, - &rdev->rmmio, &rdev->rmmio_base, &rdev->rmmio_size, 0)) { - printf(": can't map mmio space\n"); + pci_mapreg_map(pa, rmmio_bar, type, 0, NULL, + &rdev->rmmio_bsh, &rdev->rmmio_base, &rdev->rmmio_size, 0)) { + printf(": can't map rmmio space\n"); return; } @@ -614,6 +562,11 @@ radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) rdev->flags |= RADEON_IS_PCI; } + if ((radeon_runtime_pm != 0) && + radeon_has_atpx() && + ((rdev->flags & RADEON_IS_IGP) == 0)) + rdev->flags |= RADEON_IS_PX; + DRM_DEBUG("%s card detected\n", ((rdev->flags & RADEON_IS_AGP) ? "AGP" : (((rdev->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI")))); @@ -624,13 +577,13 @@ radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) printf("\n"); kms_driver.num_ioctls = radeon_max_kms_ioctl; + kms_driver.driver_features |= DRIVER_MODESET; dev = (struct drm_device *)drm_attach_pci(&kms_driver, pa, is_agp, rdev->console, self); rdev->ddev = dev; rdev->pdev = dev->pdev; - rdev->family = rdev->flags & RADEON_FAMILY_MASK; if (!radeon_msi_ok(rdev)) pa->pa_flags &= ~PCI_FLAGS_MSI_ENABLED; @@ -641,14 +594,14 @@ radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) printf(": couldn't map interrupt\n"); return; } - printf("%s: %s\n", rdev->dev.dv_xname, + printf("%s: %s\n", rdev->self.dv_xname, pci_intr_string(pa->pa_pc, rdev->intrh)); rdev->irqh = pci_intr_establish(pa->pa_pc, rdev->intrh, IPL_TTY, - radeon_driver_irq_handler_kms, rdev->ddev, rdev->dev.dv_xname); + radeon_driver_irq_handler_kms, rdev->ddev, rdev->self.dv_xname); if (rdev->irqh == NULL) { printf("%s: couldn't establish interrupt\n", - rdev->dev.dv_xname); + rdev->self.dv_xname); return; } rdev->pdev->irq = -1; @@ -668,10 +621,10 @@ radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) * an offset from the start of video memory. */ rdev->fb_offset = - bus_space_read_4(rdev->memt, rdev->rmmio, RADEON_CRTC_OFFSET); + bus_space_read_4(rdev->memt, rdev->rmmio_bsh, RADEON_CRTC_OFFSET); if (bus_space_map(rdev->memt, rdev->fb_aper_offset + rdev->fb_offset, rdev->sf.sf_fbsize, BUS_SPACE_MAP_LINEAR, &rdev->memh)) { - printf("%s: can't map video memory\n", rdev->dev.dv_xname); + printf("%s: can't map video memory\n", rdev->self.dv_xname); return; } @@ -690,10 +643,12 @@ radeondrm_attach_kms(struct device *parent, struct device *self, void *aux) config_mountroot(self, radeondrm_attachhook); } +extern void mainbus_efifb_reattach(void); + int radeondrm_forcedetach(struct radeon_device *rdev) { - struct pci_softc *sc = (struct pci_softc *)rdev->dev.dv_parent; + struct pci_softc *sc = (struct pci_softc *)rdev->self.dv_parent; pcitag_t tag = rdev->pa_tag; #if NVGA > 0 @@ -701,8 +656,19 @@ radeondrm_forcedetach(struct radeon_device *rdev) vga_console_attached = 0; #endif - config_detach(&rdev->dev, 0); - return pci_probe_device(sc, tag, NULL, NULL); + /* reprobe pci device for non efi systems */ +#if NEFIFB > 0 + if (bios_efiinfo == NULL && !efifb_cb_found()) { +#endif + config_detach(&rdev->self, 0); + return pci_probe_device(sc, tag, NULL, NULL); +#if NEFIFB > 0 + } else if (rdev->console) { + mainbus_efifb_reattach(); + } +#endif + + return 0; } void @@ -717,7 +683,7 @@ radeondrm_attachhook(struct device *self) * properly initialize the GPU MC controller and permit * VRAM allocation */ - r = radeon_device_init(rdev, rdev->ddev); + r = radeon_device_init(rdev, rdev->ddev, rdev->ddev->pdev, rdev->flags); if (r) { dev_err(&dev->pdev->dev, "Fatal error during GPU init\n"); radeon_fatal_error = 1; @@ -742,6 +708,20 @@ radeondrm_attachhook(struct device *self) DRM_DEBUG("Error during ACPI methods call\n"); } +#ifdef notyet + radeon_kfd_device_probe(rdev); + radeon_kfd_device_init(rdev); +#endif + + if (radeon_is_px(rdev->ddev)) { + pm_runtime_use_autosuspend(dev->dev); + pm_runtime_set_autosuspend_delay(dev->dev, 5000); + pm_runtime_set_active(dev->dev); + pm_runtime_allow(dev->dev); + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + } + { struct drm_fb_helper *fb_helper = (void *)rdev->mode_info.rfbdev; struct wsemuldisplaydev_attach_args aa; @@ -792,12 +772,12 @@ radeondrm_attachhook(struct device *self) * VGA legacy addresses, and opt out of arbitration. */ radeon_vga_set_state(rdev, false); - pci_disable_legacy_vga(&rdev->dev); + pci_disable_legacy_vga(&rdev->self); - printf("%s: %dx%d, %dbpp\n", rdev->dev.dv_xname, + printf("%s: %dx%d, %dbpp\n", rdev->self.dv_xname, ri->ri_width, ri->ri_height, ri->ri_depth); - config_found_sm(&rdev->dev, &aa, wsemuldisplaydevprint, + config_found_sm(&rdev->self, &aa, wsemuldisplaydevprint, wsemuldisplaydevsubmatch); } } @@ -814,14 +794,14 @@ radeondrm_activate_kms(struct device *self, int act) switch (act) { case DVACT_QUIESCE: rv = config_activate_children(self, act); - radeon_suspend_kms(rdev->ddev); + radeon_suspend_kms(rdev->ddev, true, true); break; case DVACT_SUSPEND: break; case DVACT_RESUME: break; case DVACT_WAKEUP: - radeon_resume_kms(rdev->ddev); + radeon_resume_kms(rdev->ddev, true, true); rv = config_activate_children(self, act); break; } @@ -829,6 +809,7 @@ radeondrm_activate_kms(struct device *self, int act) return (rv); } + /** * radeon_set_filp_rights - Set filp right. * @@ -844,7 +825,9 @@ static void radeon_set_filp_rights(struct drm_device *dev, struct drm_file *applier, uint32_t *value) { - mutex_lock(&dev->struct_mutex); + struct radeon_device *rdev = dev->dev_private; + + mutex_lock(&rdev->gem.mutex); if (*value == 1) { /* wants rights */ if (!*owner) @@ -855,7 +838,7 @@ static void radeon_set_filp_rights(struct drm_device *dev, *owner = NULL; } *value = *owner == applier ? 1 : 0; - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&rdev->gem.mutex); } /* @@ -873,89 +856,83 @@ static void radeon_set_filp_rights(struct drm_device *dev, * etc. (all asics). * Returns 0 on success, -EINVAL on failure. */ -int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +static int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { struct radeon_device *rdev = dev->dev_private; struct drm_radeon_info *info = data; struct radeon_mode_info *minfo = &rdev->mode_info; - uint32_t value, *value_ptr; - uint64_t value64, *value_ptr64; + uint32_t *value, value_tmp, *value_ptr, value_size; + uint64_t value64; struct drm_crtc *crtc; int i, found; - /* TIMESTAMP is a 64-bit value, needs special handling. */ - if (info->request == RADEON_INFO_TIMESTAMP) { - if (rdev->family >= CHIP_R600) { - value_ptr64 = (uint64_t*)((unsigned long)info->value); - if (rdev->family >= CHIP_TAHITI) { - value64 = si_get_gpu_clock(rdev); - } else { - value64 = r600_get_gpu_clock(rdev); - } - - if (DRM_COPY_TO_USER(value_ptr64, &value64, sizeof(value64))) { - DRM_ERROR("copy_to_user %s:%u\n", __func__, __LINE__); - return -EFAULT; - } - return 0; - } else { - DRM_DEBUG_KMS("timestamp is r6xx+ only!\n"); - return -EINVAL; - } - } - value_ptr = (uint32_t *)((unsigned long)info->value); - if (DRM_COPY_FROM_USER(&value, value_ptr, sizeof(value))) { - DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__); - return -EFAULT; - } + value = &value_tmp; + value_size = sizeof(uint32_t); switch (info->request) { case RADEON_INFO_DEVICE_ID: - value = dev->pci_device; + *value = dev->pdev->device; break; case RADEON_INFO_NUM_GB_PIPES: - value = rdev->num_gb_pipes; + *value = rdev->num_gb_pipes; break; case RADEON_INFO_NUM_Z_PIPES: - value = rdev->num_z_pipes; + *value = rdev->num_z_pipes; break; case RADEON_INFO_ACCEL_WORKING: /* xf86-video-ati 6.13.0 relies on this being false for evergreen */ if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) - value = false; + *value = false; else - value = rdev->accel_working; + *value = rdev->accel_working; break; case RADEON_INFO_CRTC_FROM_ID: + if (copy_from_user(value, value_ptr, sizeof(uint32_t))) { + DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__); + return -EFAULT; + } for (i = 0, found = 0; i < rdev->num_crtc; i++) { crtc = (struct drm_crtc *)minfo->crtcs[i]; - if (crtc && crtc->base.id == value) { + if (crtc && crtc->base.id == *value) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); - value = radeon_crtc->crtc_id; + *value = radeon_crtc->crtc_id; found = 1; break; } } if (!found) { - DRM_DEBUG_KMS("unknown crtc id %d\n", value); + DRM_DEBUG_KMS("unknown crtc id %d\n", *value); return -EINVAL; } break; case RADEON_INFO_ACCEL_WORKING2: - value = rdev->accel_working; + if (rdev->family == CHIP_HAWAII) { + if (rdev->accel_working) { + if (rdev->new_fw) + *value = 3; + else + *value = 2; + } else { + *value = 0; + } + } else { + *value = rdev->accel_working; + } break; case RADEON_INFO_TILING_CONFIG: - if (rdev->family >= CHIP_TAHITI) - value = rdev->config.si.tile_config; + if (rdev->family >= CHIP_BONAIRE) + *value = rdev->config.cik.tile_config; + else if (rdev->family >= CHIP_TAHITI) + *value = rdev->config.si.tile_config; else if (rdev->family >= CHIP_CAYMAN) - value = rdev->config.cayman.tile_config; + *value = rdev->config.cayman.tile_config; else if (rdev->family >= CHIP_CEDAR) - value = rdev->config.evergreen.tile_config; + *value = rdev->config.evergreen.tile_config; else if (rdev->family >= CHIP_RV770) - value = rdev->config.rv770.tile_config; + *value = rdev->config.rv770.tile_config; else if (rdev->family >= CHIP_R600) - value = rdev->config.r600.tile_config; + *value = rdev->config.r600.tile_config; else { DRM_DEBUG_KMS("tiling config is r6xx+ only!\n"); return -EINVAL; @@ -968,70 +945,88 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) * * When returning, the value is 1 if filp owns hyper-z access, * 0 otherwise. */ - if (value >= 2) { - DRM_DEBUG_KMS("WANT_HYPERZ: invalid value %d\n", value); + if (copy_from_user(value, value_ptr, sizeof(uint32_t))) { + DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__); + return -EFAULT; + } + if (*value >= 2) { + DRM_DEBUG_KMS("WANT_HYPERZ: invalid value %d\n", *value); return -EINVAL; } - radeon_set_filp_rights(dev, &rdev->hyperz_filp, filp, &value); + radeon_set_filp_rights(dev, &rdev->hyperz_filp, filp, value); break; case RADEON_INFO_WANT_CMASK: /* The same logic as Hyper-Z. */ - if (value >= 2) { - DRM_DEBUG_KMS("WANT_CMASK: invalid value %d\n", value); + if (copy_from_user(value, value_ptr, sizeof(uint32_t))) { + DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__); + return -EFAULT; + } + if (*value >= 2) { + DRM_DEBUG_KMS("WANT_CMASK: invalid value %d\n", *value); return -EINVAL; } - radeon_set_filp_rights(dev, &rdev->cmask_filp, filp, &value); + radeon_set_filp_rights(dev, &rdev->cmask_filp, filp, value); break; case RADEON_INFO_CLOCK_CRYSTAL_FREQ: /* return clock value in KHz */ - value = rdev->clock.spll.reference_freq * 10; + if (rdev->asic->get_xclk) + *value = radeon_get_xclk(rdev) * 10; + else + *value = rdev->clock.spll.reference_freq * 10; break; case RADEON_INFO_NUM_BACKENDS: - if (rdev->family >= CHIP_TAHITI) - value = rdev->config.si.max_backends_per_se * + if (rdev->family >= CHIP_BONAIRE) + *value = rdev->config.cik.max_backends_per_se * + rdev->config.cik.max_shader_engines; + else if (rdev->family >= CHIP_TAHITI) + *value = rdev->config.si.max_backends_per_se * rdev->config.si.max_shader_engines; else if (rdev->family >= CHIP_CAYMAN) - value = rdev->config.cayman.max_backends_per_se * + *value = rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines; else if (rdev->family >= CHIP_CEDAR) - value = rdev->config.evergreen.max_backends; + *value = rdev->config.evergreen.max_backends; else if (rdev->family >= CHIP_RV770) - value = rdev->config.rv770.max_backends; + *value = rdev->config.rv770.max_backends; else if (rdev->family >= CHIP_R600) - value = rdev->config.r600.max_backends; + *value = rdev->config.r600.max_backends; else { return -EINVAL; } break; case RADEON_INFO_NUM_TILE_PIPES: - if (rdev->family >= CHIP_TAHITI) - value = rdev->config.si.max_tile_pipes; + if (rdev->family >= CHIP_BONAIRE) + *value = rdev->config.cik.max_tile_pipes; + else if (rdev->family >= CHIP_TAHITI) + *value = rdev->config.si.max_tile_pipes; else if (rdev->family >= CHIP_CAYMAN) - value = rdev->config.cayman.max_tile_pipes; + *value = rdev->config.cayman.max_tile_pipes; else if (rdev->family >= CHIP_CEDAR) - value = rdev->config.evergreen.max_tile_pipes; + *value = rdev->config.evergreen.max_tile_pipes; else if (rdev->family >= CHIP_RV770) - value = rdev->config.rv770.max_tile_pipes; + *value = rdev->config.rv770.max_tile_pipes; else if (rdev->family >= CHIP_R600) - value = rdev->config.r600.max_tile_pipes; + *value = rdev->config.r600.max_tile_pipes; else { return -EINVAL; } break; case RADEON_INFO_FUSION_GART_WORKING: - value = 1; + *value = 1; break; case RADEON_INFO_BACKEND_MAP: - if (rdev->family >= CHIP_TAHITI) - value = rdev->config.si.backend_map; + if (rdev->family >= CHIP_BONAIRE) + *value = rdev->config.cik.backend_map; + else if (rdev->family >= CHIP_TAHITI) + *value = rdev->config.si.backend_map; else if (rdev->family >= CHIP_CAYMAN) - value = rdev->config.cayman.backend_map; + *value = rdev->config.cayman.backend_map; else if (rdev->family >= CHIP_CEDAR) - value = rdev->config.evergreen.backend_map; + *value = rdev->config.evergreen.backend_map; else if (rdev->family >= CHIP_RV770) - value = rdev->config.rv770.backend_map; + *value = rdev->config.rv770.backend_map; else if (rdev->family >= CHIP_R600) - value = rdev->config.r600.backend_map; + *value = rdev->config.r600.backend_map; else { return -EINVAL; } @@ -1040,60 +1035,204 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) /* this is where we report if vm is supported or not */ if (rdev->family < CHIP_CAYMAN) return -EINVAL; - value = RADEON_VA_RESERVED_SIZE; + *value = RADEON_VA_RESERVED_SIZE; break; case RADEON_INFO_IB_VM_MAX_SIZE: /* this is where we report if vm is supported or not */ if (rdev->family < CHIP_CAYMAN) return -EINVAL; - value = RADEON_IB_VM_MAX_SIZE; + *value = RADEON_IB_VM_MAX_SIZE; break; case RADEON_INFO_MAX_PIPES: - if (rdev->family >= CHIP_TAHITI) - value = rdev->config.si.max_cu_per_sh; + if (rdev->family >= CHIP_BONAIRE) + *value = rdev->config.cik.max_cu_per_sh; + else if (rdev->family >= CHIP_TAHITI) + *value = rdev->config.si.max_cu_per_sh; else if (rdev->family >= CHIP_CAYMAN) - value = rdev->config.cayman.max_pipes_per_simd; + *value = rdev->config.cayman.max_pipes_per_simd; else if (rdev->family >= CHIP_CEDAR) - value = rdev->config.evergreen.max_pipes; + *value = rdev->config.evergreen.max_pipes; else if (rdev->family >= CHIP_RV770) - value = rdev->config.rv770.max_pipes; + *value = rdev->config.rv770.max_pipes; else if (rdev->family >= CHIP_R600) - value = rdev->config.r600.max_pipes; + *value = rdev->config.r600.max_pipes; else { return -EINVAL; } break; + case RADEON_INFO_TIMESTAMP: + if (rdev->family < CHIP_R600) { + DRM_DEBUG_KMS("timestamp is r6xx+ only!\n"); + return -EINVAL; + } + value = (uint32_t*)&value64; + value_size = sizeof(uint64_t); + value64 = radeon_get_gpu_clock_counter(rdev); + break; case RADEON_INFO_MAX_SE: - if (rdev->family >= CHIP_TAHITI) - value = rdev->config.si.max_shader_engines; + if (rdev->family >= CHIP_BONAIRE) + *value = rdev->config.cik.max_shader_engines; + else if (rdev->family >= CHIP_TAHITI) + *value = rdev->config.si.max_shader_engines; else if (rdev->family >= CHIP_CAYMAN) - value = rdev->config.cayman.max_shader_engines; + *value = rdev->config.cayman.max_shader_engines; else if (rdev->family >= CHIP_CEDAR) - value = rdev->config.evergreen.num_ses; + *value = rdev->config.evergreen.num_ses; else - value = 1; + *value = 1; break; case RADEON_INFO_MAX_SH_PER_SE: - if (rdev->family >= CHIP_TAHITI) - value = rdev->config.si.max_sh_per_se; + if (rdev->family >= CHIP_BONAIRE) + *value = rdev->config.cik.max_sh_per_se; + else if (rdev->family >= CHIP_TAHITI) + *value = rdev->config.si.max_sh_per_se; else return -EINVAL; break; + case RADEON_INFO_FASTFB_WORKING: + *value = rdev->fastfb_working; + break; + case RADEON_INFO_RING_WORKING: + if (copy_from_user(value, value_ptr, sizeof(uint32_t))) { + DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__); + return -EFAULT; + } + switch (*value) { + case RADEON_CS_RING_GFX: + case RADEON_CS_RING_COMPUTE: + *value = rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready; + break; + case RADEON_CS_RING_DMA: + *value = rdev->ring[R600_RING_TYPE_DMA_INDEX].ready; + *value |= rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready; + break; + case RADEON_CS_RING_UVD: + *value = rdev->ring[R600_RING_TYPE_UVD_INDEX].ready; + break; + case RADEON_CS_RING_VCE: + *value = rdev->ring[TN_RING_TYPE_VCE1_INDEX].ready; + break; + default: + return -EINVAL; + } + break; + case RADEON_INFO_SI_TILE_MODE_ARRAY: + if (rdev->family >= CHIP_BONAIRE) { + value = rdev->config.cik.tile_mode_array; + value_size = sizeof(uint32_t)*32; + } else if (rdev->family >= CHIP_TAHITI) { + value = rdev->config.si.tile_mode_array; + value_size = sizeof(uint32_t)*32; + } else { + DRM_DEBUG_KMS("tile mode array is si+ only!\n"); + return -EINVAL; + } + break; + case RADEON_INFO_CIK_MACROTILE_MODE_ARRAY: + if (rdev->family >= CHIP_BONAIRE) { + value = rdev->config.cik.macrotile_mode_array; + value_size = sizeof(uint32_t)*16; + } else { + DRM_DEBUG_KMS("macrotile mode array is cik+ only!\n"); + return -EINVAL; + } + break; case RADEON_INFO_SI_CP_DMA_COMPUTE: - value = 1; + *value = 1; break; case RADEON_INFO_SI_BACKEND_ENABLED_MASK: - if (rdev->family >= CHIP_TAHITI) { - value = rdev->config.si.backend_enable_mask; + if (rdev->family >= CHIP_BONAIRE) { + *value = rdev->config.cik.backend_enable_mask; + } else if (rdev->family >= CHIP_TAHITI) { + *value = rdev->config.si.backend_enable_mask; } else { DRM_DEBUG_KMS("BACKEND_ENABLED_MASK is si+ only!\n"); } break; + case RADEON_INFO_MAX_SCLK: + if ((rdev->pm.pm_method == PM_METHOD_DPM) && + rdev->pm.dpm_enabled) + *value = rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk * 10; + else + *value = rdev->pm.default_sclk * 10; + break; + case RADEON_INFO_VCE_FW_VERSION: + *value = rdev->vce.fw_version; + break; + case RADEON_INFO_VCE_FB_VERSION: + *value = rdev->vce.fb_version; + break; + case RADEON_INFO_NUM_BYTES_MOVED: + value = (uint32_t*)&value64; + value_size = sizeof(uint64_t); + value64 = atomic64_read(&rdev->num_bytes_moved); + break; + case RADEON_INFO_VRAM_USAGE: + value = (uint32_t*)&value64; + value_size = sizeof(uint64_t); + value64 = atomic64_read(&rdev->vram_usage); + break; + case RADEON_INFO_GTT_USAGE: + value = (uint32_t*)&value64; + value_size = sizeof(uint64_t); + value64 = atomic64_read(&rdev->gtt_usage); + break; + case RADEON_INFO_ACTIVE_CU_COUNT: + if (rdev->family >= CHIP_BONAIRE) + *value = rdev->config.cik.active_cus; + else if (rdev->family >= CHIP_TAHITI) + *value = rdev->config.si.active_cus; + else if (rdev->family >= CHIP_CAYMAN) + *value = rdev->config.cayman.active_simds; + else if (rdev->family >= CHIP_CEDAR) + *value = rdev->config.evergreen.active_simds; + else if (rdev->family >= CHIP_RV770) + *value = rdev->config.rv770.active_simds; + else if (rdev->family >= CHIP_R600) + *value = rdev->config.r600.active_simds; + else + *value = 1; + break; + case RADEON_INFO_CURRENT_GPU_TEMP: + /* get temperature in millidegrees C */ + if (rdev->asic->pm.get_temperature) + *value = radeon_get_temperature(rdev); + else + *value = 0; + break; + case RADEON_INFO_CURRENT_GPU_SCLK: + /* get sclk in Mhz */ + if (rdev->pm.dpm_enabled) + *value = radeon_dpm_get_current_sclk(rdev) / 100; + else + *value = rdev->pm.current_sclk / 100; + break; + case RADEON_INFO_CURRENT_GPU_MCLK: + /* get mclk in Mhz */ + if (rdev->pm.dpm_enabled) + *value = radeon_dpm_get_current_mclk(rdev) / 100; + else + *value = rdev->pm.current_mclk / 100; + break; + case RADEON_INFO_READ_REG: + if (copy_from_user(value, value_ptr, sizeof(uint32_t))) { + DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__); + return -EFAULT; + } + if (radeon_get_allowed_info_register(rdev, *value, value)) + return -EINVAL; + break; + case RADEON_INFO_VA_UNMAP_WORKING: + *value = true; + break; + case RADEON_INFO_GPU_RESET_COUNTER: + *value = atomic_read(&rdev->gpu_reset_counter); + break; default: DRM_DEBUG_KMS("Invalid request %d\n", info->request); return -EINVAL; } - if (DRM_COPY_TO_USER(value_ptr, &value, sizeof(uint32_t))) { + if (copy_to_user(value_ptr, (char*)value, value_size)) { DRM_ERROR("copy_to_user %s:%u\n", __func__, __LINE__); return -EFAULT; } @@ -1105,38 +1244,22 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) * Outdated mess for old drm with Xorg being in charge (void function now). */ /** - * radeon_driver_firstopen_kms - drm callback for first open - * - * @dev: drm dev pointer - * - * Nothing to be done for KMS (all asics). - * Returns 0 on success. - */ -int radeon_driver_firstopen_kms(struct drm_device *dev) -{ - return 0; -} - -/** - * radeon_driver_firstopen_kms - drm callback for last close + * radeon_driver_lastclose_kms - drm callback for last close * * @dev: drm dev pointer * - * Switch vga switcheroo state after last close (all asics). + * Switch vga_switcheroo state after last close (all asics). */ void radeon_driver_lastclose_kms(struct drm_device *dev) { struct radeon_device *rdev = dev->dev_private; - struct drm_fb_helper *fb_helper = (void *)rdev->mode_info.rfbdev; #ifdef __sparc64__ fbwscons_setcolormap(&rdev->sf, radeondrm_setcolor); #endif if (rdev->mode_info.mode_config_initialized) - drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper); -#ifdef notyet + radeon_fbdev_restore_mode(rdev); vga_switcheroo_process_delayed_switch(); -#endif } /** @@ -1151,13 +1274,18 @@ void radeon_driver_lastclose_kms(struct drm_device *dev) int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) { struct radeon_device *rdev = dev->dev_private; + int r; file_priv->driver_priv = NULL; + r = pm_runtime_get_sync(dev->dev); + if (r < 0) + return r; + /* new gpu have virtual address space support */ if (rdev->family >= CHIP_CAYMAN) { struct radeon_fpriv *fpriv; - struct radeon_bo_va *bo_va; + struct radeon_vm *vm; int r; fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL); @@ -1165,33 +1293,40 @@ int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) return -ENOMEM; } - radeon_vm_init(rdev, &fpriv->vm); - if (rdev->accel_working) { + vm = &fpriv->vm; + r = radeon_vm_init(rdev, vm); + if (r) { + kfree(fpriv); + return r; + } + r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false); if (r) { - radeon_vm_fini(rdev, &fpriv->vm); + radeon_vm_fini(rdev, vm); kfree(fpriv); return r; } /* map the ib pool buffer read only into * virtual address space */ - bo_va = radeon_vm_bo_add(rdev, &fpriv->vm, - rdev->ring_tmp_bo.bo); - r = radeon_vm_bo_set_addr(rdev, bo_va, RADEON_VA_IB_OFFSET, + vm->ib_bo_va = radeon_vm_bo_add(rdev, vm, + rdev->ring_tmp_bo.bo); + r = radeon_vm_bo_set_addr(rdev, vm->ib_bo_va, + RADEON_VA_IB_OFFSET, RADEON_VM_PAGE_READABLE | RADEON_VM_PAGE_SNOOPED); - - radeon_bo_unreserve(rdev->ring_tmp_bo.bo); if (r) { - radeon_vm_fini(rdev, &fpriv->vm); + radeon_vm_fini(rdev, vm); kfree(fpriv); return r; } } file_priv->driver_priv = fpriv; } + + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); return 0; } @@ -1211,21 +1346,19 @@ void radeon_driver_postclose_kms(struct drm_device *dev, /* new gpu have virtual address space support */ if (rdev->family >= CHIP_CAYMAN && file_priv->driver_priv) { struct radeon_fpriv *fpriv = file_priv->driver_priv; - struct radeon_bo_va *bo_va; + struct radeon_vm *vm = &fpriv->vm; int r; if (rdev->accel_working) { r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false); if (!r) { - bo_va = radeon_vm_bo_find(&fpriv->vm, - rdev->ring_tmp_bo.bo); - if (bo_va) - radeon_vm_bo_rmv(rdev, bo_va); + if (vm->ib_bo_va) + radeon_vm_bo_rmv(rdev, vm->ib_bo_va); radeon_bo_unreserve(rdev->ring_tmp_bo.bo); } + radeon_vm_fini(rdev, vm); } - radeon_vm_fini(rdev, &fpriv->vm); kfree(fpriv); file_priv->driver_priv = NULL; } @@ -1244,10 +1377,16 @@ void radeon_driver_preclose_kms(struct drm_device *dev, struct drm_file *file_priv) { struct radeon_device *rdev = dev->dev_private; + + mutex_lock(&rdev->gem.mutex); if (rdev->hyperz_filp == file_priv) rdev->hyperz_filp = NULL; if (rdev->cmask_filp == file_priv) rdev->cmask_filp = NULL; + mutex_unlock(&rdev->gem.mutex); + + radeon_uvd_free_handles(rdev, file_priv); + radeon_vce_free_handles(rdev, file_priv); } /* @@ -1262,8 +1401,10 @@ void radeon_driver_preclose_kms(struct drm_device *dev, * Gets the frame count on the requested crtc (all asics). * Returns frame count on success, -EINVAL on failure. */ -u32 radeon_get_vblank_counter_kms(struct drm_device *dev, unsigned int crtc) +u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc) { + int vpos, hpos, stat; + u32 count; struct radeon_device *rdev = dev->dev_private; if (crtc < 0 || crtc >= rdev->num_crtc) { @@ -1271,7 +1412,53 @@ u32 radeon_get_vblank_counter_kms(struct drm_device *dev, unsigned int crtc) return -EINVAL; } - return radeon_get_vblank_counter(rdev, crtc); + /* The hw increments its frame counter at start of vsync, not at start + * of vblank, as is required by DRM core vblank counter handling. + * Cook the hw count here to make it appear to the caller as if it + * incremented at start of vblank. We measure distance to start of + * vblank in vpos. vpos therefore will be >= 0 between start of vblank + * and start of vsync, so vpos >= 0 means to bump the hw frame counter + * result by 1 to give the proper appearance to caller. + */ + if (rdev->mode_info.crtcs[crtc]) { + /* Repeat readout if needed to provide stable result if + * we cross start of vsync during the queries. + */ + do { + count = radeon_get_vblank_counter(rdev, crtc); + /* Ask radeon_get_crtc_scanoutpos to return vpos as + * distance to start of vblank, instead of regular + * vertical scanout pos. + */ + stat = radeon_get_crtc_scanoutpos( + dev, crtc, GET_DISTANCE_TO_VBLANKSTART, + &vpos, &hpos, NULL, NULL, + &rdev->mode_info.crtcs[crtc]->base.hwmode); + } while (count != radeon_get_vblank_counter(rdev, crtc)); + + if (((stat & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE)) != + (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE))) { + DRM_DEBUG_VBL("Query failed! stat %d\n", stat); + } + else { + DRM_DEBUG_VBL("crtc %d: dist from vblank start %d\n", + crtc, vpos); + + /* Bump counter if we are at >= leading edge of vblank, + * but before vsync where vpos would turn negative and + * the hw counter really increments. + */ + if (vpos >= 0) + count++; + } + } + else { + /* Fallback to use value as is. */ + count = radeon_get_vblank_counter(rdev, crtc); + DRM_DEBUG_VBL("NULL mode info! Returned count may be wrong.\n"); + } + + return count; } /** @@ -1283,7 +1470,7 @@ u32 radeon_get_vblank_counter_kms(struct drm_device *dev, unsigned int crtc) * Enable the interrupt on the requested crtc (all asics). * Returns 0 on success, -EINVAL on failure. */ -int radeon_enable_vblank_kms(struct drm_device *dev, unsigned int crtc) +int radeon_enable_vblank_kms(struct drm_device *dev, int crtc) { struct radeon_device *rdev = dev->dev_private; unsigned long irqflags; @@ -1309,7 +1496,7 @@ int radeon_enable_vblank_kms(struct drm_device *dev, unsigned int crtc) * * Disable the interrupt on the requested crtc (all asics). */ -void radeon_disable_vblank_kms(struct drm_device *dev, unsigned int crtc) +void radeon_disable_vblank_kms(struct drm_device *dev, int crtc) { struct radeon_device *rdev = dev->dev_private; unsigned long irqflags; @@ -1338,7 +1525,7 @@ void radeon_disable_vblank_kms(struct drm_device *dev, unsigned int crtc) * scanout position. (all asics). * Returns postive status flags on success, negative error on failure. */ -int radeon_get_vblank_timestamp_kms(struct drm_device *dev, unsigned int crtc, +int radeon_get_vblank_timestamp_kms(struct drm_device *dev, int crtc, int *max_error, struct timeval *vblank_time, unsigned flags) @@ -1362,96 +1549,49 @@ int radeon_get_vblank_timestamp_kms(struct drm_device *dev, unsigned int crtc, &drmcrtc->hwmode); } -/* - * IOCTL. - */ -int radeon_dma_ioctl_kms(struct drm_device *dev, struct drm_dma *d, - struct drm_file *file_priv) -{ - /* Not valid in KMS. */ - return -EINVAL; -} - -#define KMS_INVALID_IOCTL(name) \ -int name(struct drm_device *dev, void *data, struct drm_file *file_priv)\ -{ \ - DRM_ERROR("invalid ioctl with kms %s\n", __func__); \ - return -EINVAL; \ -} - -/* - * All these ioctls are invalid in kms world. - */ -KMS_INVALID_IOCTL(radeon_cp_init_kms) -KMS_INVALID_IOCTL(radeon_cp_start_kms) -KMS_INVALID_IOCTL(radeon_cp_stop_kms) -KMS_INVALID_IOCTL(radeon_cp_reset_kms) -KMS_INVALID_IOCTL(radeon_cp_idle_kms) -KMS_INVALID_IOCTL(radeon_cp_resume_kms) -KMS_INVALID_IOCTL(radeon_engine_reset_kms) -KMS_INVALID_IOCTL(radeon_fullscreen_kms) -KMS_INVALID_IOCTL(radeon_cp_swap_kms) -KMS_INVALID_IOCTL(radeon_cp_clear_kms) -KMS_INVALID_IOCTL(radeon_cp_vertex_kms) -KMS_INVALID_IOCTL(radeon_cp_indices_kms) -KMS_INVALID_IOCTL(radeon_cp_texture_kms) -KMS_INVALID_IOCTL(radeon_cp_stipple_kms) -KMS_INVALID_IOCTL(radeon_cp_indirect_kms) -KMS_INVALID_IOCTL(radeon_cp_vertex2_kms) -KMS_INVALID_IOCTL(radeon_cp_cmdbuf_kms) -KMS_INVALID_IOCTL(radeon_cp_getparam_kms) -KMS_INVALID_IOCTL(radeon_cp_flip_kms) -KMS_INVALID_IOCTL(radeon_mem_alloc_kms) -KMS_INVALID_IOCTL(radeon_mem_free_kms) -KMS_INVALID_IOCTL(radeon_mem_init_heap_kms) -KMS_INVALID_IOCTL(radeon_irq_emit_kms) -KMS_INVALID_IOCTL(radeon_irq_wait_kms) -KMS_INVALID_IOCTL(radeon_cp_setparam_kms) -KMS_INVALID_IOCTL(radeon_surface_alloc_kms) -KMS_INVALID_IOCTL(radeon_surface_free_kms) - - -struct drm_ioctl_desc radeon_ioctls_kms[] = { - DRM_IOCTL_DEF_DRV(RADEON_CP_INIT, radeon_cp_init_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), - DRM_IOCTL_DEF_DRV(RADEON_CP_START, radeon_cp_start_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), - DRM_IOCTL_DEF_DRV(RADEON_CP_STOP, radeon_cp_stop_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), - DRM_IOCTL_DEF_DRV(RADEON_CP_RESET, radeon_cp_reset_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), - DRM_IOCTL_DEF_DRV(RADEON_CP_IDLE, radeon_cp_idle_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_CP_RESUME, radeon_cp_resume_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_RESET, radeon_engine_reset_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_FULLSCREEN, radeon_fullscreen_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_SWAP, radeon_cp_swap_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_CLEAR, radeon_cp_clear_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_VERTEX, radeon_cp_vertex_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_INDICES, radeon_cp_indices_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_TEXTURE, radeon_cp_texture_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_STIPPLE, radeon_cp_stipple_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_INDIRECT, radeon_cp_indirect_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), - DRM_IOCTL_DEF_DRV(RADEON_VERTEX2, radeon_cp_vertex2_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_CMDBUF, radeon_cp_cmdbuf_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_GETPARAM, radeon_cp_getparam_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_FLIP, radeon_cp_flip_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_ALLOC, radeon_mem_alloc_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_FREE, radeon_mem_free_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_INIT_HEAP, radeon_mem_init_heap_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), - DRM_IOCTL_DEF_DRV(RADEON_IRQ_EMIT, radeon_irq_emit_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_IRQ_WAIT, radeon_irq_wait_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_SETPARAM, radeon_cp_setparam_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_SURF_ALLOC, radeon_surface_alloc_kms, DRM_AUTH), - DRM_IOCTL_DEF_DRV(RADEON_SURF_FREE, radeon_surface_free_kms, DRM_AUTH), +const struct drm_ioctl_desc radeon_ioctls_kms[] = { + DRM_IOCTL_DEF_DRV(RADEON_CP_INIT, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(RADEON_CP_START, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(RADEON_CP_STOP, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(RADEON_CP_RESET, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(RADEON_CP_IDLE, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_CP_RESUME, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_RESET, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_FULLSCREEN, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_SWAP, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_CLEAR, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_VERTEX, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_INDICES, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_TEXTURE, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_STIPPLE, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_INDIRECT, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(RADEON_VERTEX2, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_CMDBUF, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_GETPARAM, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_FLIP, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_ALLOC, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_FREE, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_INIT_HEAP, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(RADEON_IRQ_EMIT, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_IRQ_WAIT, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_SETPARAM, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_SURF_ALLOC, drm_invalid_op, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_SURF_FREE, drm_invalid_op, DRM_AUTH), /* KMS */ - DRM_IOCTL_DEF_DRV(RADEON_GEM_INFO, radeon_gem_info_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_GEM_CREATE, radeon_gem_create_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_GEM_MMAP, radeon_gem_mmap_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_DOMAIN, radeon_gem_set_domain_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_GEM_PREAD, radeon_gem_pread_ioctl, DRM_AUTH|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(RADEON_GEM_PWRITE, radeon_gem_pwrite_ioctl, DRM_AUTH|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(RADEON_GEM_WAIT_IDLE, radeon_gem_wait_idle_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_CS, radeon_cs_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_INFO, radeon_info_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_TILING, radeon_gem_set_tiling_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_GEM_GET_TILING, radeon_gem_get_tiling_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_GEM_BUSY, radeon_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(RADEON_GEM_VA, radeon_gem_va_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_INFO, radeon_gem_info_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_CREATE, radeon_gem_create_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_MMAP, radeon_gem_mmap_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_DOMAIN, radeon_gem_set_domain_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_PREAD, radeon_gem_pread_ioctl, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_GEM_PWRITE, radeon_gem_pwrite_ioctl, DRM_AUTH), + DRM_IOCTL_DEF_DRV(RADEON_GEM_WAIT_IDLE, radeon_gem_wait_idle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_CS, radeon_cs_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_INFO, radeon_info_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_TILING, radeon_gem_set_tiling_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_GET_TILING, radeon_gem_get_tiling_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_BUSY, radeon_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_VA, radeon_gem_va_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_OP, radeon_gem_op_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RADEON_GEM_USERPTR, radeon_gem_userptr_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), }; -int radeon_max_kms_ioctl = DRM_ARRAY_SIZE(radeon_ioctls_kms); +int radeon_max_kms_ioctl = ARRAY_SIZE(radeon_ioctls_kms); diff --git a/sys/dev/pci/drm/radeon/radeon_legacy_crtc.c b/sys/dev/pci/drm/radeon/radeon_legacy_crtc.c index a360e714717..5dc0dc09277 100644 --- a/sys/dev/pci/drm/radeon/radeon_legacy_crtc.c +++ b/sys/dev/pci/drm/radeon/radeon_legacy_crtc.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_legacy_crtc.c,v 1.6 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2007-8 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -332,6 +331,8 @@ static void radeon_crtc_dpms(struct drm_crtc *crtc, int mode) WREG32_P(RADEON_CRTC_EXT_CNTL, crtc_ext_cntl, ~(mask | crtc_ext_cntl)); } drm_vblank_post_modeset(dev, radeon_crtc->crtc_id); + /* Make sure vblank interrupt is still enabled if needed */ + radeon_irq_set(rdev); radeon_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: @@ -709,9 +710,6 @@ static bool radeon_set_crtc_timing(struct drm_crtc *crtc, struct drm_display_mod RADEON_CRTC_VSYNC_DIS | RADEON_CRTC_HSYNC_DIS | RADEON_CRTC_DISPLAY_DIS); - crtc_ext_cntl &= ~(RADEON_CRTC_SYNC_TRISTAT | - RADEON_CRTC_VSYNC_TRISTAT | - RADEON_CRTC_HSYNC_TRISTAT); disp_merge_cntl = RREG32(RADEON_DISP_MERGE_CNTL); disp_merge_cntl &= ~RADEON_DISP_RGB_OFFSET_EN; @@ -1058,16 +1056,15 @@ static int radeon_crtc_mode_set(struct drm_crtc *crtc, DRM_ERROR("Mode need scaling but only first crtc can do that.\n"); } } + radeon_cursor_reset(crtc); return 0; } static void radeon_crtc_prepare(struct drm_crtc *crtc) { - struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct drm_device *dev = crtc->dev; struct drm_crtc *crtci; - radeon_crtc->in_mode_set = true; /* * The hardware wedges sometimes if you reconfigure one CRTC * whilst another is running (see fdo bug #24611). @@ -1078,7 +1075,6 @@ static void radeon_crtc_prepare(struct drm_crtc *crtc) static void radeon_crtc_commit(struct drm_crtc *crtc) { - struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct drm_device *dev = crtc->dev; struct drm_crtc *crtci; @@ -1089,7 +1085,26 @@ static void radeon_crtc_commit(struct drm_crtc *crtc) if (crtci->enabled) radeon_crtc_dpms(crtci, DRM_MODE_DPMS_ON); } - radeon_crtc->in_mode_set = false; +} + +static void radeon_crtc_disable(struct drm_crtc *crtc) +{ + radeon_crtc_dpms(crtc, DRM_MODE_DPMS_OFF); + if (crtc->primary->fb) { + int r; + struct radeon_framebuffer *radeon_fb; + struct radeon_bo *rbo; + + radeon_fb = to_radeon_framebuffer(crtc->primary->fb); + rbo = gem_to_radeon_bo(radeon_fb->obj); + r = radeon_bo_reserve(rbo, false); + if (unlikely(r)) + DRM_ERROR("failed to reserve rbo before unpin\n"); + else { + radeon_bo_unpin(rbo); + radeon_bo_unreserve(rbo); + } + } } static const struct drm_crtc_helper_funcs legacy_helper_funcs = { @@ -1101,6 +1116,7 @@ static const struct drm_crtc_helper_funcs legacy_helper_funcs = { .prepare = radeon_crtc_prepare, .commit = radeon_crtc_commit, .load_lut = radeon_crtc_load_lut, + .disable = radeon_crtc_disable }; diff --git a/sys/dev/pci/drm/radeon/radeon_legacy_encoders.c b/sys/dev/pci/drm/radeon/radeon_legacy_encoders.c index b7abddd2626..46a8aa14ff8 100644 --- a/sys/dev/pci/drm/radeon/radeon_legacy_encoders.c +++ b/sys/dev/pci/drm/radeon/radeon_legacy_encoders.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_legacy_encoders.c,v 1.8 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2007-8 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -29,6 +28,9 @@ #include #include "radeon.h" #include "atom.h" +#ifdef CONFIG_PMAC_BACKLIGHT +#include +#endif static void radeon_legacy_encoder_disable(struct drm_encoder *encoder) { @@ -387,11 +389,14 @@ void radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder, memset(&props, 0, sizeof(props)); props.max_brightness = RADEON_MAX_BL_LEVEL; props.type = BACKLIGHT_RAW; -#ifdef __linux__ +#ifdef notyet snprintf(bl_name, sizeof(bl_name), "radeon_bl%d", dev->primary->index); +#else + snprintf(bl_name, sizeof(bl_name), + "radeon_bl%d", 0); #endif - bd = backlight_device_register(bl_name, &drm_connector->kdev, + bd = backlight_device_register(bl_name, drm_connector->kdev, pdata, &radeon_backlight_ops, &props); if (IS_ERR(bd)) { DRM_ERROR("Backlight registration failed\n"); @@ -440,6 +445,7 @@ void radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder, backlight_update_status(bd); DRM_INFO("radeon legacy LVDS backlight initialized\n"); + rdev->mode_info.bl_encoder = radeon_encoder; return; diff --git a/sys/dev/pci/drm/radeon/radeon_legacy_tv.c b/sys/dev/pci/drm/radeon/radeon_legacy_tv.c index 4fb2fabc099..2ea4887f3ee 100644 --- a/sys/dev/pci/drm/radeon/radeon_legacy_tv.c +++ b/sys/dev/pci/drm/radeon/radeon_legacy_tv.c @@ -1,5 +1,3 @@ -/* $OpenBSD: radeon_legacy_tv.c,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ - #include #include #include "radeon.h" diff --git a/sys/dev/pci/drm/radeon/radeon_mn.c b/sys/dev/pci/drm/radeon/radeon_mn.c new file mode 100644 index 00000000000..460a6a40a69 --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_mn.c @@ -0,0 +1,319 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Christian König + */ + +#include +#include + +#include "radeon.h" + +struct radeon_mn { + /* constant after initialisation */ + struct radeon_device *rdev; + struct mm_struct *mm; + struct mmu_notifier mn; + + /* only used on destruction */ + struct work_struct work; + + /* protected by rdev->mn_lock */ + struct hlist_node node; + + /* objects protected by lock */ + struct rwlock lock; + struct rb_root objects; +}; + +struct radeon_mn_node { + struct interval_tree_node it; + struct list_head bos; +}; + +/** + * radeon_mn_destroy - destroy the rmn + * + * @work: previously sheduled work item + * + * Lazy destroys the notifier from a work item + */ +static void radeon_mn_destroy(struct work_struct *work) +{ + struct radeon_mn *rmn = container_of(work, struct radeon_mn, work); + struct radeon_device *rdev = rmn->rdev; + struct radeon_mn_node *node, *next_node; + struct radeon_bo *bo, *next_bo; + + mutex_lock(&rdev->mn_lock); + mutex_lock(&rmn->lock); + hash_del(&rmn->node); + rbtree_postorder_for_each_entry_safe(node, next_node, &rmn->objects, + it.rb) { + + interval_tree_remove(&node->it, &rmn->objects); + list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) { + bo->mn = NULL; + list_del_init(&bo->mn_list); + } + kfree(node); + } + mutex_unlock(&rmn->lock); + mutex_unlock(&rdev->mn_lock); + mmu_notifier_unregister(&rmn->mn, rmn->mm); + kfree(rmn); +} + +/** + * radeon_mn_release - callback to notify about mm destruction + * + * @mn: our notifier + * @mn: the mm this callback is about + * + * Shedule a work item to lazy destroy our notifier. + */ +static void radeon_mn_release(struct mmu_notifier *mn, + struct mm_struct *mm) +{ + struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn); + INIT_WORK(&rmn->work, radeon_mn_destroy); + schedule_work(&rmn->work); +} + +/** + * radeon_mn_invalidate_range_start - callback to notify about mm change + * + * @mn: our notifier + * @mn: the mm this callback is about + * @start: start of updated range + * @end: end of updated range + * + * We block for all BOs between start and end to be idle and + * unmap them by move them into system domain again. + */ +static void radeon_mn_invalidate_range_start(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn); + struct interval_tree_node *it; + + /* notification is exclusive, but interval is inclusive */ + end -= 1; + + mutex_lock(&rmn->lock); + + it = interval_tree_iter_first(&rmn->objects, start, end); + while (it) { + struct radeon_mn_node *node; + struct radeon_bo *bo; + long r; + + node = container_of(it, struct radeon_mn_node, it); + it = interval_tree_iter_next(it, start, end); + + list_for_each_entry(bo, &node->bos, mn_list) { + + if (!bo->tbo.ttm || bo->tbo.ttm->state != tt_bound) + continue; + + r = radeon_bo_reserve(bo, true); + if (r) { + DRM_ERROR("(%ld) failed to reserve user bo\n", r); + continue; + } + + r = reservation_object_wait_timeout_rcu(bo->tbo.resv, + true, false, MAX_SCHEDULE_TIMEOUT); + if (r <= 0) + DRM_ERROR("(%ld) failed to wait for user bo\n", r); + + radeon_ttm_placement_from_domain(bo, RADEON_GEM_DOMAIN_CPU); + r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); + if (r) + DRM_ERROR("(%ld) failed to validate user bo\n", r); + + radeon_bo_unreserve(bo); + } + } + + mutex_unlock(&rmn->lock); +} + +static const struct mmu_notifier_ops radeon_mn_ops = { + .release = radeon_mn_release, + .invalidate_range_start = radeon_mn_invalidate_range_start, +}; + +/** + * radeon_mn_get - create notifier context + * + * @rdev: radeon device pointer + * + * Creates a notifier context for current->mm. + */ +static struct radeon_mn *radeon_mn_get(struct radeon_device *rdev) +{ + struct mm_struct *mm = current->mm; + struct radeon_mn *rmn; + int r; + + down_write(&mm->mmap_sem); + mutex_lock(&rdev->mn_lock); + + hash_for_each_possible(rdev->mn_hash, rmn, node, (unsigned long)mm) + if (rmn->mm == mm) + goto release_locks; + + rmn = kzalloc(sizeof(*rmn), GFP_KERNEL); + if (!rmn) { + rmn = ERR_PTR(-ENOMEM); + goto release_locks; + } + + rmn->rdev = rdev; + rmn->mm = mm; + rmn->mn.ops = &radeon_mn_ops; + rw_init(&rmn->lock, "rmn"); + rmn->objects = RB_ROOT; + + r = __mmu_notifier_register(&rmn->mn, mm); + if (r) + goto free_rmn; + + hash_add(rdev->mn_hash, &rmn->node, (unsigned long)mm); + +release_locks: + mutex_unlock(&rdev->mn_lock); + up_write(&mm->mmap_sem); + + return rmn; + +free_rmn: + mutex_unlock(&rdev->mn_lock); + up_write(&mm->mmap_sem); + kfree(rmn); + + return ERR_PTR(r); +} + +/** + * radeon_mn_register - register a BO for notifier updates + * + * @bo: radeon buffer object + * @addr: userptr addr we should monitor + * + * Registers an MMU notifier for the given BO at the specified address. + * Returns 0 on success, -ERRNO if anything goes wrong. + */ +int radeon_mn_register(struct radeon_bo *bo, unsigned long addr) +{ + unsigned long end = addr + radeon_bo_size(bo) - 1; + struct radeon_device *rdev = bo->rdev; + struct radeon_mn *rmn; + struct radeon_mn_node *node = NULL; + struct list_head bos; + struct interval_tree_node *it; + + rmn = radeon_mn_get(rdev); + if (IS_ERR(rmn)) + return PTR_ERR(rmn); + + INIT_LIST_HEAD(&bos); + + mutex_lock(&rmn->lock); + + while ((it = interval_tree_iter_first(&rmn->objects, addr, end))) { + kfree(node); + node = container_of(it, struct radeon_mn_node, it); + interval_tree_remove(&node->it, &rmn->objects); + addr = min(it->start, addr); + end = max(it->last, end); + list_splice(&node->bos, &bos); + } + + if (!node) { + node = kmalloc(sizeof(struct radeon_mn_node), GFP_KERNEL); + if (!node) { + mutex_unlock(&rmn->lock); + return -ENOMEM; + } + } + + bo->mn = rmn; + + node->it.start = addr; + node->it.last = end; + INIT_LIST_HEAD(&node->bos); + list_splice(&bos, &node->bos); + list_add(&bo->mn_list, &node->bos); + + interval_tree_insert(&node->it, &rmn->objects); + + mutex_unlock(&rmn->lock); + + return 0; +} + +/** + * radeon_mn_unregister - unregister a BO for notifier updates + * + * @bo: radeon buffer object + * + * Remove any registration of MMU notifier updates from the buffer object. + */ +void radeon_mn_unregister(struct radeon_bo *bo) +{ + struct radeon_device *rdev = bo->rdev; + struct radeon_mn *rmn; + struct list_head *head; + + mutex_lock(&rdev->mn_lock); + rmn = bo->mn; + if (rmn == NULL) { + mutex_unlock(&rdev->mn_lock); + return; + } + + mutex_lock(&rmn->lock); + /* save the next list entry for later */ + head = bo->mn_list.next; + + bo->mn = NULL; + list_del(&bo->mn_list); + + if (list_empty(head)) { + struct radeon_mn_node *node; + node = container_of(head, struct radeon_mn_node, bos); + interval_tree_remove(&node->it, &rmn->objects); + kfree(node); + } + + mutex_unlock(&rmn->lock); + mutex_unlock(&rdev->mn_lock); +} diff --git a/sys/dev/pci/drm/radeon/radeon_mode.h b/sys/dev/pci/drm/radeon/radeon_mode.h index 0975dd44d76..81a6df8913e 100644 --- a/sys/dev/pci/drm/radeon/radeon_mode.h +++ b/sys/dev/pci/drm/radeon/radeon_mode.h @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_mode.h,v 1.6 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2000 ATI Technologies Inc., Markham, Ontario, and * VA Linux Systems Inc., Fremont, California. @@ -34,6 +33,7 @@ #include #include #include +#include #include #include @@ -45,6 +45,10 @@ struct radeon_device; #define to_radeon_encoder(x) container_of(x, struct radeon_encoder, base) #define to_radeon_framebuffer(x) container_of(x, struct radeon_framebuffer, base) +#define RADEON_MAX_HPD_PINS 7 +#define RADEON_MAX_CRTCS 6 +#define RADEON_MAX_AFMT_BLOCKS 7 + enum radeon_rmx_type { RMX_OFF, RMX_FULL, @@ -80,6 +84,13 @@ enum radeon_hpd_id { RADEON_HPD_NONE = 0xff, }; +enum radeon_output_csc { + RADEON_OUTPUT_CSC_BYPASS = 0, + RADEON_OUTPUT_CSC_TVRGB = 1, + RADEON_OUTPUT_CSC_YCBCR601 = 2, + RADEON_OUTPUT_CSC_YCBCR709 = 3, +}; + #define RADEON_MAX_I2C_BUS 16 /* radeon gpio-based i2c @@ -186,16 +197,11 @@ struct radeon_pll { struct radeon_i2c_chan { struct i2c_adapter adapter; struct drm_device *dev; -#if 0 - union { - struct i2c_algo_dp_aux_data dp; - } algo; -#else + struct i2c_algo_bit_data bit; + struct radeon_i2c_bus_rec rec; struct drm_dp_aux aux; bool has_aux; struct rwlock mutex; -#endif - struct radeon_i2c_bus_rec rec; }; /* mostly for macs, but really any system without connector tables */ @@ -236,8 +242,8 @@ struct radeon_mode_info { struct card_info *atom_card_info; enum radeon_connector_table connector_table; bool mode_config_initialized; - struct radeon_crtc *crtcs[6]; - struct radeon_afmt *afmt[6]; + struct radeon_crtc *crtcs[RADEON_MAX_CRTCS]; + struct radeon_afmt *afmt[RADEON_MAX_AFMT_BLOCKS]; /* DVI-I properties */ struct drm_property *coherent_mode_property; /* DAC enable load detect */ @@ -250,6 +256,12 @@ struct radeon_mode_info { struct drm_property *underscan_property; struct drm_property *underscan_hborder_property; struct drm_property *underscan_vborder_property; + /* audio */ + struct drm_property *audio_property; + /* FMT dithering */ + struct drm_property *dither_property; + /* Output CSC */ + struct drm_property *output_csc_property; /* hardcoded DFP edid from BIOS */ struct edid *bios_hardcoded_edid; int bios_hardcoded_edid_size; @@ -260,6 +272,9 @@ struct radeon_mode_info { u16 firmware_flags; /* pointer to backlight encoder */ struct radeon_encoder *bl_encoder; + + /* bitmask for active encoder frontends */ + uint32_t active_encoders; }; #define RADEON_MAX_BL_LEVEL 0xFF @@ -290,6 +305,7 @@ struct radeon_tv_regs { struct radeon_atom_ss { uint16_t percentage; + uint16_t percentage_divider; uint8_t type; uint16_t step; uint8_t delay; @@ -300,20 +316,31 @@ struct radeon_atom_ss { uint16_t amount; }; +enum radeon_flip_status { + RADEON_FLIP_NONE, + RADEON_FLIP_PENDING, + RADEON_FLIP_SUBMITTED +}; + struct radeon_crtc { struct drm_crtc base; int crtc_id; u16 lut_r[256], lut_g[256], lut_b[256]; bool enabled; bool can_tile; - bool in_mode_set; + bool cursor_out_of_bounds; uint32_t crtc_offset; struct drm_gem_object *cursor_bo; uint64_t cursor_addr; + int cursor_x; + int cursor_y; + int cursor_hot_x; + int cursor_hot_y; int cursor_width; int cursor_height; + int max_cursor_width; + int max_cursor_height; uint32_t legacy_display_base_addr; - uint32_t legacy_cursor_offset; enum radeon_rmx_type rmx_type; u8 h_border; u8 v_border; @@ -322,8 +349,9 @@ struct radeon_crtc { struct drm_display_mode native_mode; int pll_id; /* page flipping */ - struct radeon_unpin_work *unpin_work; - int deferred_flip_completion; + struct workqueue_struct *flip_queue; + struct radeon_flip_work *flip_work; + enum radeon_flip_status flip_status; /* pll sharing */ struct radeon_atom_ss ss; bool ss_enabled; @@ -334,6 +362,13 @@ struct radeon_crtc { u32 pll_flags; struct drm_encoder *encoder; struct drm_connector *connector; + /* for dpm */ + u32 line_time; + u32 wm_low; + u32 wm_high; + u32 lb_vblank_lead_lines; + struct drm_display_mode hw_mode; + enum radeon_output_csc output_csc; }; struct radeon_encoder_primary_dac { @@ -403,12 +438,25 @@ struct radeon_encoder_atom_dig { uint8_t backlight_level; int panel_mode; struct radeon_afmt *afmt; + struct r600_audio_pin *pin; + int active_mst_links; }; struct radeon_encoder_atom_dac { enum radeon_tv_std tv_std; }; +struct radeon_encoder_mst { + int crtc; + struct radeon_encoder *primary; + struct radeon_connector *connector; + struct drm_dp_mst_port *port; + int pbn; + int fe; + bool fe_from_be; + bool enc_active; +}; + struct radeon_encoder { struct drm_encoder base; uint32_t encoder_enum; @@ -426,17 +474,23 @@ struct radeon_encoder { int audio_polling_active; bool is_ext_encoder; u16 caps; + struct radeon_audio_funcs *audio; + enum radeon_output_csc output_csc; + bool can_mst; + uint32_t offset; + bool is_mst_encoder; + /* front end for this mst encoder */ }; struct radeon_connector_atom_dig { uint32_t igp_lane_info; /* displayport */ - struct radeon_i2c_chan *dp_i2c_bus; u8 dpcd[DP_RECEIVER_CAP_SIZE]; u8 dp_sink_type; int dp_clock; int dp_lane_count; bool edp_on; + bool is_mst; }; struct radeon_gpio_rec { @@ -444,6 +498,7 @@ struct radeon_gpio_rec { u8 id; u32 reg; u32 mask; + u32 shift; }; struct radeon_hpd { @@ -468,6 +523,22 @@ struct radeon_router { u8 cd_mux_state; }; +enum radeon_connector_audio { + RADEON_AUDIO_DISABLE = 0, + RADEON_AUDIO_ENABLE = 1, + RADEON_AUDIO_AUTO = 2 +}; + +enum radeon_connector_dither { + RADEON_FMT_DITHER_DISABLE = 0, + RADEON_FMT_DITHER_ENABLE = 1, +}; + +struct stream_attribs { + uint16_t fe; + uint16_t slots; +}; + struct radeon_connector { struct drm_connector base; uint32_t connector_id; @@ -482,10 +553,22 @@ struct radeon_connector { void *con_priv; bool dac_load_detect; bool detected_by_load; /* if the connection status was determined by load */ + bool detected_hpd_without_ddc; /* if an HPD signal was detected on DVI, but ddc probing failed */ uint16_t connector_object_id; struct radeon_hpd hpd; struct radeon_router router; struct radeon_i2c_chan *router_bus; + enum radeon_connector_audio audio; + enum radeon_connector_dither dither; + int pixelclock_for_modeset; + bool is_mst_connector; + struct radeon_connector *mst_port; + struct drm_dp_mst_port *port; + struct drm_dp_mst_topology_mgr mst_mgr; + + struct radeon_encoder *mst_encoder; + struct stream_attribs cur_stream_attribs[6]; + int enabled_attribs; }; struct radeon_framebuffer { @@ -496,10 +579,156 @@ struct radeon_framebuffer { #define ENCODER_MODE_IS_DP(em) (((em) == ATOM_ENCODER_MODE_DP) || \ ((em) == ATOM_ENCODER_MODE_DP_MST)) +struct atom_clock_dividers { + u32 post_div; + union { + struct { +#ifdef __BIG_ENDIAN + u32 reserved : 6; + u32 whole_fb_div : 12; + u32 frac_fb_div : 14; +#else + u32 frac_fb_div : 14; + u32 whole_fb_div : 12; + u32 reserved : 6; +#endif + }; + u32 fb_div; + }; + u32 ref_div; + bool enable_post_div; + bool enable_dithen; + u32 vco_mode; + u32 real_clock; + /* added for CI */ + u32 post_divider; + u32 flags; +}; + +struct atom_mpll_param { + union { + struct { +#ifdef __BIG_ENDIAN + u32 reserved : 8; + u32 clkfrac : 12; + u32 clkf : 12; +#else + u32 clkf : 12; + u32 clkfrac : 12; + u32 reserved : 8; +#endif + }; + u32 fb_div; + }; + u32 post_div; + u32 bwcntl; + u32 dll_speed; + u32 vco_mode; + u32 yclk_sel; + u32 qdr; + u32 half_rate; +}; + +#define MEM_TYPE_GDDR5 0x50 +#define MEM_TYPE_GDDR4 0x40 +#define MEM_TYPE_GDDR3 0x30 +#define MEM_TYPE_DDR2 0x20 +#define MEM_TYPE_GDDR1 0x10 +#define MEM_TYPE_DDR3 0xb0 +#define MEM_TYPE_MASK 0xf0 + +struct atom_memory_info { + u8 mem_vendor; + u8 mem_type; +}; + +#define MAX_AC_TIMING_ENTRIES 16 + +struct atom_memory_clock_range_table +{ + u8 num_entries; + u8 rsv[3]; + u32 mclk[MAX_AC_TIMING_ENTRIES]; +}; + +#define VBIOS_MC_REGISTER_ARRAY_SIZE 32 +#define VBIOS_MAX_AC_TIMING_ENTRIES 20 + +struct atom_mc_reg_entry { + u32 mclk_max; + u32 mc_data[VBIOS_MC_REGISTER_ARRAY_SIZE]; +}; + +struct atom_mc_register_address { + u16 s1; + u8 pre_reg_data; +}; + +struct atom_mc_reg_table { + u8 last; + u8 num_entries; + struct atom_mc_reg_entry mc_reg_table_entry[VBIOS_MAX_AC_TIMING_ENTRIES]; + struct atom_mc_register_address mc_reg_address[VBIOS_MC_REGISTER_ARRAY_SIZE]; +}; + +#define MAX_VOLTAGE_ENTRIES 32 + +struct atom_voltage_table_entry +{ + u16 value; + u32 smio_low; +}; + +struct atom_voltage_table +{ + u32 count; + u32 mask_low; + u32 phase_delay; + struct atom_voltage_table_entry entries[MAX_VOLTAGE_ENTRIES]; +}; + +/* Driver internal use only flags of radeon_get_crtc_scanoutpos() */ +#define USE_REAL_VBLANKSTART (1 << 30) +#define GET_DISTANCE_TO_VBLANKSTART (1 << 31) + +extern void +radeon_add_atom_connector(struct drm_device *dev, + uint32_t connector_id, + uint32_t supported_device, + int connector_type, + struct radeon_i2c_bus_rec *i2c_bus, + uint32_t igp_lane_info, + uint16_t connector_object_id, + struct radeon_hpd *hpd, + struct radeon_router *router); +extern void +radeon_add_legacy_connector(struct drm_device *dev, + uint32_t connector_id, + uint32_t supported_device, + int connector_type, + struct radeon_i2c_bus_rec *i2c_bus, + uint16_t connector_object_id, + struct radeon_hpd *hpd); +extern uint32_t +radeon_get_encoder_enum(struct drm_device *dev, uint32_t supported_device, + uint8_t dac); +extern void radeon_link_encoder_connector(struct drm_device *dev); + extern enum radeon_tv_std radeon_combios_get_tv_info(struct radeon_device *rdev); extern enum radeon_tv_std radeon_atombios_get_tv_info(struct radeon_device *rdev); +extern void radeon_atombios_get_default_voltages(struct radeon_device *rdev, + u16 *vddc, u16 *vddci, u16 *mvdd); + +extern void +radeon_combios_connected_scratch_regs(struct drm_connector *connector, + struct drm_encoder *encoder, + bool connected); +extern void +radeon_atombios_connected_scratch_regs(struct drm_connector *connector, + struct drm_encoder *encoder, + bool connected); extern struct drm_connector * radeon_get_connector_for_encoder(struct drm_encoder *encoder); @@ -510,10 +739,11 @@ extern bool radeon_dig_monitor_is_duallink(struct drm_encoder *encoder, extern u16 radeon_encoder_get_dp_bridge_encoder_id(struct drm_encoder *encoder); extern u16 radeon_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *connector); -extern bool radeon_connector_encoder_is_hbr2(struct drm_connector *connector); extern bool radeon_connector_is_dp12_capable(struct drm_connector *connector); extern int radeon_get_monitor_bpc(struct drm_connector *connector); +extern struct edid *radeon_connector_edid(struct drm_connector *connector); + extern void radeon_connector_hotplug(struct drm_connector *connector); extern int radeon_dp_mode_valid_helper(struct drm_connector *connector, struct drm_display_mode *mode); @@ -526,20 +756,31 @@ extern u8 radeon_dp_getsinktype(struct radeon_connector *radeon_connector); extern bool radeon_dp_getdpcd(struct radeon_connector *radeon_connector); extern int radeon_dp_get_panel_mode(struct drm_encoder *encoder, struct drm_connector *connector); +extern int radeon_dp_get_dp_link_config(struct drm_connector *connector, + const u8 *dpcd, + unsigned pix_clock, + unsigned *dp_lanes, unsigned *dp_rate); +extern void radeon_dp_set_rx_power_state(struct drm_connector *connector, + u8 power_state); extern void radeon_dp_aux_init(struct radeon_connector *radeon_connector); extern ssize_t radeon_dp_aux_transfer_native(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg); extern void atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mode); +extern void atombios_dig_encoder_setup2(struct drm_encoder *encoder, int action, int panel_mode, int enc_override); extern void radeon_atom_encoder_init(struct radeon_device *rdev); extern void radeon_atom_disp_eng_pll_init(struct radeon_device *rdev); extern void atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t lane_num, uint8_t lane_set); +extern void atombios_dig_transmitter_setup2(struct drm_encoder *encoder, + int action, uint8_t lane_num, + uint8_t lane_set, int fe); +extern void atombios_set_mst_encoder_crtc_source(struct drm_encoder *encoder, + int fe); extern void radeon_atom_ext_encoder_setup_ddc(struct drm_encoder *encoder); extern struct drm_encoder *radeon_get_external_encoder(struct drm_encoder *encoder); -extern int radeon_dp_i2c_aux_ch(struct i2c_controller *adapter, int mode, - u8 write_byte, u8 *read_byte); +void radeon_atom_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le); extern void radeon_i2c_init(struct radeon_device *rdev); extern void radeon_i2c_fini(struct radeon_device *rdev); @@ -550,9 +791,6 @@ extern void radeon_i2c_add(struct radeon_device *rdev, const char *name); extern struct radeon_i2c_chan *radeon_i2c_lookup(struct radeon_device *rdev, struct radeon_i2c_bus_rec *i2c_bus); -extern struct radeon_i2c_chan *radeon_i2c_create_dp(struct drm_device *dev, - struct radeon_i2c_bus_rec *rec, - const char *name); extern struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev, struct radeon_i2c_bus_rec *rec, const char *name); @@ -568,9 +806,6 @@ extern void radeon_i2c_put_byte(struct radeon_i2c_chan *i2c, extern void radeon_router_select_ddc_port(struct radeon_connector *radeon_connector); extern void radeon_router_select_cd_port(struct radeon_connector *radeon_connector); extern bool radeon_ddc_probe(struct radeon_connector *radeon_connector, bool use_aux); -extern int radeon_ddc_get_modes(struct radeon_connector *radeon_connector); - -extern struct drm_encoder *radeon_best_encoder(struct drm_connector *connector); extern bool radeon_atombios_get_ppll_ss_info(struct radeon_device *rdev, struct radeon_atom_ss *ss, @@ -578,6 +813,8 @@ extern bool radeon_atombios_get_ppll_ss_info(struct radeon_device *rdev, extern bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev, struct radeon_atom_ss *ss, int id, u32 clock); +extern struct radeon_gpio_rec radeon_atombios_lookup_gpio(struct radeon_device *rdev, + u8 id); extern void radeon_compute_pll_legacy(struct radeon_pll *pll, uint64_t freq, @@ -607,6 +844,7 @@ extern void atombios_digital_setup(struct drm_encoder *encoder, int action); extern int atombios_get_encoder_mode(struct drm_encoder *encoder); extern bool atombios_set_edp_panel_power(struct drm_connector *connector, int action); extern void radeon_encoder_set_active_device(struct drm_encoder *encoder); +extern bool radeon_encoder_is_digital(struct drm_encoder *encoder); extern void radeon_crtc_load_lut(struct drm_crtc *crtc); extern int atombios_crtc_set_base(struct drm_crtc *crtc, int x, int y, @@ -631,13 +869,16 @@ extern int radeon_crtc_set_base_atomic(struct drm_crtc *crtc, extern int radeon_crtc_do_set_base(struct drm_crtc *crtc, struct drm_framebuffer *fb, int x, int y, int atomic); -extern int radeon_crtc_cursor_set(struct drm_crtc *crtc, - struct drm_file *file_priv, - uint32_t handle, - uint32_t width, - uint32_t height); +extern int radeon_crtc_cursor_set2(struct drm_crtc *crtc, + struct drm_file *file_priv, + uint32_t handle, + uint32_t width, + uint32_t height, + int32_t hot_x, + int32_t hot_y); extern int radeon_crtc_cursor_move(struct drm_crtc *crtc, int x, int y); +extern void radeon_cursor_reset(struct drm_crtc *crtc); extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, unsigned int flags, int *vpos, int *hpos, @@ -734,18 +975,39 @@ void radeon_legacy_tv_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode); +/* fmt blocks */ +void avivo_program_fmt(struct drm_encoder *encoder); +void dce3_program_fmt(struct drm_encoder *encoder); +void dce4_program_fmt(struct drm_encoder *encoder); +void dce8_program_fmt(struct drm_encoder *encoder); + /* fbdev layer */ int radeon_fbdev_init(struct radeon_device *rdev); void radeon_fbdev_fini(struct radeon_device *rdev); void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state); -int radeon_fbdev_total_size(struct radeon_device *rdev); bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj); +void radeon_fbdev_restore_mode(struct radeon_device *rdev); void radeon_fb_output_poll_changed(struct radeon_device *rdev); +void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id); + +void radeon_fb_add_connector(struct radeon_device *rdev, struct drm_connector *connector); +void radeon_fb_remove_connector(struct radeon_device *rdev, struct drm_connector *connector); + void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id); int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled); -void radeondrm_burner(void *, u_int, u_int); +/* mst */ +int radeon_dp_mst_init(struct radeon_connector *radeon_connector); +int radeon_dp_mst_probe(struct radeon_connector *radeon_connector); +int radeon_dp_mst_check_status(struct radeon_connector *radeon_connector); +int radeon_mst_debugfs_init(struct radeon_device *rdev); +void radeon_dp_mst_prepare_pll(struct drm_crtc *crtc, struct drm_display_mode *mode); + +void radeon_setup_mst_connector(struct drm_device *dev); + +int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder, int fe_idx); +void radeon_atom_release_dig_encoder(struct radeon_device *rdev, int enc_idx); #endif diff --git a/sys/dev/pci/drm/radeon/radeon_object.c b/sys/dev/pci/drm/radeon/radeon_object.c index 42f0da42ddf..9b45d533ce7 100644 --- a/sys/dev/pci/drm/radeon/radeon_object.c +++ b/sys/dev/pci/drm/radeon/radeon_object.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_object.c,v 1.12 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2009 Jerome Glisse. * All Rights Reserved. @@ -32,6 +31,7 @@ */ #include #include +#include #include "radeon.h" #include "radeon_trace.h" @@ -45,13 +45,25 @@ static void radeon_bo_clear_surface_reg(struct radeon_bo *bo); * function are calling it. */ -void radeon_bo_clear_va(struct radeon_bo *bo) +static void radeon_update_memory_usage(struct radeon_bo *bo, + unsigned mem_type, int sign) { - struct radeon_bo_va *bo_va, *tmp; - - list_for_each_entry_safe(bo_va, tmp, &bo->va, bo_list) { - /* remove from all vm address space */ - radeon_vm_bo_rmv(bo->rdev, bo_va); + struct radeon_device *rdev = bo->rdev; + u64 size = (u64)bo->tbo.num_pages << PAGE_SHIFT; + + switch (mem_type) { + case TTM_PL_TT: + if (sign > 0) + atomic64_add(size, &rdev->gtt_usage); + else + atomic64_sub(size, &rdev->gtt_usage); + break; + case TTM_PL_VRAM: + if (sign > 0) + atomic64_add(size, &rdev->vram_usage); + else + atomic64_sub(size, &rdev->vram_usage); + break; } } @@ -60,11 +72,14 @@ static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo) struct radeon_bo *bo; bo = container_of(tbo, struct radeon_bo, tbo); + + radeon_update_memory_usage(bo, bo->tbo.mem.mem_type, -1); + mutex_lock(&bo->rdev->gem.mutex); list_del_init(&bo->list); mutex_unlock(&bo->rdev->gem.mutex); radeon_bo_clear_surface_reg(bo); - radeon_bo_clear_va(bo); + WARN_ON(!list_empty(&bo->va)); drm_gem_object_release(&bo->gem_base); pool_put(&bo->rdev->ddev->objpl, bo); } @@ -78,38 +93,91 @@ bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo) void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain) { - u32 c = 0; + u32 c = 0, i; - rbo->placement.fpfn = 0; - rbo->placement.lpfn = 0; rbo->placement.placement = rbo->placements; rbo->placement.busy_placement = rbo->placements; - if (domain & RADEON_GEM_DOMAIN_VRAM) - rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | - TTM_PL_FLAG_VRAM; + if (domain & RADEON_GEM_DOMAIN_VRAM) { + /* Try placing BOs which don't need CPU access outside of the + * CPU accessible part of VRAM + */ + if ((rbo->flags & RADEON_GEM_NO_CPU_ACCESS) && + rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size) { + rbo->placements[c].fpfn = + rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; + rbo->placements[c++].flags = TTM_PL_FLAG_WC | + TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_VRAM; + } + + rbo->placements[c].fpfn = 0; + rbo->placements[c++].flags = TTM_PL_FLAG_WC | + TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_VRAM; + } + if (domain & RADEON_GEM_DOMAIN_GTT) { - if (rbo->rdev->flags & RADEON_IS_AGP) { - rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT; + if (rbo->flags & RADEON_GEM_GTT_UC) { + rbo->placements[c].fpfn = 0; + rbo->placements[c++].flags = TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_TT; + + } else if ((rbo->flags & RADEON_GEM_GTT_WC) || + (rbo->rdev->flags & RADEON_IS_AGP)) { + rbo->placements[c].fpfn = 0; + rbo->placements[c++].flags = TTM_PL_FLAG_WC | + TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_TT; } else { - rbo->placements[c++] = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT; + rbo->placements[c].fpfn = 0; + rbo->placements[c++].flags = TTM_PL_FLAG_CACHED | + TTM_PL_FLAG_TT; } } + if (domain & RADEON_GEM_DOMAIN_CPU) { - if (rbo->rdev->flags & RADEON_IS_AGP) { - rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_SYSTEM; + if (rbo->flags & RADEON_GEM_GTT_UC) { + rbo->placements[c].fpfn = 0; + rbo->placements[c++].flags = TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_SYSTEM; + + } else if ((rbo->flags & RADEON_GEM_GTT_WC) || + rbo->rdev->flags & RADEON_IS_AGP) { + rbo->placements[c].fpfn = 0; + rbo->placements[c++].flags = TTM_PL_FLAG_WC | + TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_SYSTEM; } else { - rbo->placements[c++] = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM; + rbo->placements[c].fpfn = 0; + rbo->placements[c++].flags = TTM_PL_FLAG_CACHED | + TTM_PL_FLAG_SYSTEM; } } - if (!c) - rbo->placements[c++] = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; + if (!c) { + rbo->placements[c].fpfn = 0; + rbo->placements[c++].flags = TTM_PL_MASK_CACHING | + TTM_PL_FLAG_SYSTEM; + } + rbo->placement.num_placement = c; rbo->placement.num_busy_placement = c; + + for (i = 0; i < c; ++i) { + if ((rbo->flags & RADEON_GEM_CPU_ACCESS) && + (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) && + !rbo->placements[i].fpfn) + rbo->placements[i].lpfn = + rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; + else + rbo->placements[i].lpfn = 0; + } } int radeon_bo_create(struct radeon_device *rdev, - unsigned long size, int byte_align, bool kernel, u32 domain, - struct sg_table *sg, struct radeon_bo **bo_ptr) + unsigned long size, int byte_align, bool kernel, + u32 domain, u32 flags, struct sg_table *sg, + struct reservation_object *resv, + struct radeon_bo **bo_ptr) { struct radeon_bo *bo; enum ttm_bo_type type; @@ -119,9 +187,6 @@ int radeon_bo_create(struct radeon_device *rdev, size = PAGE_ALIGN(size); -#ifdef notyet - rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping; -#endif if (kernel) { type = ttm_bo_type_kernel; } else if (sg) { @@ -146,12 +211,54 @@ int radeon_bo_create(struct radeon_device *rdev, bo->surface_reg = -1; INIT_LIST_HEAD(&bo->list); INIT_LIST_HEAD(&bo->va); + bo->initial_domain = domain & (RADEON_GEM_DOMAIN_VRAM | + RADEON_GEM_DOMAIN_GTT | + RADEON_GEM_DOMAIN_CPU); + + bo->flags = flags; + /* PCI GART is always snooped */ + if (!(rdev->flags & RADEON_IS_PCIE)) + bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC); + + /* Write-combined CPU mappings of GTT cause GPU hangs with RV6xx + * See https://bugs.freedesktop.org/show_bug.cgi?id=91268 + */ + if (rdev->family >= CHIP_RV610 && rdev->family <= CHIP_RV635) + bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC); + +#ifdef CONFIG_X86_32 + /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit + * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 + */ + bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC); +#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) + /* Don't try to enable write-combining when it can't work, or things + * may be slow + * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 + */ +#ifndef CONFIG_COMPILE_TEST +#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ + thanks to write-combining +#endif + + if (bo->flags & RADEON_GEM_GTT_WC) + DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " + "better performance thanks to write-combining\n"); + bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC); +#else + /* For architectures that don't support WC memory, + * mask out the WC flag from the BO + */ + if (!drm_arch_can_wc_memory()) + bo->flags &= ~RADEON_GEM_GTT_WC; +#endif + radeon_ttm_placement_from_domain(bo, domain); /* Kernel allocation are uninterruptible */ down_read(&rdev->pm.mclk_lock); r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type, &bo->placement, page_align, !kernel, NULL, - acc_size, sg, &radeon_ttm_bo_destroy); + acc_size, sg, resv, &radeon_ttm_bo_destroy); up_read(&rdev->pm.mclk_lock); if (unlikely(r != 0)) { return r; @@ -195,6 +302,15 @@ void radeon_bo_kunmap(struct radeon_bo *bo) ttm_bo_kunmap(&bo->kmap); } +struct radeon_bo *radeon_bo_ref(struct radeon_bo *bo) +{ + if (bo == NULL) + return NULL; + + ttm_bo_reference(&bo->tbo); + return bo; +} + void radeon_bo_unref(struct radeon_bo **bo) { struct ttm_buffer_object *tbo; @@ -204,9 +320,7 @@ void radeon_bo_unref(struct radeon_bo **bo) return; rdev = (*bo)->rdev; tbo = &((*bo)->tbo); - down_read(&rdev->pm.mclk_lock); ttm_bo_unref(&tbo); - up_read(&rdev->pm.mclk_lock); if (tbo == NULL) *bo = NULL; } @@ -216,6 +330,9 @@ int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset, { int r, i; + if (radeon_ttm_tt_has_userptr(bo->tbo.ttm)) + return -EPERM; + if (bo->pin_count) { bo->pin_count++; if (gpu_addr) @@ -235,29 +352,31 @@ int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset, return 0; } radeon_ttm_placement_from_domain(bo, domain); - if (domain == RADEON_GEM_DOMAIN_VRAM) { + for (i = 0; i < bo->placement.num_placement; i++) { /* force to pin into visible video ram */ - bo->placement.lpfn = bo->rdev->mc.visible_vram_size >> PAGE_SHIFT; + if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) && + !(bo->flags & RADEON_GEM_NO_CPU_ACCESS) && + (!max_offset || max_offset > bo->rdev->mc.visible_vram_size)) + bo->placements[i].lpfn = + bo->rdev->mc.visible_vram_size >> PAGE_SHIFT; + else + bo->placements[i].lpfn = max_offset >> PAGE_SHIFT; + + bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; } - if (max_offset) { - u64 lpfn = max_offset >> PAGE_SHIFT; - if (!bo->placement.lpfn) - bo->placement.lpfn = bo->rdev->mc.gtt_size >> PAGE_SHIFT; - - if (lpfn < bo->placement.lpfn) - bo->placement.lpfn = lpfn; - } - for (i = 0; i < bo->placement.num_placement; i++) - bo->placements[i] |= TTM_PL_FLAG_NO_EVICT; r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); if (likely(r == 0)) { bo->pin_count = 1; if (gpu_addr != NULL) *gpu_addr = radeon_bo_gpu_offset(bo); - } - if (unlikely(r != 0)) + if (domain == RADEON_GEM_DOMAIN_VRAM) + bo->rdev->vram_pin_size += radeon_bo_size(bo); + else + bo->rdev->gart_pin_size += radeon_bo_size(bo); + } else { dev_err(bo->rdev->dev, "%p pin failed\n", bo); + } return r; } @@ -277,11 +396,19 @@ int radeon_bo_unpin(struct radeon_bo *bo) bo->pin_count--; if (bo->pin_count) return 0; - for (i = 0; i < bo->placement.num_placement; i++) - bo->placements[i] &= ~TTM_PL_FLAG_NO_EVICT; + for (i = 0; i < bo->placement.num_placement; i++) { + bo->placements[i].lpfn = 0; + bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT; + } r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); - if (unlikely(r != 0)) + if (likely(r == 0)) { + if (bo->tbo.mem.mem_type == TTM_PL_VRAM) + bo->rdev->vram_pin_size -= radeon_bo_size(bo); + else + bo->rdev->gart_pin_size -= radeon_bo_size(bo); + } else { dev_err(bo->rdev->dev, "%p validate failed for unpin\n", bo); + } return r; } @@ -305,18 +432,14 @@ void radeon_bo_force_delete(struct radeon_device *rdev) } dev_err(rdev->dev, "Userspace still has active objects !\n"); list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) { - mutex_lock(&rdev->ddev->struct_mutex); -#ifdef notyet dev_err(rdev->dev, "%p %p %lu %lu force free\n", &bo->gem_base, bo, (unsigned long)bo->gem_base.size, *((unsigned long *)&bo->gem_base.refcount)); -#endif mutex_lock(&bo->rdev->gem.mutex); list_del_init(&bo->list); mutex_unlock(&bo->rdev->gem.mutex); /* this should unref the ttm bo */ - drm_gem_object_unreference(&bo->gem_base); - mutex_unlock(&rdev->ddev->struct_mutex); + drm_gem_object_unreference_unlocked(&bo->gem_base); } } @@ -325,80 +448,161 @@ int radeon_bo_init(struct radeon_device *rdev) paddr_t start, end; /* Add an MTRR for the VRAM */ - drm_mtrr_add(rdev->mc.aper_base, rdev->mc.aper_size, DRM_MTRR_WC); - /* fake a 'cookie', seems to be unused? */ - rdev->mc.vram_mtrr = 1; + if (!rdev->fastfb_working) { +#ifdef __linux__ + rdev->mc.vram_mtrr = arch_phys_wc_add(rdev->mc.aper_base, + rdev->mc.aper_size); +#else + drm_mtrr_add(rdev->mc.aper_base, rdev->mc.aper_size, DRM_MTRR_WC); + /* fake a 'cookie', seems to be unused? */ + rdev->mc.vram_mtrr = 1; +#endif + } start = atop(bus_space_mmap(rdev->memt, rdev->mc.aper_base, 0, 0, 0)); end = start + atop(rdev->mc.aper_size); uvm_page_physload(start, end, start, end, PHYSLOAD_DEVICE); -#ifdef DRMDEBUG DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", rdev->mc.mc_vram_size >> 20, (unsigned long long)rdev->mc.aper_size >> 20); DRM_INFO("RAM width %dbits %cDR\n", rdev->mc.vram_width, rdev->mc.vram_is_ddr ? 'D' : 'S'); -#endif return radeon_ttm_init(rdev); } void radeon_bo_fini(struct radeon_device *rdev) { radeon_ttm_fini(rdev); +#ifdef __linux__ + arch_phys_wc_del(rdev->mc.vram_mtrr); +#else + drm_mtrr_del(0, rdev->mc.aper_base, rdev->mc.aper_size, DRM_MTRR_WC); +#endif } -void radeon_bo_list_add_object(struct radeon_bo_list *lobj, - struct list_head *head) +/* Returns how many bytes TTM can move per IB. + */ +static u64 radeon_bo_get_threshold_for_moves(struct radeon_device *rdev) { - if (lobj->wdomain) { - list_add(&lobj->tv.head, head); - } else { - list_add_tail(&lobj->tv.head, head); - } + u64 real_vram_size = rdev->mc.real_vram_size; + u64 vram_usage = atomic64_read(&rdev->vram_usage); + + /* This function is based on the current VRAM usage. + * + * - If all of VRAM is free, allow relocating the number of bytes that + * is equal to 1/4 of the size of VRAM for this IB. + + * - If more than one half of VRAM is occupied, only allow relocating + * 1 MB of data for this IB. + * + * - From 0 to one half of used VRAM, the threshold decreases + * linearly. + * __________________ + * 1/4 of -|\ | + * VRAM | \ | + * | \ | + * | \ | + * | \ | + * | \ | + * | \ | + * | \________|1 MB + * |----------------| + * VRAM 0 % 100 % + * used used + * + * Note: It's a threshold, not a limit. The threshold must be crossed + * for buffer relocations to stop, so any buffer of an arbitrary size + * can be moved as long as the threshold isn't crossed before + * the relocation takes place. We don't want to disable buffer + * relocations completely. + * + * The idea is that buffers should be placed in VRAM at creation time + * and TTM should only do a minimum number of relocations during + * command submission. In practice, you need to submit at least + * a dozen IBs to move all buffers to VRAM if they are in GTT. + * + * Also, things can get pretty crazy under memory pressure and actual + * VRAM usage can change a lot, so playing safe even at 50% does + * consistently increase performance. + */ + + u64 half_vram = real_vram_size >> 1; + u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage; + u64 bytes_moved_threshold = half_free_vram >> 1; + return max(bytes_moved_threshold, 1024*1024ull); } -int radeon_bo_list_validate(struct list_head *head) +int radeon_bo_list_validate(struct radeon_device *rdev, + struct ww_acquire_ctx *ticket, + struct list_head *head, int ring) { struct radeon_bo_list *lobj; - struct radeon_bo *bo; - u32 domain; + struct list_head duplicates; int r; + u64 bytes_moved = 0, initial_bytes_moved; + u64 bytes_moved_threshold = radeon_bo_get_threshold_for_moves(rdev); - r = ttm_eu_reserve_buffers(head); + INIT_LIST_HEAD(&duplicates); + r = ttm_eu_reserve_buffers(ticket, head, true, &duplicates); if (unlikely(r != 0)) { return r; } + list_for_each_entry(lobj, head, tv.head) { - bo = lobj->bo; + struct radeon_bo *bo = lobj->robj; if (!bo->pin_count) { - domain = lobj->wdomain ? lobj->wdomain : lobj->rdomain; - + u32 domain = lobj->prefered_domains; + u32 allowed = lobj->allowed_domains; + u32 current_domain = + radeon_mem_type_to_domain(bo->tbo.mem.mem_type); + + /* Check if this buffer will be moved and don't move it + * if we have moved too many buffers for this IB already. + * + * Note that this allows moving at least one buffer of + * any size, because it doesn't take the current "bo" + * into account. We don't want to disallow buffer moves + * completely. + */ + if ((allowed & current_domain) != 0 && + (domain & current_domain) == 0 && /* will be moved */ + bytes_moved > bytes_moved_threshold) { + /* don't move it */ + domain = current_domain; + } + retry: radeon_ttm_placement_from_domain(bo, domain); - r = ttm_bo_validate(&bo->tbo, &bo->placement, - true, false); + if (ring == R600_RING_TYPE_UVD_INDEX) + radeon_uvd_force_into_uvd_segment(bo, allowed); + + initial_bytes_moved = atomic64_read(&rdev->num_bytes_moved); + r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); + bytes_moved += atomic64_read(&rdev->num_bytes_moved) - + initial_bytes_moved; + if (unlikely(r)) { - if (r != -ERESTARTSYS && domain == RADEON_GEM_DOMAIN_VRAM) { - domain |= RADEON_GEM_DOMAIN_GTT; + if (r != -ERESTARTSYS && + domain != lobj->allowed_domains) { + domain = lobj->allowed_domains; goto retry; } + ttm_eu_backoff_reservation(ticket, head); return r; } } lobj->gpu_offset = radeon_bo_gpu_offset(bo); lobj->tiling_flags = bo->tiling_flags; } - return 0; -} -#ifdef notyet -int radeon_bo_fbdev_mmap(struct radeon_bo *bo, - struct vm_area_struct *vma) -{ - return ttm_fbdev_mmap(vma, &bo->tbo); + list_for_each_entry(lobj, &duplicates, tv.head) { + lobj->gpu_offset = radeon_bo_gpu_offset(lobj->robj); + lobj->tiling_flags = lobj->robj->tiling_flags; + } + + return 0; } -#endif int radeon_bo_get_surface_reg(struct radeon_bo *bo) { @@ -408,7 +612,9 @@ int radeon_bo_get_surface_reg(struct radeon_bo *bo) int steal; int i; - BUG_ON(!radeon_bo_is_reserved(bo)); +#ifdef notyet + lockdep_assert_held(&bo->tbo.resv->lock.base); +#endif if (!bo->tiling_flags) return 0; @@ -534,7 +740,10 @@ void radeon_bo_get_tiling_flags(struct radeon_bo *bo, uint32_t *tiling_flags, uint32_t *pitch) { - BUG_ON(!radeon_bo_is_reserved(bo)); +#ifdef notyet + lockdep_assert_held(&bo->tbo.resv->lock.base); +#endif + if (tiling_flags) *tiling_flags = bo->tiling_flags; if (pitch) @@ -544,7 +753,10 @@ void radeon_bo_get_tiling_flags(struct radeon_bo *bo, int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved, bool force_drop) { - BUG_ON(!radeon_bo_is_reserved(bo) && !force_drop); +#ifdef notyet + if (!force_drop) + lockdep_assert_held(&bo->tbo.resv->lock.base); +#endif if (!(bo->tiling_flags & RADEON_TILING_SURFACE)) return 0; @@ -570,22 +782,31 @@ int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved, } void radeon_bo_move_notify(struct ttm_buffer_object *bo, - struct ttm_mem_reg *mem) + struct ttm_mem_reg *new_mem) { struct radeon_bo *rbo; + if (!radeon_ttm_bo_is_radeon_bo(bo)) return; + rbo = container_of(bo, struct radeon_bo, tbo); radeon_bo_check_tiling(rbo, 0, 1); radeon_vm_bo_invalidate(rbo->rdev, rbo); + + /* update statistics */ + if (!new_mem) + return; + + radeon_update_memory_usage(rbo, bo->mem.mem_type, -1); + radeon_update_memory_usage(rbo, new_mem->mem_type, 1); } int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo) { struct radeon_device *rdev; struct radeon_bo *rbo; - unsigned long offset, size; - int r; + unsigned long offset, size, lpfn; + int i, r; if (!radeon_ttm_bo_is_radeon_bo(bo)) return 0; @@ -602,7 +823,13 @@ int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo) /* hurrah the memory is not visible ! */ radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM); - rbo->placement.lpfn = rdev->mc.visible_vram_size >> PAGE_SHIFT; + lpfn = rdev->mc.visible_vram_size >> PAGE_SHIFT; + for (i = 0; i < rbo->placement.num_placement; i++) { + /* Force into visible VRAM */ + if ((rbo->placements[i].flags & TTM_PL_FLAG_VRAM) && + (!rbo->placements[i].lpfn || rbo->placements[i].lpfn > lpfn)) + rbo->placements[i].lpfn = lpfn; + } r = ttm_bo_validate(bo, &rbo->placement, false, false); if (unlikely(r == -ENOMEM)) { radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); @@ -623,38 +850,32 @@ int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait) { int r; - r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, 0); + r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, NULL); if (unlikely(r != 0)) return r; - spin_lock(&bo->tbo.bdev->fence_lock); if (mem_type) *mem_type = bo->tbo.mem.mem_type; - if (bo->tbo.sync_obj) - r = ttm_bo_wait(&bo->tbo, true, true, no_wait); - spin_unlock(&bo->tbo.bdev->fence_lock); + + r = ttm_bo_wait(&bo->tbo, true, true, no_wait); ttm_bo_unreserve(&bo->tbo); return r; } - /** - * radeon_bo_reserve - reserve bo - * @bo: bo structure - * @no_intr: don't return -ERESTARTSYS on pending signal + * radeon_bo_fence - add fence to buffer object + * + * @bo: buffer object in question + * @fence: fence to add + * @shared: true if fence should be added shared * - * Returns: - * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by - * a signal. Release all buffer reservations and return to user-space. */ -int radeon_bo_reserve(struct radeon_bo *bo, bool no_intr) +void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence, + bool shared) { - int r; + struct reservation_object *resv = bo->tbo.resv; - r = ttm_bo_reserve(&bo->tbo, !no_intr, false, false, 0); - if (unlikely(r != 0)) { - if (r != -ERESTARTSYS) - dev_err(bo->rdev->dev, "%p reserve failed\n", bo); - return r; - } - return 0; + if (shared) + reservation_object_add_shared_fence(resv, &fence->base); + else + reservation_object_add_excl_fence(resv, &fence->base); } diff --git a/sys/dev/pci/drm/radeon/radeon_object.h b/sys/dev/pci/drm/radeon/radeon_object.h index 827a3559b76..7b07e8f32ce 100644 --- a/sys/dev/pci/drm/radeon/radeon_object.h +++ b/sys/dev/pci/drm/radeon/radeon_object.h @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_object.h,v 1.5 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -53,7 +52,27 @@ static inline unsigned radeon_mem_type_to_domain(u32 mem_type) return 0; } -int radeon_bo_reserve(struct radeon_bo *bo, bool no_intr); +/** + * radeon_bo_reserve - reserve bo + * @bo: bo structure + * @no_intr: don't return -ERESTARTSYS on pending signal + * + * Returns: + * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by + * a signal. Release all buffer reservations and return to user-space. + */ +static inline int radeon_bo_reserve(struct radeon_bo *bo, bool no_intr) +{ + int r; + + r = ttm_bo_reserve(&bo->tbo, !no_intr, false, false, 0); + if (unlikely(r != 0)) { + if (r != -ERESTARTSYS) + dev_err(bo->rdev->dev, "%p reserve failed\n", bo); + return r; + } + return 0; +} static inline void radeon_bo_unreserve(struct radeon_bo *bo) { @@ -79,11 +98,6 @@ static inline unsigned long radeon_bo_size(struct radeon_bo *bo) return bo->tbo.num_pages << PAGE_SHIFT; } -static inline bool radeon_bo_is_reserved(struct radeon_bo *bo) -{ - return ttm_bo_is_reserved(&bo->tbo); -} - static inline unsigned radeon_bo_ngpu_pages(struct radeon_bo *bo) { return (bo->tbo.num_pages << PAGE_SHIFT) / RADEON_GPU_PAGE_SIZE; @@ -110,11 +124,13 @@ extern int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, extern int radeon_bo_create(struct radeon_device *rdev, unsigned long size, int byte_align, - bool kernel, u32 domain, + bool kernel, u32 domain, u32 flags, struct sg_table *sg, + struct reservation_object *resv, struct radeon_bo **bo_ptr); extern int radeon_bo_kmap(struct radeon_bo *bo, void **ptr); extern void radeon_bo_kunmap(struct radeon_bo *bo); +extern struct radeon_bo *radeon_bo_ref(struct radeon_bo *bo); extern void radeon_bo_unref(struct radeon_bo **bo); extern int radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr); extern int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, @@ -124,13 +140,9 @@ extern int radeon_bo_evict_vram(struct radeon_device *rdev); extern void radeon_bo_force_delete(struct radeon_device *rdev); extern int radeon_bo_init(struct radeon_device *rdev); extern void radeon_bo_fini(struct radeon_device *rdev); -extern void radeon_bo_list_add_object(struct radeon_bo_list *lobj, - struct list_head *head); -extern int radeon_bo_list_validate(struct list_head *head); -#ifdef notyet -extern int radeon_bo_fbdev_mmap(struct radeon_bo *bo, - struct vm_area_struct *vma); -#endif +extern int radeon_bo_list_validate(struct radeon_device *rdev, + struct ww_acquire_ctx *ticket, + struct list_head *head, int ring); extern int radeon_bo_set_tiling_flags(struct radeon_bo *bo, u32 tiling_flags, u32 pitch); extern void radeon_bo_get_tiling_flags(struct radeon_bo *bo, @@ -138,9 +150,11 @@ extern void radeon_bo_get_tiling_flags(struct radeon_bo *bo, extern int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved, bool force_drop); extern void radeon_bo_move_notify(struct ttm_buffer_object *bo, - struct ttm_mem_reg *mem); + struct ttm_mem_reg *new_mem); extern int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo); extern int radeon_bo_get_surface_reg(struct radeon_bo *bo); +extern void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence, + bool shared); /* * sub allocation @@ -158,7 +172,8 @@ static inline void * radeon_sa_bo_cpu_addr(struct radeon_sa_bo *sa_bo) extern int radeon_sa_bo_manager_init(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager, - unsigned size, u32 align, u32 domain); + unsigned size, u32 align, u32 domain, + u32 flags); extern void radeon_sa_bo_manager_fini(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager); extern int radeon_sa_bo_manager_start(struct radeon_device *rdev, @@ -168,7 +183,7 @@ extern int radeon_sa_bo_manager_suspend(struct radeon_device *rdev, extern int radeon_sa_bo_new(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager, struct radeon_sa_bo **sa_bo, - unsigned size, unsigned align, bool block); + unsigned size, unsigned align); extern void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **sa_bo, struct radeon_fence *fence); diff --git a/sys/dev/pci/drm/radeon/radeon_pm.c b/sys/dev/pci/drm/radeon/radeon_pm.c index 99f8756fa3f..08829521923 100644 --- a/sys/dev/pci/drm/radeon/radeon_pm.c +++ b/sys/dev/pci/drm/radeon/radeon_pm.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_pm.c,v 1.17 2018/04/20 16:09:37 deraadt Exp $ */ /* * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -25,6 +24,7 @@ #include "radeon.h" #include "avivod.h" #include "atom.h" +#include "r600_dpm.h" #define RADEON_IDLE_LOOP_MS 100 #define RADEON_RECLOCK_DELAY_MS 200 @@ -67,7 +67,18 @@ int radeon_pm_get_type_index(struct radeon_device *rdev, void radeon_pm_acpi_event_handler(struct radeon_device *rdev) { - if (rdev->pm.pm_method == PM_METHOD_PROFILE) { + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + mutex_lock(&rdev->pm.mutex); + if (power_supply_is_system_supplied() > 0) + rdev->pm.dpm.ac_power = true; + else + rdev->pm.dpm.ac_power = false; + if (rdev->family == CHIP_ARUBA) { + if (rdev->asic->dpm.enable_bapm) + radeon_dpm_enable_bapm(rdev, rdev->pm.dpm.ac_power); + } + mutex_unlock(&rdev->pm.mutex); + } else if (rdev->pm.pm_method == PM_METHOD_PROFILE) { if (rdev->pm.profile == PM_PROFILE_AUTO) { mutex_lock(&rdev->pm.mutex); radeon_pm_update_profile(rdev); @@ -146,7 +157,8 @@ static void radeon_sync_with_vblank(struct radeon_device *rdev) { if (rdev->pm.active_crtcs) { rdev->pm.vblank_sync = false; - tsleep(&rdev->irq.vblank_queue, PZERO, "rdnsvb", + wait_event_timeout( + rdev->irq.vblank_queue, rdev->pm.vblank_sync, msecs_to_jiffies(RADEON_WAIT_VBLANK_TIMEOUT)); } } @@ -240,7 +252,6 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev) (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index)) return; - mutex_lock(&rdev->ddev->struct_mutex); down_write(&rdev->pm.mclk_lock); mutex_lock(&rdev->ring_lock); @@ -250,12 +261,11 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev) if (!ring->ready) { continue; } - r = radeon_fence_wait_empty_locked(rdev, i); + r = radeon_fence_wait_empty(rdev, i); if (r) { /* needs a GPU reset dont reset here */ mutex_unlock(&rdev->ring_lock); up_write(&rdev->pm.mclk_lock); - mutex_unlock(&rdev->ddev->struct_mutex); return; } } @@ -291,7 +301,6 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev) mutex_unlock(&rdev->ring_lock); up_write(&rdev->pm.mclk_lock); - mutex_unlock(&rdev->ddev->struct_mutex); } static void radeon_pm_print_states(struct radeon_device *rdev) @@ -333,7 +342,7 @@ static ssize_t radeon_get_pm_profile(struct device *dev, struct device_attribute *attr, char *buf) { - struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev)); + struct drm_device *ddev = dev_get_drvdata(dev); struct radeon_device *rdev = ddev->dev_private; int cp = rdev->pm.profile; @@ -349,9 +358,16 @@ static ssize_t radeon_set_pm_profile(struct device *dev, const char *buf, size_t count) { - struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev)); + struct drm_device *ddev = dev_get_drvdata(dev); struct radeon_device *rdev = ddev->dev_private; + /* Can't set profile when the card is off */ +#ifdef notyet + if ((rdev->flags & RADEON_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; +#endif + mutex_lock(&rdev->pm.mutex); if (rdev->pm.pm_method == PM_METHOD_PROFILE) { if (strncmp("default", buf, strlen("default")) == 0) @@ -383,12 +399,13 @@ static ssize_t radeon_get_pm_method(struct device *dev, struct device_attribute *attr, char *buf) { - struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev)); + struct drm_device *ddev = dev_get_drvdata(dev); struct radeon_device *rdev = ddev->dev_private; int pm = rdev->pm.pm_method; return snprintf(buf, PAGE_SIZE, "%s\n", - (pm == PM_METHOD_DYNPM) ? "dynpm" : "profile"); + (pm == PM_METHOD_DYNPM) ? "dynpm" : + (pm == PM_METHOD_PROFILE) ? "profile" : "dpm"); } static ssize_t radeon_set_pm_method(struct device *dev, @@ -396,9 +413,23 @@ static ssize_t radeon_set_pm_method(struct device *dev, const char *buf, size_t count) { - struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev)); + struct drm_device *ddev = dev_get_drvdata(dev); struct radeon_device *rdev = ddev->dev_private; +#ifdef notyet + /* Can't set method when the card is off */ + if ((rdev->flags & RADEON_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) { + count = -EINVAL; + goto fail; + } +#endif + + /* we don't support the legacy modes with dpm */ + if (rdev->pm.pm_method == PM_METHOD_DPM) { + count = -EINVAL; + goto fail; + } if (strncmp("dynpm", buf, strlen("dynpm")) == 0) { mutex_lock(&rdev->pm.mutex); @@ -423,62 +454,330 @@ fail: return count; } +static ssize_t radeon_get_dpm_state(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct radeon_device *rdev = ddev->dev_private; + enum radeon_pm_state_type pm = rdev->pm.dpm.user_state; + + return snprintf(buf, PAGE_SIZE, "%s\n", + (pm == POWER_STATE_TYPE_BATTERY) ? "battery" : + (pm == POWER_STATE_TYPE_BALANCED) ? "balanced" : "performance"); +} + +static ssize_t radeon_set_dpm_state(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct radeon_device *rdev = ddev->dev_private; + + mutex_lock(&rdev->pm.mutex); + if (strncmp("battery", buf, strlen("battery")) == 0) + rdev->pm.dpm.user_state = POWER_STATE_TYPE_BATTERY; + else if (strncmp("balanced", buf, strlen("balanced")) == 0) + rdev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED; + else if (strncmp("performance", buf, strlen("performance")) == 0) + rdev->pm.dpm.user_state = POWER_STATE_TYPE_PERFORMANCE; + else { + mutex_unlock(&rdev->pm.mutex); + count = -EINVAL; + goto fail; + } + mutex_unlock(&rdev->pm.mutex); + + /* Can't set dpm state when the card is off */ +#ifdef notyet + if (!(rdev->flags & RADEON_IS_PX) || + (ddev->switch_power_state == DRM_SWITCH_POWER_ON)) +#endif + radeon_pm_compute_clocks(rdev); + +fail: + return count; +} + +static ssize_t radeon_get_dpm_forced_performance_level(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct radeon_device *rdev = ddev->dev_private; + enum radeon_dpm_forced_level level = rdev->pm.dpm.forced_level; + +#ifdef notyet + if ((rdev->flags & RADEON_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return snprintf(buf, PAGE_SIZE, "off\n"); +#endif + + return snprintf(buf, PAGE_SIZE, "%s\n", + (level == RADEON_DPM_FORCED_LEVEL_AUTO) ? "auto" : + (level == RADEON_DPM_FORCED_LEVEL_LOW) ? "low" : "high"); +} + +static ssize_t radeon_set_dpm_forced_performance_level(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct radeon_device *rdev = ddev->dev_private; + enum radeon_dpm_forced_level level; + int ret = 0; + + /* Can't force performance level when the card is off */ +#ifdef notyet + if ((rdev->flags & RADEON_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; +#endif + + mutex_lock(&rdev->pm.mutex); + if (strncmp("low", buf, strlen("low")) == 0) { + level = RADEON_DPM_FORCED_LEVEL_LOW; + } else if (strncmp("high", buf, strlen("high")) == 0) { + level = RADEON_DPM_FORCED_LEVEL_HIGH; + } else if (strncmp("auto", buf, strlen("auto")) == 0) { + level = RADEON_DPM_FORCED_LEVEL_AUTO; + } else { + count = -EINVAL; + goto fail; + } + if (rdev->asic->dpm.force_performance_level) { + if (rdev->pm.dpm.thermal_active) { + count = -EINVAL; + goto fail; + } + ret = radeon_dpm_force_performance_level(rdev, level); + if (ret) + count = -EINVAL; + } +fail: + mutex_unlock(&rdev->pm.mutex); + + return count; +} +#endif + +#ifdef notyet +static ssize_t radeon_hwmon_get_pwm1_enable(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct radeon_device *rdev = dev_get_drvdata(dev); + u32 pwm_mode = 0; + + if (rdev->asic->dpm.fan_ctrl_get_mode) + pwm_mode = rdev->asic->dpm.fan_ctrl_get_mode(rdev); + + /* never 0 (full-speed), fuse or smc-controlled always */ + return sprintf(buf, "%i\n", pwm_mode == FDO_PWM_MODE_STATIC ? 1 : 2); +} + +static ssize_t radeon_hwmon_set_pwm1_enable(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct radeon_device *rdev = dev_get_drvdata(dev); + int err; + int value; + + if(!rdev->asic->dpm.fan_ctrl_set_mode) + return -EINVAL; + + err = kstrtoint(buf, 10, &value); + if (err) + return err; + + switch (value) { + case 1: /* manual, percent-based */ + rdev->asic->dpm.fan_ctrl_set_mode(rdev, FDO_PWM_MODE_STATIC); + break; + default: /* disable */ + rdev->asic->dpm.fan_ctrl_set_mode(rdev, 0); + break; + } + + return count; +} + +static ssize_t radeon_hwmon_get_pwm1_min(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%i\n", 0); +} + +static ssize_t radeon_hwmon_get_pwm1_max(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%i\n", 255); +} + +static ssize_t radeon_hwmon_set_pwm1(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct radeon_device *rdev = dev_get_drvdata(dev); + int err; + u32 value; + + err = kstrtou32(buf, 10, &value); + if (err) + return err; + + value = (value * 100) / 255; + + err = rdev->asic->dpm.set_fan_speed_percent(rdev, value); + if (err) + return err; + + return count; +} + +static ssize_t radeon_hwmon_get_pwm1(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct radeon_device *rdev = dev_get_drvdata(dev); + int err; + u32 speed; + + err = rdev->asic->dpm.get_fan_speed_percent(rdev, &speed); + if (err) + return err; + + speed = (speed * 255) / 100; + + return sprintf(buf, "%i\n", speed); +} + static DEVICE_ATTR(power_profile, S_IRUGO | S_IWUSR, radeon_get_pm_profile, radeon_set_pm_profile); static DEVICE_ATTR(power_method, S_IRUGO | S_IWUSR, radeon_get_pm_method, radeon_set_pm_method); -#endif /* notyet */ +static DEVICE_ATTR(power_dpm_state, S_IRUGO | S_IWUSR, radeon_get_dpm_state, radeon_set_dpm_state); +static DEVICE_ATTR(power_dpm_force_performance_level, S_IRUGO | S_IWUSR, + radeon_get_dpm_forced_performance_level, + radeon_set_dpm_forced_performance_level); -#ifdef notyet static ssize_t radeon_hwmon_show_temp(struct device *dev, struct device_attribute *attr, char *buf) { - struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev)); - struct radeon_device *rdev = ddev->dev_private; + struct radeon_device *rdev = dev_get_drvdata(dev); + struct drm_device *ddev = rdev->ddev; int temp; - switch (rdev->pm.int_thermal_type) { - case THERMAL_TYPE_RV6XX: - temp = rv6xx_get_temp(rdev); - break; - case THERMAL_TYPE_RV770: - temp = rv770_get_temp(rdev); - break; - case THERMAL_TYPE_EVERGREEN: - case THERMAL_TYPE_NI: - temp = evergreen_get_temp(rdev); - break; - case THERMAL_TYPE_SUMO: - temp = sumo_get_temp(rdev); - break; - case THERMAL_TYPE_SI: - temp = si_get_temp(rdev); - break; - default: + /* Can't get temperature when the card is off */ + if ((rdev->flags & RADEON_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + if (rdev->asic->pm.get_temperature) + temp = radeon_get_temperature(rdev); + else temp = 0; - break; - } return snprintf(buf, PAGE_SIZE, "%d\n", temp); } -static ssize_t radeon_hwmon_show_name(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t radeon_hwmon_show_temp_thresh(struct device *dev, + struct device_attribute *attr, + char *buf) { - return sprintf(buf, "radeon\n"); + struct radeon_device *rdev = dev_get_drvdata(dev); + int hyst = to_sensor_dev_attr(attr)->index; + int temp; + + if (hyst) + temp = rdev->pm.dpm.thermal.min_temp; + else + temp = rdev->pm.dpm.thermal.max_temp; + + return snprintf(buf, PAGE_SIZE, "%d\n", temp); } static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, radeon_hwmon_show_temp, NULL, 0); -static SENSOR_DEVICE_ATTR(name, S_IRUGO, radeon_hwmon_show_name, NULL, 0); +static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO, radeon_hwmon_show_temp_thresh, NULL, 0); +static SENSOR_DEVICE_ATTR(temp1_crit_hyst, S_IRUGO, radeon_hwmon_show_temp_thresh, NULL, 1); +static SENSOR_DEVICE_ATTR(pwm1, S_IRUGO | S_IWUSR, radeon_hwmon_get_pwm1, radeon_hwmon_set_pwm1, 0); +static SENSOR_DEVICE_ATTR(pwm1_enable, S_IRUGO | S_IWUSR, radeon_hwmon_get_pwm1_enable, radeon_hwmon_set_pwm1_enable, 0); +static SENSOR_DEVICE_ATTR(pwm1_min, S_IRUGO, radeon_hwmon_get_pwm1_min, NULL, 0); +static SENSOR_DEVICE_ATTR(pwm1_max, S_IRUGO, radeon_hwmon_get_pwm1_max, NULL, 0); + static struct attribute *hwmon_attributes[] = { &sensor_dev_attr_temp1_input.dev_attr.attr, - &sensor_dev_attr_name.dev_attr.attr, + &sensor_dev_attr_temp1_crit.dev_attr.attr, + &sensor_dev_attr_temp1_crit_hyst.dev_attr.attr, + &sensor_dev_attr_pwm1.dev_attr.attr, + &sensor_dev_attr_pwm1_enable.dev_attr.attr, + &sensor_dev_attr_pwm1_min.dev_attr.attr, + &sensor_dev_attr_pwm1_max.dev_attr.attr, NULL }; +static umode_t hwmon_attributes_visible(struct kobject *kobj, + struct attribute *attr, int index) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct radeon_device *rdev = dev_get_drvdata(dev); + umode_t effective_mode = attr->mode; + + /* Skip attributes if DPM is not enabled */ + if (rdev->pm.pm_method != PM_METHOD_DPM && + (attr == &sensor_dev_attr_temp1_crit.dev_attr.attr || + attr == &sensor_dev_attr_temp1_crit_hyst.dev_attr.attr || + attr == &sensor_dev_attr_pwm1.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_max.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_min.dev_attr.attr)) + return 0; + + /* Skip fan attributes if fan is not present */ + if (rdev->pm.no_fan && + (attr == &sensor_dev_attr_pwm1.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_max.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_min.dev_attr.attr)) + return 0; + + /* mask fan attributes if we have no bindings for this asic to expose */ + if ((!rdev->asic->dpm.get_fan_speed_percent && + attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't query fan */ + (!rdev->asic->dpm.fan_ctrl_get_mode && + attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't query state */ + effective_mode &= ~S_IRUGO; + + if ((!rdev->asic->dpm.set_fan_speed_percent && + attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't manage fan */ + (!rdev->asic->dpm.fan_ctrl_set_mode && + attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't manage state */ + effective_mode &= ~S_IWUSR; + + /* hide max/min values if we can't both query and manage the fan */ + if ((!rdev->asic->dpm.set_fan_speed_percent && + !rdev->asic->dpm.get_fan_speed_percent) && + (attr == &sensor_dev_attr_pwm1_max.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_min.dev_attr.attr)) + return 0; + + return effective_mode; +} + static const struct attribute_group hwmon_attrgroup = { .attrs = hwmon_attributes, + .is_visible = hwmon_attributes_visible, +}; + +static const struct attribute_group *hwmon_groups[] = { + &hwmon_attrgroup, + NULL }; #endif @@ -486,8 +785,6 @@ static int radeon_hwmon_init(struct radeon_device *rdev) { int err = 0; - rdev->pm.int_hwmon_dev = NULL; - switch (rdev->pm.int_thermal_type) { case THERMAL_TYPE_RV6XX: case THERMAL_TYPE_RV770: @@ -495,27 +792,21 @@ static int radeon_hwmon_init(struct radeon_device *rdev) case THERMAL_TYPE_NI: case THERMAL_TYPE_SUMO: case THERMAL_TYPE_SI: - /* No support for TN yet */ - if (rdev->family == CHIP_ARUBA) + case THERMAL_TYPE_CI: + case THERMAL_TYPE_KV: + if (rdev->asic->pm.get_temperature == NULL) return err; #ifdef notyet - rdev->pm.int_hwmon_dev = hwmon_device_register(rdev->dev); + rdev->pm.int_hwmon_dev = hwmon_device_register_with_groups(rdev->dev, + "radeon", rdev, + hwmon_groups); if (IS_ERR(rdev->pm.int_hwmon_dev)) { err = PTR_ERR(rdev->pm.int_hwmon_dev); dev_err(rdev->dev, "Unable to register hwmon device: %d\n", err); - break; } - dev_set_drvdata(rdev->pm.int_hwmon_dev, rdev->ddev); - err = sysfs_create_group(&rdev->pm.int_hwmon_dev->kobj, - &hwmon_attrgroup); - if (err) { - dev_err(rdev->dev, - "Unable to create hwmon sysfs file: %d\n", err); - hwmon_device_unregister(rdev->dev); - } - break; #endif + break; default: break; } @@ -525,16 +816,382 @@ static int radeon_hwmon_init(struct radeon_device *rdev) static void radeon_hwmon_fini(struct radeon_device *rdev) { - printf("%s stub\n", __func__); #ifdef notyet - if (rdev->pm.int_hwmon_dev) { - sysfs_remove_group(&rdev->pm.int_hwmon_dev->kobj, &hwmon_attrgroup); + if (rdev->pm.int_hwmon_dev) hwmon_device_unregister(rdev->pm.int_hwmon_dev); +#endif +} + +static void radeon_dpm_thermal_work_handler(struct work_struct *work) +{ + struct radeon_device *rdev = + container_of(work, struct radeon_device, + pm.dpm.thermal.work); + /* switch to the thermal state */ + enum radeon_pm_state_type dpm_state = POWER_STATE_TYPE_INTERNAL_THERMAL; + + if (!rdev->pm.dpm_enabled) + return; + + if (rdev->asic->pm.get_temperature) { + int temp = radeon_get_temperature(rdev); + + if (temp < rdev->pm.dpm.thermal.min_temp) + /* switch back the user state */ + dpm_state = rdev->pm.dpm.user_state; + } else { + if (rdev->pm.dpm.thermal.high_to_low) + /* switch back the user state */ + dpm_state = rdev->pm.dpm.user_state; + } + mutex_lock(&rdev->pm.mutex); + if (dpm_state == POWER_STATE_TYPE_INTERNAL_THERMAL) + rdev->pm.dpm.thermal_active = true; + else + rdev->pm.dpm.thermal_active = false; + rdev->pm.dpm.state = dpm_state; + mutex_unlock(&rdev->pm.mutex); + + radeon_pm_compute_clocks(rdev); +} + +static bool radeon_dpm_single_display(struct radeon_device *rdev) +{ + bool single_display = (rdev->pm.dpm.new_active_crtc_count < 2) ? + true : false; + + /* check if the vblank period is too short to adjust the mclk */ + if (single_display && rdev->asic->dpm.vblank_too_short) { + if (radeon_dpm_vblank_too_short(rdev)) + single_display = false; + } + + /* 120hz tends to be problematic even if they are under the + * vblank limit. + */ + if (single_display && (r600_dpm_get_vrefresh(rdev) >= 120)) + single_display = false; + + return single_display; +} + +static struct radeon_ps *radeon_dpm_pick_power_state(struct radeon_device *rdev, + enum radeon_pm_state_type dpm_state) +{ + int i; + struct radeon_ps *ps; + u32 ui_class; + bool single_display = radeon_dpm_single_display(rdev); + + /* certain older asics have a separare 3D performance state, + * so try that first if the user selected performance + */ + if (dpm_state == POWER_STATE_TYPE_PERFORMANCE) + dpm_state = POWER_STATE_TYPE_INTERNAL_3DPERF; + /* balanced states don't exist at the moment */ + if (dpm_state == POWER_STATE_TYPE_BALANCED) + dpm_state = POWER_STATE_TYPE_PERFORMANCE; + +restart_search: + /* Pick the best power state based on current conditions */ + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + ps = &rdev->pm.dpm.ps[i]; + ui_class = ps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK; + switch (dpm_state) { + /* user states */ + case POWER_STATE_TYPE_BATTERY: + if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) { + if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { + if (single_display) + return ps; + } else + return ps; + } + break; + case POWER_STATE_TYPE_BALANCED: + if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BALANCED) { + if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { + if (single_display) + return ps; + } else + return ps; + } + break; + case POWER_STATE_TYPE_PERFORMANCE: + if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) { + if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { + if (single_display) + return ps; + } else + return ps; + } + break; + /* internal states */ + case POWER_STATE_TYPE_INTERNAL_UVD: + if (rdev->pm.dpm.uvd_ps) + return rdev->pm.dpm.uvd_ps; + else + break; + case POWER_STATE_TYPE_INTERNAL_UVD_SD: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_UVD_HD: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_UVD_HD2: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_HD2STATE) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_UVD_MVC: + if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_MVC) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_BOOT: + return rdev->pm.dpm.boot_ps; + case POWER_STATE_TYPE_INTERNAL_THERMAL: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_ACPI: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_ULV: + if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_3DPERF: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE) + return ps; + break; + default: + break; + } + } + /* use a fallback state if we didn't match */ + switch (dpm_state) { + case POWER_STATE_TYPE_INTERNAL_UVD_SD: + dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_HD; + goto restart_search; + case POWER_STATE_TYPE_INTERNAL_UVD_HD: + case POWER_STATE_TYPE_INTERNAL_UVD_HD2: + case POWER_STATE_TYPE_INTERNAL_UVD_MVC: + if (rdev->pm.dpm.uvd_ps) { + return rdev->pm.dpm.uvd_ps; + } else { + dpm_state = POWER_STATE_TYPE_PERFORMANCE; + goto restart_search; + } + case POWER_STATE_TYPE_INTERNAL_THERMAL: + dpm_state = POWER_STATE_TYPE_INTERNAL_ACPI; + goto restart_search; + case POWER_STATE_TYPE_INTERNAL_ACPI: + dpm_state = POWER_STATE_TYPE_BATTERY; + goto restart_search; + case POWER_STATE_TYPE_BATTERY: + case POWER_STATE_TYPE_BALANCED: + case POWER_STATE_TYPE_INTERNAL_3DPERF: + dpm_state = POWER_STATE_TYPE_PERFORMANCE; + goto restart_search; + default: + break; + } + + return NULL; +} + +static void radeon_dpm_change_power_state_locked(struct radeon_device *rdev) +{ + int i; + struct radeon_ps *ps; + enum radeon_pm_state_type dpm_state; + int ret; + bool single_display = radeon_dpm_single_display(rdev); + + /* if dpm init failed */ + if (!rdev->pm.dpm_enabled) + return; + + if (rdev->pm.dpm.user_state != rdev->pm.dpm.state) { + /* add other state override checks here */ + if ((!rdev->pm.dpm.thermal_active) && + (!rdev->pm.dpm.uvd_active)) + rdev->pm.dpm.state = rdev->pm.dpm.user_state; + } + dpm_state = rdev->pm.dpm.state; + + ps = radeon_dpm_pick_power_state(rdev, dpm_state); + if (ps) + rdev->pm.dpm.requested_ps = ps; + else + return; + + /* no need to reprogram if nothing changed unless we are on BTC+ */ + if (rdev->pm.dpm.current_ps == rdev->pm.dpm.requested_ps) { + /* vce just modifies an existing state so force a change */ + if (ps->vce_active != rdev->pm.dpm.vce_active) + goto force; + /* user has made a display change (such as timing) */ + if (rdev->pm.dpm.single_display != single_display) + goto force; + if ((rdev->family < CHIP_BARTS) || (rdev->flags & RADEON_IS_IGP)) { + /* for pre-BTC and APUs if the num crtcs changed but state is the same, + * all we need to do is update the display configuration. + */ + if (rdev->pm.dpm.new_active_crtcs != rdev->pm.dpm.current_active_crtcs) { + /* update display watermarks based on new power state */ + radeon_bandwidth_update(rdev); + /* update displays */ + radeon_dpm_display_configuration_changed(rdev); + rdev->pm.dpm.current_active_crtcs = rdev->pm.dpm.new_active_crtcs; + rdev->pm.dpm.current_active_crtc_count = rdev->pm.dpm.new_active_crtc_count; + } + return; + } else { + /* for BTC+ if the num crtcs hasn't changed and state is the same, + * nothing to do, if the num crtcs is > 1 and state is the same, + * update display configuration. + */ + if (rdev->pm.dpm.new_active_crtcs == + rdev->pm.dpm.current_active_crtcs) { + return; + } else { + if ((rdev->pm.dpm.current_active_crtc_count > 1) && + (rdev->pm.dpm.new_active_crtc_count > 1)) { + /* update display watermarks based on new power state */ + radeon_bandwidth_update(rdev); + /* update displays */ + radeon_dpm_display_configuration_changed(rdev); + rdev->pm.dpm.current_active_crtcs = rdev->pm.dpm.new_active_crtcs; + rdev->pm.dpm.current_active_crtc_count = rdev->pm.dpm.new_active_crtc_count; + return; + } + } + } + } + +force: + if (radeon_dpm == 1) { + printk("switching from power state:\n"); + radeon_dpm_print_power_state(rdev, rdev->pm.dpm.current_ps); + printk("switching to power state:\n"); + radeon_dpm_print_power_state(rdev, rdev->pm.dpm.requested_ps); + } + + down_write(&rdev->pm.mclk_lock); + mutex_lock(&rdev->ring_lock); + + /* update whether vce is active */ + ps->vce_active = rdev->pm.dpm.vce_active; + + ret = radeon_dpm_pre_set_power_state(rdev); + if (ret) + goto done; + + /* update display watermarks based on new power state */ + radeon_bandwidth_update(rdev); + /* update displays */ + radeon_dpm_display_configuration_changed(rdev); + + /* wait for the rings to drain */ + for (i = 0; i < RADEON_NUM_RINGS; i++) { + struct radeon_ring *ring = &rdev->ring[i]; + if (ring->ready) + radeon_fence_wait_empty(rdev, i); + } + + /* program the new power state */ + radeon_dpm_set_power_state(rdev); + + /* update current power state */ + rdev->pm.dpm.current_ps = rdev->pm.dpm.requested_ps; + + radeon_dpm_post_set_power_state(rdev); + + rdev->pm.dpm.current_active_crtcs = rdev->pm.dpm.new_active_crtcs; + rdev->pm.dpm.current_active_crtc_count = rdev->pm.dpm.new_active_crtc_count; + rdev->pm.dpm.single_display = single_display; + + if (rdev->asic->dpm.force_performance_level) { + if (rdev->pm.dpm.thermal_active) { + enum radeon_dpm_forced_level level = rdev->pm.dpm.forced_level; + /* force low perf level for thermal */ + radeon_dpm_force_performance_level(rdev, RADEON_DPM_FORCED_LEVEL_LOW); + /* save the user's level */ + rdev->pm.dpm.forced_level = level; + } else { + /* otherwise, user selected level */ + radeon_dpm_force_performance_level(rdev, rdev->pm.dpm.forced_level); + } } + +done: + mutex_unlock(&rdev->ring_lock); + up_write(&rdev->pm.mclk_lock); +} + +void radeon_dpm_enable_uvd(struct radeon_device *rdev, bool enable) +{ + enum radeon_pm_state_type dpm_state; + + if (rdev->asic->dpm.powergate_uvd) { + mutex_lock(&rdev->pm.mutex); + /* don't powergate anything if we + have active but pause streams */ + enable |= rdev->pm.dpm.sd > 0; + enable |= rdev->pm.dpm.hd > 0; + /* enable/disable UVD */ + radeon_dpm_powergate_uvd(rdev, !enable); + mutex_unlock(&rdev->pm.mutex); + } else { + if (enable) { + mutex_lock(&rdev->pm.mutex); + rdev->pm.dpm.uvd_active = true; + /* disable this for now */ +#if 0 + if ((rdev->pm.dpm.sd == 1) && (rdev->pm.dpm.hd == 0)) + dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_SD; + else if ((rdev->pm.dpm.sd == 2) && (rdev->pm.dpm.hd == 0)) + dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_HD; + else if ((rdev->pm.dpm.sd == 0) && (rdev->pm.dpm.hd == 1)) + dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_HD; + else if ((rdev->pm.dpm.sd == 0) && (rdev->pm.dpm.hd == 2)) + dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_HD2; + else #endif + dpm_state = POWER_STATE_TYPE_INTERNAL_UVD; + rdev->pm.dpm.state = dpm_state; + mutex_unlock(&rdev->pm.mutex); + } else { + mutex_lock(&rdev->pm.mutex); + rdev->pm.dpm.uvd_active = false; + mutex_unlock(&rdev->pm.mutex); + } + + radeon_pm_compute_clocks(rdev); + } } -void radeon_pm_suspend(struct radeon_device *rdev) +void radeon_dpm_enable_vce(struct radeon_device *rdev, bool enable) +{ + if (enable) { + mutex_lock(&rdev->pm.mutex); + rdev->pm.dpm.vce_active = true; + /* XXX select vce level based on ring/task */ + rdev->pm.dpm.vce_level = RADEON_VCE_LEVEL_AC_ALL; + mutex_unlock(&rdev->pm.mutex); + } else { + mutex_lock(&rdev->pm.mutex); + rdev->pm.dpm.vce_active = false; + mutex_unlock(&rdev->pm.mutex); + } + + radeon_pm_compute_clocks(rdev); +} + +static void radeon_pm_suspend_old(struct radeon_device *rdev) { mutex_lock(&rdev->pm.mutex); if (rdev->pm.pm_method == PM_METHOD_DYNPM) { @@ -546,7 +1203,26 @@ void radeon_pm_suspend(struct radeon_device *rdev) cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work); } -void radeon_pm_resume(struct radeon_device *rdev) +static void radeon_pm_suspend_dpm(struct radeon_device *rdev) +{ + mutex_lock(&rdev->pm.mutex); + /* disable dpm */ + radeon_dpm_disable(rdev); + /* reset the power state */ + rdev->pm.dpm.current_ps = rdev->pm.dpm.requested_ps = rdev->pm.dpm.boot_ps; + rdev->pm.dpm_enabled = false; + mutex_unlock(&rdev->pm.mutex); +} + +void radeon_pm_suspend(struct radeon_device *rdev) +{ + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_suspend_dpm(rdev); + else + radeon_pm_suspend_old(rdev); +} + +static void radeon_pm_resume_old(struct radeon_device *rdev) { /* set up the default clocks if the MC ucode is loaded */ if ((rdev->family >= CHIP_BARTS) && @@ -583,12 +1259,51 @@ void radeon_pm_resume(struct radeon_device *rdev) radeon_pm_compute_clocks(rdev); } -int radeon_pm_init(struct radeon_device *rdev) +static void radeon_pm_resume_dpm(struct radeon_device *rdev) +{ + int ret; + + /* asic init will reset to the boot state */ + mutex_lock(&rdev->pm.mutex); + rdev->pm.dpm.current_ps = rdev->pm.dpm.requested_ps = rdev->pm.dpm.boot_ps; + radeon_dpm_setup_asic(rdev); + ret = radeon_dpm_enable(rdev); + mutex_unlock(&rdev->pm.mutex); + if (ret) + goto dpm_resume_fail; + rdev->pm.dpm_enabled = true; + return; + +dpm_resume_fail: + DRM_ERROR("radeon: dpm resume failed\n"); + if ((rdev->family >= CHIP_BARTS) && + (rdev->family <= CHIP_CAYMAN) && + rdev->mc_fw) { + if (rdev->pm.default_vddc) + radeon_atom_set_voltage(rdev, rdev->pm.default_vddc, + SET_VOLTAGE_TYPE_ASIC_VDDC); + if (rdev->pm.default_vddci) + radeon_atom_set_voltage(rdev, rdev->pm.default_vddci, + SET_VOLTAGE_TYPE_ASIC_VDDCI); + if (rdev->pm.default_sclk) + radeon_set_engine_clock(rdev, rdev->pm.default_sclk); + if (rdev->pm.default_mclk) + radeon_set_memory_clock(rdev, rdev->pm.default_mclk); + } +} + +void radeon_pm_resume(struct radeon_device *rdev) +{ + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_resume_dpm(rdev); + else + radeon_pm_resume_old(rdev); +} + +static int radeon_pm_init_old(struct radeon_device *rdev) { int ret; - /* default to profile method */ - rdev->pm.pm_method = PM_METHOD_PROFILE; rdev->pm.profile = PM_PROFILE_DEFAULT; rdev->pm.dynpm_state = DYNPM_STATE_DISABLED; rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE; @@ -632,28 +1347,256 @@ int radeon_pm_init(struct radeon_device *rdev) INIT_DELAYED_WORK(&rdev->pm.dynpm_idle_work, radeon_dynpm_idle_work_handler); if (rdev->pm.num_power_states > 1) { -#ifdef notyet - /* where's the best place to put these? */ - ret = device_create_file(rdev->dev, &dev_attr_power_profile); - if (ret) - DRM_ERROR("failed to create device file for power profile\n"); - ret = device_create_file(rdev->dev, &dev_attr_power_method); - if (ret) - DRM_ERROR("failed to create device file for power method\n"); -#endif if (radeon_debugfs_pm_init(rdev)) { DRM_ERROR("Failed to register debugfs file for PM!\n"); } -#ifdef DRMDEBUG DRM_INFO("radeon: power management initialized\n"); -#endif } return 0; } -void radeon_pm_fini(struct radeon_device *rdev) +static void radeon_dpm_print_power_states(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + printk("== power state %d ==\n", i); + radeon_dpm_print_power_state(rdev, &rdev->pm.dpm.ps[i]); + } +} + +static int radeon_pm_init_dpm(struct radeon_device *rdev) +{ + int ret; + + /* default to balanced state */ + rdev->pm.dpm.state = POWER_STATE_TYPE_BALANCED; + rdev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED; + rdev->pm.dpm.forced_level = RADEON_DPM_FORCED_LEVEL_AUTO; + rdev->pm.default_sclk = rdev->clock.default_sclk; + rdev->pm.default_mclk = rdev->clock.default_mclk; + rdev->pm.current_sclk = rdev->clock.default_sclk; + rdev->pm.current_mclk = rdev->clock.default_mclk; + rdev->pm.int_thermal_type = THERMAL_TYPE_NONE; + + if (rdev->bios && rdev->is_atom_bios) + radeon_atombios_get_power_modes(rdev); + else + return -EINVAL; + + /* set up the internal thermal sensor if applicable */ + ret = radeon_hwmon_init(rdev); + if (ret) + return ret; + + INIT_WORK(&rdev->pm.dpm.thermal.work, radeon_dpm_thermal_work_handler); + mutex_lock(&rdev->pm.mutex); + radeon_dpm_init(rdev); + rdev->pm.dpm.current_ps = rdev->pm.dpm.requested_ps = rdev->pm.dpm.boot_ps; + if (radeon_dpm == 1) + radeon_dpm_print_power_states(rdev); + radeon_dpm_setup_asic(rdev); + ret = radeon_dpm_enable(rdev); + mutex_unlock(&rdev->pm.mutex); + if (ret) + goto dpm_failed; + rdev->pm.dpm_enabled = true; + + if (radeon_debugfs_pm_init(rdev)) { + DRM_ERROR("Failed to register debugfs file for dpm!\n"); + } + + DRM_INFO("radeon: dpm initialized\n"); + + return 0; + +dpm_failed: + rdev->pm.dpm_enabled = false; + if ((rdev->family >= CHIP_BARTS) && + (rdev->family <= CHIP_CAYMAN) && + rdev->mc_fw) { + if (rdev->pm.default_vddc) + radeon_atom_set_voltage(rdev, rdev->pm.default_vddc, + SET_VOLTAGE_TYPE_ASIC_VDDC); + if (rdev->pm.default_vddci) + radeon_atom_set_voltage(rdev, rdev->pm.default_vddci, + SET_VOLTAGE_TYPE_ASIC_VDDCI); + if (rdev->pm.default_sclk) + radeon_set_engine_clock(rdev, rdev->pm.default_sclk); + if (rdev->pm.default_mclk) + radeon_set_memory_clock(rdev, rdev->pm.default_mclk); + } + DRM_ERROR("radeon: dpm initialization failed\n"); + return ret; +} + +struct radeon_dpm_quirk { + u32 chip_vendor; + u32 chip_device; + u32 subsys_vendor; + u32 subsys_device; +}; + +/* cards with dpm stability problems */ +static struct radeon_dpm_quirk radeon_dpm_quirk_list[] = { + /* TURKS - https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1386534 */ + { PCI_VENDOR_ID_ATI, 0x6759, 0x1682, 0x3195 }, + /* TURKS - https://bugzilla.kernel.org/show_bug.cgi?id=83731 */ + { PCI_VENDOR_ID_ATI, 0x6840, 0x1179, 0xfb81 }, + { 0, 0, 0, 0 }, +}; + +int radeon_pm_init(struct radeon_device *rdev) +{ + struct radeon_dpm_quirk *p = radeon_dpm_quirk_list; + bool disable_dpm = false; + + /* Apply dpm quirks */ + while (p && p->chip_device != 0) { + if (rdev->pdev->vendor == p->chip_vendor && + rdev->pdev->device == p->chip_device && + rdev->pdev->subsystem_vendor == p->subsys_vendor && + rdev->pdev->subsystem_device == p->subsys_device) { + disable_dpm = true; + break; + } + ++p; + } + + /* enable dpm on rv6xx+ */ + switch (rdev->family) { + case CHIP_RV610: + case CHIP_RV630: + case CHIP_RV620: + case CHIP_RV635: + case CHIP_RV670: + case CHIP_RS780: + case CHIP_RS880: + case CHIP_RV770: + /* DPM requires the RLC, RV770+ dGPU requires SMC */ + if (!rdev->rlc_fw) + rdev->pm.pm_method = PM_METHOD_PROFILE; + else if ((rdev->family >= CHIP_RV770) && + (!(rdev->flags & RADEON_IS_IGP)) && + (!rdev->smc_fw)) + rdev->pm.pm_method = PM_METHOD_PROFILE; + else if (radeon_dpm == 1) + rdev->pm.pm_method = PM_METHOD_DPM; + else + rdev->pm.pm_method = PM_METHOD_PROFILE; + break; + case CHIP_RV730: + case CHIP_RV710: + case CHIP_RV740: + case CHIP_CEDAR: + case CHIP_REDWOOD: + case CHIP_JUNIPER: + case CHIP_CYPRESS: + case CHIP_HEMLOCK: + case CHIP_PALM: + case CHIP_SUMO: + case CHIP_SUMO2: + case CHIP_BARTS: + case CHIP_TURKS: + case CHIP_CAICOS: + case CHIP_CAYMAN: + case CHIP_ARUBA: + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_VERDE: + case CHIP_OLAND: + case CHIP_HAINAN: + case CHIP_BONAIRE: + case CHIP_KABINI: + case CHIP_KAVERI: + case CHIP_HAWAII: + case CHIP_MULLINS: + /* DPM requires the RLC, RV770+ dGPU requires SMC */ + if (!rdev->rlc_fw) + rdev->pm.pm_method = PM_METHOD_PROFILE; + else if ((rdev->family >= CHIP_RV770) && + (!(rdev->flags & RADEON_IS_IGP)) && + (!rdev->smc_fw)) + rdev->pm.pm_method = PM_METHOD_PROFILE; + else if (disable_dpm && (radeon_dpm == -1)) + rdev->pm.pm_method = PM_METHOD_PROFILE; + else if (radeon_dpm == 0) + rdev->pm.pm_method = PM_METHOD_PROFILE; + else + rdev->pm.pm_method = PM_METHOD_DPM; + break; + default: + /* default to profile method */ + rdev->pm.pm_method = PM_METHOD_PROFILE; + break; + } + + if (rdev->pm.pm_method == PM_METHOD_DPM) + return radeon_pm_init_dpm(rdev); + else + return radeon_pm_init_old(rdev); +} + +int radeon_pm_late_init(struct radeon_device *rdev) +{ + int ret = 0; + + if (rdev->pm.pm_method == PM_METHOD_DPM) { + if (rdev->pm.dpm_enabled) { +#ifdef __linux__ + if (!rdev->pm.sysfs_initialized) { + ret = device_create_file(rdev->dev, &dev_attr_power_dpm_state); + if (ret) + DRM_ERROR("failed to create device file for dpm state\n"); + ret = device_create_file(rdev->dev, &dev_attr_power_dpm_force_performance_level); + if (ret) + DRM_ERROR("failed to create device file for dpm state\n"); + /* XXX: these are noops for dpm but are here for backwards compat */ + ret = device_create_file(rdev->dev, &dev_attr_power_profile); + if (ret) + DRM_ERROR("failed to create device file for power profile\n"); + ret = device_create_file(rdev->dev, &dev_attr_power_method); + if (ret) + DRM_ERROR("failed to create device file for power method\n"); + rdev->pm.sysfs_initialized = true; + } +#endif + + mutex_lock(&rdev->pm.mutex); + ret = radeon_dpm_late_enable(rdev); + mutex_unlock(&rdev->pm.mutex); + if (ret) { + rdev->pm.dpm_enabled = false; + DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n"); + } else { + /* set the dpm state for PX since there won't be + * a modeset to call this. + */ + radeon_pm_compute_clocks(rdev); + } + } + } else { +#ifdef __linux__ + if ((rdev->pm.num_power_states > 1) && + (!rdev->pm.sysfs_initialized)) { + /* where's the best place to put these? */ + ret = device_create_file(rdev->dev, &dev_attr_power_profile); + if (ret) + DRM_ERROR("failed to create device file for power profile\n"); + ret = device_create_file(rdev->dev, &dev_attr_power_method); + if (ret) + DRM_ERROR("failed to create device file for power method\n"); + if (!ret) + rdev->pm.sysfs_initialized = true; + } +#endif + } + return ret; +} + +static void radeon_pm_fini_old(struct radeon_device *rdev) { if (rdev->pm.num_power_states > 1) { mutex_lock(&rdev->pm.mutex); @@ -671,19 +1614,46 @@ void radeon_pm_fini(struct radeon_device *rdev) cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work); -#ifdef notyet +#ifdef __linux__ device_remove_file(rdev->dev, &dev_attr_power_profile); device_remove_file(rdev->dev, &dev_attr_power_method); #endif } - if (rdev->pm.power_state) - kfree(rdev->pm.power_state); + radeon_hwmon_fini(rdev); + kfree(rdev->pm.power_state); +} + +static void radeon_pm_fini_dpm(struct radeon_device *rdev) +{ + if (rdev->pm.num_power_states > 1) { + mutex_lock(&rdev->pm.mutex); + radeon_dpm_disable(rdev); + mutex_unlock(&rdev->pm.mutex); + +#ifdef __linux__ + device_remove_file(rdev->dev, &dev_attr_power_dpm_state); + device_remove_file(rdev->dev, &dev_attr_power_dpm_force_performance_level); + /* XXX backwards compat */ + device_remove_file(rdev->dev, &dev_attr_power_profile); + device_remove_file(rdev->dev, &dev_attr_power_method); +#endif + } + radeon_dpm_fini(rdev); radeon_hwmon_fini(rdev); + kfree(rdev->pm.power_state); } -void radeon_pm_compute_clocks(struct radeon_device *rdev) +void radeon_pm_fini(struct radeon_device *rdev) +{ + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_fini_dpm(rdev); + else + radeon_pm_fini_old(rdev); +} + +static void radeon_pm_compute_clocks_old(struct radeon_device *rdev) { struct drm_device *ddev = rdev->ddev; struct drm_crtc *crtc; @@ -696,12 +1666,14 @@ void radeon_pm_compute_clocks(struct radeon_device *rdev) rdev->pm.active_crtcs = 0; rdev->pm.active_crtc_count = 0; - list_for_each_entry(crtc, - &ddev->mode_config.crtc_list, head) { - radeon_crtc = to_radeon_crtc(crtc); - if (radeon_crtc->enabled) { - rdev->pm.active_crtcs |= (1 << radeon_crtc->crtc_id); - rdev->pm.active_crtc_count++; + if (rdev->num_crtc && rdev->mode_info.mode_config_initialized) { + list_for_each_entry(crtc, + &ddev->mode_config.crtc_list, head) { + radeon_crtc = to_radeon_crtc(crtc); + if (radeon_crtc->enabled) { + rdev->pm.active_crtcs |= (1 << radeon_crtc->crtc_id); + rdev->pm.active_crtc_count++; + } } } @@ -754,6 +1726,51 @@ void radeon_pm_compute_clocks(struct radeon_device *rdev) mutex_unlock(&rdev->pm.mutex); } +static void radeon_pm_compute_clocks_dpm(struct radeon_device *rdev) +{ + struct drm_device *ddev = rdev->ddev; + struct drm_crtc *crtc; + struct radeon_crtc *radeon_crtc; + + if (!rdev->pm.dpm_enabled) + return; + + mutex_lock(&rdev->pm.mutex); + + /* update active crtc counts */ + rdev->pm.dpm.new_active_crtcs = 0; + rdev->pm.dpm.new_active_crtc_count = 0; + if (rdev->num_crtc && rdev->mode_info.mode_config_initialized) { + list_for_each_entry(crtc, + &ddev->mode_config.crtc_list, head) { + radeon_crtc = to_radeon_crtc(crtc); + if (crtc->enabled) { + rdev->pm.dpm.new_active_crtcs |= (1 << radeon_crtc->crtc_id); + rdev->pm.dpm.new_active_crtc_count++; + } + } + } + + /* update battery/ac status */ + if (power_supply_is_system_supplied() > 0) + rdev->pm.dpm.ac_power = true; + else + rdev->pm.dpm.ac_power = false; + + radeon_dpm_change_power_state_locked(rdev); + + mutex_unlock(&rdev->pm.mutex); + +} + +void radeon_pm_compute_clocks(struct radeon_device *rdev) +{ + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_compute_clocks_dpm(rdev); + else + radeon_pm_compute_clocks_old(rdev); +} + static bool radeon_pm_in_vbl(struct radeon_device *rdev) { int crtc, vpos, hpos, vbl_status; @@ -765,7 +1782,8 @@ static bool radeon_pm_in_vbl(struct radeon_device *rdev) for (crtc = 0; (crtc < rdev->num_crtc) && in_vbl; crtc++) { if (rdev->pm.active_crtcs & (1 << crtc)) { vbl_status = radeon_get_crtc_scanoutpos(rdev->ddev, - crtc, 0, + crtc, + USE_REAL_VBLANKSTART, &vpos, &hpos, NULL, NULL, &rdev->mode_info.crtcs[crtc]->base.hwmode); if ((vbl_status & DRM_SCANOUTPOS_VALID) && @@ -861,20 +1879,33 @@ static int radeon_debugfs_pm_info(struct seq_file *m, void *data) struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; struct radeon_device *rdev = dev->dev_private; + struct drm_device *ddev = rdev->ddev; - seq_printf(m, "default engine clock: %u0 kHz\n", rdev->pm.default_sclk); - /* radeon_get_engine_clock is not reliable on APUs so just print the current clock */ - if ((rdev->family >= CHIP_PALM) && (rdev->flags & RADEON_IS_IGP)) - seq_printf(m, "current engine clock: %u0 kHz\n", rdev->pm.current_sclk); - else - seq_printf(m, "current engine clock: %u0 kHz\n", radeon_get_engine_clock(rdev)); - seq_printf(m, "default memory clock: %u0 kHz\n", rdev->pm.default_mclk); - if (rdev->asic->pm.get_memory_clock) - seq_printf(m, "current memory clock: %u0 kHz\n", radeon_get_memory_clock(rdev)); - if (rdev->pm.current_vddc) - seq_printf(m, "voltage: %u mV\n", rdev->pm.current_vddc); - if (rdev->asic->pm.get_pcie_lanes) - seq_printf(m, "PCIE lanes: %d\n", radeon_get_pcie_lanes(rdev)); + if ((rdev->flags & RADEON_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) { + seq_printf(m, "PX asic powered off\n"); + } else if (rdev->pm.dpm_enabled) { + mutex_lock(&rdev->pm.mutex); + if (rdev->asic->dpm.debugfs_print_current_performance_level) + radeon_dpm_debugfs_print_current_performance_level(rdev, m); + else + seq_printf(m, "Debugfs support not implemented for this asic\n"); + mutex_unlock(&rdev->pm.mutex); + } else { + seq_printf(m, "default engine clock: %u0 kHz\n", rdev->pm.default_sclk); + /* radeon_get_engine_clock is not reliable on APUs so just print the current clock */ + if ((rdev->family >= CHIP_PALM) && (rdev->flags & RADEON_IS_IGP)) + seq_printf(m, "current engine clock: %u0 kHz\n", rdev->pm.current_sclk); + else + seq_printf(m, "current engine clock: %u0 kHz\n", radeon_get_engine_clock(rdev)); + seq_printf(m, "default memory clock: %u0 kHz\n", rdev->pm.default_mclk); + if (rdev->asic->pm.get_memory_clock) + seq_printf(m, "current memory clock: %u0 kHz\n", radeon_get_memory_clock(rdev)); + if (rdev->pm.current_vddc) + seq_printf(m, "voltage: %u mV\n", rdev->pm.current_vddc); + if (rdev->asic->pm.get_pcie_lanes) + seq_printf(m, "PCIE lanes: %d\n", radeon_get_pcie_lanes(rdev)); + } return 0; } diff --git a/sys/dev/pci/drm/radeon/radeon_prime.c b/sys/dev/pci/drm/radeon/radeon_prime.c index 628d1534823..975d2d58416 100644 --- a/sys/dev/pci/drm/radeon/radeon_prime.c +++ b/sys/dev/pci/drm/radeon/radeon_prime.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_prime.c,v 1.6 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2012 Advanced Micro Devices, Inc. * @@ -29,197 +28,99 @@ #include "radeon.h" #include -static struct sg_table *radeon_gem_map_dma_buf(struct dma_buf_attachment *attachment, - enum dma_data_direction dir) +struct sg_table *radeon_gem_prime_get_sg_table(struct drm_gem_object *obj) { - struct radeon_bo *bo = attachment->dmabuf->priv; - struct drm_device *dev = bo->rdev->ddev; + struct radeon_bo *bo = gem_to_radeon_bo(obj); int npages = bo->tbo.num_pages; - struct sg_table *sg; - int nents; - - mutex_lock(&dev->struct_mutex); - sg = drm_prime_pages_to_sg(bo->tbo.ttm->pages, npages); - nents = dma_map_sg(attachment->dev, sg->sgl, sg->nents, dir); - mutex_unlock(&dev->struct_mutex); - return sg; -} - -static void radeon_gem_unmap_dma_buf(struct dma_buf_attachment *attachment, - struct sg_table *sg, enum dma_data_direction dir) -{ - dma_unmap_sg(attachment->dev, sg->sgl, sg->nents, dir); - sg_free_table(sg); - kfree(sg); -} - -static void radeon_gem_dmabuf_release(struct dma_buf *dma_buf) -{ - struct radeon_bo *bo = dma_buf->priv; - - if (bo->gem_base.export_dma_buf == dma_buf) { - DRM_ERROR("unreference dmabuf %p\n", &bo->gem_base); - bo->gem_base.export_dma_buf = NULL; - drm_gem_object_unreference_unlocked(&bo->gem_base); - } -} - -static void *radeon_gem_kmap_atomic(struct dma_buf *dma_buf, unsigned long page_num) -{ - return NULL; -} - -static void radeon_gem_kunmap_atomic(struct dma_buf *dma_buf, unsigned long page_num, void *addr) -{ -} -static void *radeon_gem_kmap(struct dma_buf *dma_buf, unsigned long page_num) -{ - return NULL; -} - -static void radeon_gem_kunmap(struct dma_buf *dma_buf, unsigned long page_num, void *addr) -{ - -} - -static int radeon_gem_prime_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) -{ - return -EINVAL; + return drm_prime_pages_to_sg(bo->tbo.ttm->pages, npages); } -static void *radeon_gem_prime_vmap(struct dma_buf *dma_buf) +void *radeon_gem_prime_vmap(struct drm_gem_object *obj) { - struct radeon_bo *bo = dma_buf->priv; - struct drm_device *dev = bo->rdev->ddev; + struct radeon_bo *bo = gem_to_radeon_bo(obj); int ret; - mutex_lock(&dev->struct_mutex); - if (bo->vmapping_count) { - bo->vmapping_count++; - goto out_unlock; - } - ret = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->dma_buf_vmap); - if (ret) { - mutex_unlock(&dev->struct_mutex); + if (ret) return ERR_PTR(ret); - } - bo->vmapping_count = 1; -out_unlock: - mutex_unlock(&dev->struct_mutex); + return bo->dma_buf_vmap.virtual; } -static void radeon_gem_prime_vunmap(struct dma_buf *dma_buf, void *vaddr) +void radeon_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr) { - struct radeon_bo *bo = dma_buf->priv; - struct drm_device *dev = bo->rdev->ddev; - - mutex_lock(&dev->struct_mutex); - bo->vmapping_count--; - if (bo->vmapping_count == 0) { - ttm_bo_kunmap(&bo->dma_buf_vmap); - } - mutex_unlock(&dev->struct_mutex); + struct radeon_bo *bo = gem_to_radeon_bo(obj); + + ttm_bo_kunmap(&bo->dma_buf_vmap); } -const static struct dma_buf_ops radeon_dmabuf_ops = { - .map_dma_buf = radeon_gem_map_dma_buf, - .unmap_dma_buf = radeon_gem_unmap_dma_buf, - .release = radeon_gem_dmabuf_release, - .kmap = radeon_gem_kmap, - .kmap_atomic = radeon_gem_kmap_atomic, - .kunmap = radeon_gem_kunmap, - .kunmap_atomic = radeon_gem_kunmap_atomic, - .mmap = radeon_gem_prime_mmap, - .vmap = radeon_gem_prime_vmap, - .vunmap = radeon_gem_prime_vunmap, -}; - -static int radeon_prime_create(struct drm_device *dev, - size_t size, - struct sg_table *sg, - struct radeon_bo **pbo) + +struct drm_gem_object *radeon_gem_prime_import_sg_table(struct drm_device *dev, + struct dma_buf_attachment *attach, + struct sg_table *sg) { + struct reservation_object *resv = attach->dmabuf->resv; struct radeon_device *rdev = dev->dev_private; struct radeon_bo *bo; int ret; - ret = radeon_bo_create(rdev, size, PAGE_SIZE, false, - RADEON_GEM_DOMAIN_GTT, sg, pbo); + ww_mutex_lock(&resv->lock, NULL); + ret = radeon_bo_create(rdev, attach->dmabuf->size, PAGE_SIZE, false, + RADEON_GEM_DOMAIN_GTT, 0, sg, resv, &bo); + ww_mutex_unlock(&resv->lock); if (ret) - return ret; - bo = *pbo; - bo->gem_base.driver_private = bo; + return ERR_PTR(ret); mutex_lock(&rdev->gem.mutex); list_add_tail(&bo->list, &rdev->gem.objects); mutex_unlock(&rdev->gem.mutex); - return 0; + return &bo->gem_base; } -struct dma_buf *radeon_gem_prime_export(struct drm_device *dev, - struct drm_gem_object *obj, - int flags) +int radeon_gem_prime_pin(struct drm_gem_object *obj) { struct radeon_bo *bo = gem_to_radeon_bo(obj); int ret = 0; ret = radeon_bo_reserve(bo, false); if (unlikely(ret != 0)) - return ERR_PTR(ret); + return ret; /* pin buffer into GTT */ ret = radeon_bo_pin(bo, RADEON_GEM_DOMAIN_GTT, NULL); - if (ret) { - radeon_bo_unreserve(bo); - return ERR_PTR(ret); - } radeon_bo_unreserve(bo); - return dma_buf_export(bo, &radeon_dmabuf_ops, obj->size, flags); + return ret; } -struct drm_gem_object *radeon_gem_prime_import(struct drm_device *dev, - struct dma_buf *dma_buf) +void radeon_gem_prime_unpin(struct drm_gem_object *obj) { - struct dma_buf_attachment *attach; - struct sg_table *sg; - struct radeon_bo *bo; - int ret; + struct radeon_bo *bo = gem_to_radeon_bo(obj); + int ret = 0; - if (dma_buf->ops == &radeon_dmabuf_ops) { - bo = dma_buf->priv; - if (bo->gem_base.dev == dev) { - drm_gem_object_reference(&bo->gem_base); - dma_buf_put(dma_buf); - return &bo->gem_base; - } - } - - /* need to attach */ - attach = dma_buf_attach(dma_buf, dev->dev); - if (IS_ERR(attach)) - return ERR_CAST(attach); - - sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); - if (IS_ERR(sg)) { - ret = PTR_ERR(sg); - goto fail_detach; - } - - ret = radeon_prime_create(dev, dma_buf->size, sg, &bo); - if (ret) - goto fail_unmap; + ret = radeon_bo_reserve(bo, false); + if (unlikely(ret != 0)) + return; - bo->gem_base.import_attach = attach; + radeon_bo_unpin(bo); + radeon_bo_unreserve(bo); +} - return &bo->gem_base; -fail_unmap: - dma_buf_unmap_attachment(attach, sg, DMA_BIDIRECTIONAL); -fail_detach: - dma_buf_detach(dma_buf, attach); - return ERR_PTR(ret); +struct reservation_object *radeon_gem_prime_res_obj(struct drm_gem_object *obj) +{ + struct radeon_bo *bo = gem_to_radeon_bo(obj); + + return bo->tbo.resv; +} + +struct dma_buf *radeon_gem_prime_export(struct drm_device *dev, + struct drm_gem_object *gobj, + int flags) +{ + struct radeon_bo *bo = gem_to_radeon_bo(gobj); + if (radeon_ttm_tt_has_userptr(bo->tbo.ttm)) + return ERR_PTR(-EPERM); + return drm_gem_prime_export(dev, gobj, flags); } diff --git a/sys/dev/pci/drm/radeon/radeon_reg.h b/sys/dev/pci/drm/radeon/radeon_reg.h index d6e1c9444d7..62d54976d24 100644 --- a/sys/dev/pci/drm/radeon/radeon_reg.h +++ b/sys/dev/pci/drm/radeon/radeon_reg.h @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_reg.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2000 ATI Technologies Inc., Markham, Ontario, and * VA Linux Systems Inc., Fremont, California. @@ -58,6 +57,7 @@ #include "evergreen_reg.h" #include "ni_reg.h" #include "si_reg.h" +#include "cik_reg.h" #define RADEON_MC_AGP_LOCATION 0x014c #define RADEON_MC_AGP_START_MASK 0x0000FFFF @@ -427,8 +427,6 @@ # define RADEON_CRTC_VSYNC_DIS (1 << 9) # define RADEON_CRTC_DISPLAY_DIS (1 << 10) # define RADEON_CRTC_SYNC_TRISTAT (1 << 11) -# define RADEON_CRTC_HSYNC_TRISTAT (1 << 12) -# define RADEON_CRTC_VSYNC_TRISTAT (1 << 13) # define RADEON_CRTC_CRT_ON (1 << 15) #define RADEON_CRTC_EXT_CNTL_DPMS_BYTE 0x0055 # define RADEON_CRTC_HSYNC_DIS_BYTE (1 << 0) @@ -3709,4 +3707,19 @@ #define RV530_GB_PIPE_SELECT2 0x4124 +#define RADEON_CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) +#define RADEON_CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) +#define RADEON_CP_PACKET0_GET_ONE_REG_WR(h) (((h) >> 15) & 1) +#define RADEON_CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) +#define R100_CP_PACKET0_GET_REG(h) (((h) & 0x1FFF) << 2) +#define R600_CP_PACKET0_GET_REG(h) (((h) & 0xFFFF) << 2) +#define RADEON_PACKET_TYPE0 0 +#define RADEON_PACKET_TYPE1 1 +#define RADEON_PACKET_TYPE2 2 +#define RADEON_PACKET_TYPE3 3 + +#define RADEON_PACKET3_NOP 0x10 + +#define RADEON_VLINE_STAT (1 << 12) + #endif diff --git a/sys/dev/pci/drm/radeon/radeon_ring.c b/sys/dev/pci/drm/radeon/radeon_ring.c index b5f19f716d1..54d2e7ef9c2 100644 --- a/sys/dev/pci/drm/radeon/radeon_ring.c +++ b/sys/dev/pci/drm/radeon/radeon_ring.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_ring.c,v 1.10 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -28,256 +27,7 @@ * Christian König */ #include -#include -#include "radeon_reg.h" #include "radeon.h" -#include "atom.h" - -/* - * IB - * IBs (Indirect Buffers) and areas of GPU accessible memory where - * commands are stored. You can put a pointer to the IB in the - * command ring and the hw will fetch the commands from the IB - * and execute them. Generally userspace acceleration drivers - * produce command buffers which are send to the kernel and - * put in IBs for execution by the requested ring. - */ -static int radeon_debugfs_sa_init(struct radeon_device *rdev); - -/** - * radeon_ib_get - request an IB (Indirect Buffer) - * - * @rdev: radeon_device pointer - * @ring: ring index the IB is associated with - * @ib: IB object returned - * @size: requested IB size - * - * Request an IB (all asics). IBs are allocated using the - * suballocator. - * Returns 0 on success, error on failure. - */ -int radeon_ib_get(struct radeon_device *rdev, int ring, - struct radeon_ib *ib, struct radeon_vm *vm, - unsigned size) -{ - int i, r; - - r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256, true); - if (r) { - dev_err(rdev->dev, "failed to get a new IB (%d)\n", r); - return r; - } - - r = radeon_semaphore_create(rdev, &ib->semaphore); - if (r) { - return r; - } - - ib->ring = ring; - ib->fence = NULL; - ib->ptr = radeon_sa_bo_cpu_addr(ib->sa_bo); - ib->vm = vm; - if (vm) { - /* ib pool is bound at RADEON_VA_IB_OFFSET in virtual address - * space and soffset is the offset inside the pool bo - */ - ib->gpu_addr = ib->sa_bo->soffset + RADEON_VA_IB_OFFSET; - } else { - ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo); - } - ib->is_const_ib = false; - for (i = 0; i < RADEON_NUM_RINGS; ++i) - ib->sync_to[i] = NULL; - - return 0; -} - -/** - * radeon_ib_free - free an IB (Indirect Buffer) - * - * @rdev: radeon_device pointer - * @ib: IB object to free - * - * Free an IB (all asics). - */ -void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib) -{ - radeon_semaphore_free(rdev, &ib->semaphore, ib->fence); - radeon_sa_bo_free(rdev, &ib->sa_bo, ib->fence); - radeon_fence_unref(&ib->fence); -} - -/** - * radeon_ib_schedule - schedule an IB (Indirect Buffer) on the ring - * - * @rdev: radeon_device pointer - * @ib: IB object to schedule - * @const_ib: Const IB to schedule (SI only) - * - * Schedule an IB on the associated ring (all asics). - * Returns 0 on success, error on failure. - * - * On SI, there are two parallel engines fed from the primary ring, - * the CE (Constant Engine) and the DE (Drawing Engine). Since - * resource descriptors have moved to memory, the CE allows you to - * prime the caches while the DE is updating register state so that - * the resource descriptors will be already in cache when the draw is - * processed. To accomplish this, the userspace driver submits two - * IBs, one for the CE and one for the DE. If there is a CE IB (called - * a CONST_IB), it will be put on the ring prior to the DE IB. Prior - * to SI there was just a DE IB. - */ -int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib, - struct radeon_ib *const_ib) -{ - struct radeon_ring *ring = &rdev->ring[ib->ring]; - bool need_sync = false; - int i, r = 0; - - if (!ib->length_dw || !ring->ready) { - /* TODO: Nothings in the ib we should report. */ - dev_err(rdev->dev, "couldn't schedule ib\n"); - return -EINVAL; - } - - /* 64 dwords should be enough for fence too */ - r = radeon_ring_lock(rdev, ring, 64 + RADEON_NUM_RINGS * 8); - if (r) { - dev_err(rdev->dev, "scheduling IB failed (%d).\n", r); - return r; - } - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - struct radeon_fence *fence = ib->sync_to[i]; - if (radeon_fence_need_sync(fence, ib->ring)) { - need_sync = true; - radeon_semaphore_sync_rings(rdev, ib->semaphore, - fence->ring, ib->ring); - radeon_fence_note_sync(fence, ib->ring); - } - } - /* immediately free semaphore when we don't need to sync */ - if (!need_sync) { - radeon_semaphore_free(rdev, &ib->semaphore, NULL); - } - /* if we can't remember our last VM flush then flush now! */ - /* XXX figure out why we have to flush for every IB */ - if (ib->vm /*&& !ib->vm->last_flush*/) { - radeon_ring_vm_flush(rdev, ib->ring, ib->vm); - } - if (const_ib) { - radeon_ring_ib_execute(rdev, const_ib->ring, const_ib); - radeon_semaphore_free(rdev, &const_ib->semaphore, NULL); - } - radeon_ring_ib_execute(rdev, ib->ring, ib); - r = radeon_fence_emit(rdev, &ib->fence, ib->ring); - if (r) { - dev_err(rdev->dev, "failed to emit fence for new IB (%d)\n", r); - radeon_ring_unlock_undo(rdev, ring); - return r; - } - if (const_ib) { - const_ib->fence = radeon_fence_ref(ib->fence); - } - /* we just flushed the VM, remember that */ - if (ib->vm && !ib->vm->last_flush) { - ib->vm->last_flush = radeon_fence_ref(ib->fence); - } - radeon_ring_unlock_commit(rdev, ring); - return 0; -} - -/** - * radeon_ib_pool_init - Init the IB (Indirect Buffer) pool - * - * @rdev: radeon_device pointer - * - * Initialize the suballocator to manage a pool of memory - * for use as IBs (all asics). - * Returns 0 on success, error on failure. - */ -int radeon_ib_pool_init(struct radeon_device *rdev) -{ - int r; - - if (rdev->ib_pool_ready) { - return 0; - } - r = radeon_sa_bo_manager_init(rdev, &rdev->ring_tmp_bo, - RADEON_IB_POOL_SIZE*64*1024, - RADEON_GPU_PAGE_SIZE, - RADEON_GEM_DOMAIN_GTT); - if (r) { - return r; - } - - r = radeon_sa_bo_manager_start(rdev, &rdev->ring_tmp_bo); - if (r) { - return r; - } - - rdev->ib_pool_ready = true; - if (radeon_debugfs_sa_init(rdev)) { - dev_err(rdev->dev, "failed to register debugfs file for SA\n"); - } - return 0; -} - -/** - * radeon_ib_pool_fini - Free the IB (Indirect Buffer) pool - * - * @rdev: radeon_device pointer - * - * Tear down the suballocator managing the pool of memory - * for use as IBs (all asics). - */ -void radeon_ib_pool_fini(struct radeon_device *rdev) -{ - if (rdev->ib_pool_ready) { - radeon_sa_bo_manager_suspend(rdev, &rdev->ring_tmp_bo); - radeon_sa_bo_manager_fini(rdev, &rdev->ring_tmp_bo); - rdev->ib_pool_ready = false; - } -} - -/** - * radeon_ib_ring_tests - test IBs on the rings - * - * @rdev: radeon_device pointer - * - * Test an IB (Indirect Buffer) on each ring. - * If the test fails, disable the ring. - * Returns 0 on success, error if the primary GFX ring - * IB test fails. - */ -int radeon_ib_ring_tests(struct radeon_device *rdev) -{ - unsigned i; - int r; - - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - struct radeon_ring *ring = &rdev->ring[i]; - - if (!ring->ready) - continue; - - r = radeon_ib_test(rdev, i, ring); - if (r) { - ring->ready = false; - - if (i == RADEON_RING_TYPE_GFX_INDEX) { - /* oh, oh, that's really bad */ - DRM_ERROR("radeon: failed testing IB on GFX ring (%d).\n", r); - rdev->accel_working = false; - return r; - - } else { - /* still not good, but we can live with it */ - DRM_ERROR("radeon: failed testing IB on ring %d (%d).\n", i, r); - } - } - } - return 0; -} /* * Rings @@ -294,29 +44,6 @@ int radeon_ib_ring_tests(struct radeon_device *rdev) */ static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring); -#if defined(DRM_DEBUG_CODE) && DRM_DEBUG_CODE != 0 -/** - * radeon_ring_write - write a value to the ring - * - * @ring: radeon_ring structure holding ring information - * @v: dword (dw) value to write - * - * Write a value to the requested ring buffer (all asics). - */ -void radeon_ring_write(struct radeon_ring *ring, uint32_t v) -{ -#if DRM_DEBUG_CODE - if (ring->count_dw <= 0) { - DRM_ERROR("radeon: writing more dwords to the ring than expected!\n"); - } -#endif - ring->ring[ring->wptr++] = v; - ring->wptr &= ring->ptr_mask; - ring->count_dw--; - ring->ring_free_dw--; -} -#endif - /** * radeon_ring_supports_scratch_reg - check if the ring supports * writing to scratch registers @@ -350,19 +77,17 @@ bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev, */ void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring) { - u32 rptr; + uint32_t rptr = radeon_ring_get_rptr(rdev, ring); - if (rdev->wb.enabled) - rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]); - else - rptr = RREG32(ring->rptr_reg); - ring->rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift; /* This works because ring_size is a power of 2 */ - ring->ring_free_dw = (ring->rptr + (ring->ring_size / 4)); + ring->ring_free_dw = rptr + (ring->ring_size / 4); ring->ring_free_dw -= ring->wptr; ring->ring_free_dw &= ring->ptr_mask; if (!ring->ring_free_dw) { + /* this is an empty ring */ ring->ring_free_dw = ring->ring_size / 4; + /* update lockup info to avoid false positive */ + radeon_ring_lockup_update(rdev, ring); } } @@ -386,19 +111,13 @@ int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsi /* Align requested size with padding so unlock_commit can * pad safely */ radeon_ring_free_size(rdev, ring); - if (ring->ring_free_dw == (ring->ring_size / 4)) { - /* This is an empty ring update lockup info to avoid - * false positive. - */ - radeon_ring_lockup_update(ring); - } ndw = (ndw + ring->align_mask) & ~ring->align_mask; while (ndw > (ring->ring_free_dw - 1)) { radeon_ring_free_size(rdev, ring); if (ndw < ring->ring_free_dw) { break; } - r = radeon_fence_wait_next_locked(rdev, ring->idx); + r = radeon_fence_wait_next(rdev, ring->idx); if (r) return r; } @@ -437,19 +156,30 @@ int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsig * * @rdev: radeon_device pointer * @ring: radeon_ring structure holding ring information + * @hdp_flush: Whether or not to perform an HDP cache flush * * Update the wptr (write pointer) to tell the GPU to * execute new commands on the ring buffer (all asics). */ -void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring) +void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring, + bool hdp_flush) { + /* If we are emitting the HDP flush via the ring buffer, we need to + * do it before padding. + */ + if (hdp_flush && rdev->asic->ring[ring->idx]->hdp_flush) + rdev->asic->ring[ring->idx]->hdp_flush(rdev, ring); /* We pad to match fetch size */ while (ring->wptr & ring->align_mask) { radeon_ring_write(ring, ring->nop); } - DRM_MEMORYBARRIER(); - WREG32(ring->wptr_reg, (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask); - (void)RREG32(ring->wptr_reg); + mb(); + /* If we are emitting the HDP flush via MMIO, we need to do it after + * all CPU writes to VRAM finished. + */ + if (hdp_flush && rdev->asic->mmio_hdp_flush) + rdev->asic->mmio_hdp_flush(rdev); + radeon_ring_set_wptr(rdev, ring); } /** @@ -458,12 +188,14 @@ void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring) * * @rdev: radeon_device pointer * @ring: radeon_ring structure holding ring information + * @hdp_flush: Whether or not to perform an HDP cache flush * * Call radeon_ring_commit() then unlock the ring (all asics). */ -void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring) +void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring, + bool hdp_flush) { - radeon_ring_commit(rdev, ring); + radeon_ring_commit(rdev, ring, hdp_flush); mutex_unlock(&rdev->ring_lock); } @@ -492,29 +224,6 @@ void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *rin mutex_unlock(&rdev->ring_lock); } -/** - * radeon_ring_force_activity - add some nop packets to the ring - * - * @rdev: radeon_device pointer - * @ring: radeon_ring structure holding ring information - * - * Add some nop packets to the ring to force activity (all asics). - * Used for lockup detection to see if the rptr is advancing. - */ -void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring *ring) -{ - int r; - - radeon_ring_free_size(rdev, ring); - if (ring->rptr == ring->wptr) { - r = radeon_ring_alloc(rdev, ring, 1); - if (!r) { - radeon_ring_write(ring, ring->nop); - radeon_ring_commit(rdev, ring); - } - } -} - /** * radeon_ring_lockup_update - update lockup variables * @@ -522,10 +231,11 @@ void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring * * * Update the last rptr value and timestamp (all asics). */ -void radeon_ring_lockup_update(struct radeon_ring *ring) +void radeon_ring_lockup_update(struct radeon_device *rdev, + struct radeon_ring *ring) { - ring->last_rptr = ring->rptr; - ring->last_activity = jiffies; + atomic_set(&ring->last_rptr, radeon_ring_get_rptr(rdev, ring)); + atomic64_set(&ring->last_activity, jiffies_64); } /** @@ -533,42 +243,23 @@ void radeon_ring_lockup_update(struct radeon_ring *ring) * @rdev: radeon device structure * @ring: radeon_ring structure holding ring information * - * We don't need to initialize the lockup tracking information as we will either - * have CP rptr to a different value of jiffies wrap around which will force - * initialization of the lockup tracking informations. - * - * A possible false positivie is if we get call after while and last_cp_rptr == - * the current CP rptr, even if it's unlikely it might happen. To avoid this - * if the elapsed time since last call is bigger than 2 second than we return - * false and update the tracking information. Due to this the caller must call - * radeon_ring_test_lockup several time in less than 2sec for lockup to be reported - * the fencing code should be cautious about that. - * - * Caller should write to the ring to force CP to do something so we don't get - * false positive when CP is just gived nothing to do. - * - **/ + */ bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring) { - unsigned long cjiffies, elapsed; - uint32_t rptr; + uint32_t rptr = radeon_ring_get_rptr(rdev, ring); + uint64_t last = atomic64_read(&ring->last_activity); + uint64_t elapsed; - cjiffies = jiffies; - if (!time_after(cjiffies, ring->last_activity)) { - /* likely a wrap around */ - radeon_ring_lockup_update(ring); - return false; - } - rptr = RREG32(ring->rptr_reg); - ring->rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift; - if (ring->rptr != ring->last_rptr) { - /* CP is still working no lockup */ - radeon_ring_lockup_update(ring); + if (rptr != atomic_read(&ring->last_rptr)) { + /* ring is still working, no lockup */ + radeon_ring_lockup_update(rdev, ring); return false; } - elapsed = jiffies_to_msecs(cjiffies - ring->last_activity); + + elapsed = jiffies_to_msecs(jiffies_64 - last); if (radeon_lockup_timeout && elapsed >= radeon_lockup_timeout) { - dev_err(rdev->dev, "GPU lockup CP stall for more than %lumsec\n", elapsed); + dev_err(rdev->dev, "ring %d stalled for more than %llumsec\n", + ring->idx, elapsed); return true; } /* give a chance to the GPU ... */ @@ -623,7 +314,7 @@ unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring } /* and then save the content of the ring */ - *data = kmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); + *data = drm_malloc_ab(size, sizeof(uint32_t)); if (!*data) { mutex_unlock(&rdev->ring_lock); return 0; @@ -664,8 +355,8 @@ int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring, radeon_ring_write(ring, data[i]); } - radeon_ring_unlock_commit(rdev, ring); - kfree(data); + radeon_ring_unlock_commit(rdev, ring, false); + drm_free_large(data); return 0; } @@ -676,32 +367,23 @@ int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring, * @ring: radeon_ring structure holding ring information * @ring_size: size of the ring * @rptr_offs: offset of the rptr writeback location in the WB buffer - * @rptr_reg: MMIO offset of the rptr register - * @wptr_reg: MMIO offset of the wptr register - * @ptr_reg_shift: bit offset of the rptr/wptr values - * @ptr_reg_mask: bit mask of the rptr/wptr values * @nop: nop packet for this ring * * Initialize the driver information for the selected ring (all asics). * Returns 0 on success, error on failure. */ int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size, - unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg, - u32 ptr_reg_shift, u32 ptr_reg_mask, u32 nop) + unsigned rptr_offs, u32 nop) { int r; ring->ring_size = ring_size; ring->rptr_offs = rptr_offs; - ring->rptr_reg = rptr_reg; - ring->wptr_reg = wptr_reg; - ring->ptr_reg_shift = ptr_reg_shift; - ring->ptr_reg_mask = ptr_reg_mask; ring->nop = nop; /* Allocate ring buffer */ if (ring->ring_obj == NULL) { r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_GTT, + RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL, &ring->ring_obj); if (r) { dev_err(rdev->dev, "(%d) ring create failed\n", r); @@ -735,7 +417,7 @@ int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsig if (radeon_debugfs_ring_init(rdev, ring)) { DRM_ERROR("Failed to register debugfs file for rings !\n"); } - radeon_ring_lockup_update(ring); + radeon_ring_lockup_update(rdev, ring); return 0; } @@ -782,66 +464,74 @@ static int radeon_debugfs_ring_info(struct seq_file *m, void *data) struct radeon_device *rdev = dev->dev_private; int ridx = *(int*)node->info_ent->data; struct radeon_ring *ring = &rdev->ring[ridx]; + + uint32_t rptr, wptr, rptr_next; unsigned count, i, j; - u32 tmp; radeon_ring_free_size(rdev, ring); count = (ring->ring_size / 4) - ring->ring_free_dw; - tmp = RREG32(ring->wptr_reg) >> ring->ptr_reg_shift; - seq_printf(m, "wptr(0x%04x): 0x%08x [%5d]\n", ring->wptr_reg, tmp, tmp); - tmp = RREG32(ring->rptr_reg) >> ring->ptr_reg_shift; - seq_printf(m, "rptr(0x%04x): 0x%08x [%5d]\n", ring->rptr_reg, tmp, tmp); + + wptr = radeon_ring_get_wptr(rdev, ring); + seq_printf(m, "wptr: 0x%08x [%5d]\n", + wptr, wptr); + + rptr = radeon_ring_get_rptr(rdev, ring); + seq_printf(m, "rptr: 0x%08x [%5d]\n", + rptr, rptr); + if (ring->rptr_save_reg) { - seq_printf(m, "rptr next(0x%04x): 0x%08x\n", ring->rptr_save_reg, - RREG32(ring->rptr_save_reg)); - } - seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n", ring->wptr, ring->wptr); - seq_printf(m, "driver's copy of the rptr: 0x%08x [%5d]\n", ring->rptr, ring->rptr); - seq_printf(m, "last semaphore signal addr : 0x%016llx\n", ring->last_semaphore_signal_addr); - seq_printf(m, "last semaphore wait addr : 0x%016llx\n", ring->last_semaphore_wait_addr); + rptr_next = RREG32(ring->rptr_save_reg); + seq_printf(m, "rptr next(0x%04x): 0x%08x [%5d]\n", + ring->rptr_save_reg, rptr_next, rptr_next); + } else + rptr_next = ~0; + + seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n", + ring->wptr, ring->wptr); + seq_printf(m, "last semaphore signal addr : 0x%016llx\n", + ring->last_semaphore_signal_addr); + seq_printf(m, "last semaphore wait addr : 0x%016llx\n", + ring->last_semaphore_wait_addr); seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw); seq_printf(m, "%u dwords in ring\n", count); + + if (!ring->ring) + return 0; + /* print 8 dw before current rptr as often it's the last executed * packet that is the root issue */ - i = (ring->rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask; - if (ring->ready) { - for (j = 0; j <= (count + 32); j++) { - seq_printf(m, "r[%5d]=0x%08x\n", i, ring->ring[i]); - i = (i + 1) & ring->ptr_mask; - } + i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask; + for (j = 0; j <= (count + 32); j++) { + seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]); + if (rptr == i) + seq_puts(m, " *"); + if (rptr_next == i) + seq_puts(m, " #"); + seq_puts(m, "\n"); + i = (i + 1) & ring->ptr_mask; } return 0; } -static int radeon_ring_type_gfx_index = RADEON_RING_TYPE_GFX_INDEX; -static int cayman_ring_type_cp1_index = CAYMAN_RING_TYPE_CP1_INDEX; -static int cayman_ring_type_cp2_index = CAYMAN_RING_TYPE_CP2_INDEX; -static int radeon_ring_type_dma1_index = R600_RING_TYPE_DMA_INDEX; -static int radeon_ring_type_dma2_index = CAYMAN_RING_TYPE_DMA1_INDEX; +static int radeon_gfx_index = RADEON_RING_TYPE_GFX_INDEX; +static int cayman_cp1_index = CAYMAN_RING_TYPE_CP1_INDEX; +static int cayman_cp2_index = CAYMAN_RING_TYPE_CP2_INDEX; +static int radeon_dma1_index = R600_RING_TYPE_DMA_INDEX; +static int radeon_dma2_index = CAYMAN_RING_TYPE_DMA1_INDEX; +static int r600_uvd_index = R600_RING_TYPE_UVD_INDEX; +static int si_vce1_index = TN_RING_TYPE_VCE1_INDEX; +static int si_vce2_index = TN_RING_TYPE_VCE2_INDEX; static struct drm_info_list radeon_debugfs_ring_info_list[] = { - {"radeon_ring_gfx", radeon_debugfs_ring_info, 0, &radeon_ring_type_gfx_index}, - {"radeon_ring_cp1", radeon_debugfs_ring_info, 0, &cayman_ring_type_cp1_index}, - {"radeon_ring_cp2", radeon_debugfs_ring_info, 0, &cayman_ring_type_cp2_index}, - {"radeon_ring_dma1", radeon_debugfs_ring_info, 0, &radeon_ring_type_dma1_index}, - {"radeon_ring_dma2", radeon_debugfs_ring_info, 0, &radeon_ring_type_dma2_index}, -}; - -static int radeon_debugfs_sa_info(struct seq_file *m, void *data) -{ - struct drm_info_node *node = (struct drm_info_node *) m->private; - struct drm_device *dev = node->minor->dev; - struct radeon_device *rdev = dev->dev_private; - - radeon_sa_bo_dump_debug_info(&rdev->ring_tmp_bo, m); - - return 0; - -} - -static struct drm_info_list radeon_debugfs_sa_list[] = { - {"radeon_sa_info", &radeon_debugfs_sa_info, 0, NULL}, + {"radeon_ring_gfx", radeon_debugfs_ring_info, 0, &radeon_gfx_index}, + {"radeon_ring_cp1", radeon_debugfs_ring_info, 0, &cayman_cp1_index}, + {"radeon_ring_cp2", radeon_debugfs_ring_info, 0, &cayman_cp2_index}, + {"radeon_ring_dma1", radeon_debugfs_ring_info, 0, &radeon_dma1_index}, + {"radeon_ring_dma2", radeon_debugfs_ring_info, 0, &radeon_dma2_index}, + {"radeon_ring_uvd", radeon_debugfs_ring_info, 0, &r600_uvd_index}, + {"radeon_ring_vce1", radeon_debugfs_ring_info, 0, &si_vce1_index}, + {"radeon_ring_vce2", radeon_debugfs_ring_info, 0, &si_vce2_index}, }; #endif @@ -865,12 +555,3 @@ static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ri #endif return 0; } - -static int radeon_debugfs_sa_init(struct radeon_device *rdev) -{ -#if defined(CONFIG_DEBUG_FS) - return radeon_debugfs_add_files(rdev, radeon_debugfs_sa_list, 1); -#else - return 0; -#endif -} diff --git a/sys/dev/pci/drm/radeon/radeon_sa.c b/sys/dev/pci/drm/radeon/radeon_sa.c index 5eef79ecf47..a9e3a2308c8 100644 --- a/sys/dev/pci/drm/radeon/radeon_sa.c +++ b/sys/dev/pci/drm/radeon/radeon_sa.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_sa.c,v 1.10 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2011 Red Hat Inc. * All Rights Reserved. @@ -50,7 +49,7 @@ static void radeon_sa_bo_try_free(struct radeon_sa_manager *sa_manager); int radeon_sa_bo_manager_init(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager, - unsigned size, u32 align, u32 domain) + unsigned size, u32 align, u32 domain, u32 flags) { int i, r; @@ -66,7 +65,7 @@ int radeon_sa_bo_manager_init(struct radeon_device *rdev, } r = radeon_bo_create(rdev, size, align, true, - domain, NULL, &sa_manager->bo); + domain, flags, NULL, NULL, &sa_manager->bo); if (r) { dev_err(rdev->dev, "(%d) failed to allocate bo for manager\n", r); return r; @@ -313,11 +312,11 @@ static bool radeon_sa_bo_next_hole(struct radeon_sa_manager *sa_manager, int radeon_sa_bo_new(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager, struct radeon_sa_bo **sa_bo, - unsigned size, unsigned align, bool block) + unsigned size, unsigned align) { struct radeon_fence *fences[RADEON_NUM_RINGS]; unsigned tries[RADEON_NUM_RINGS]; - int i, r, error; + int i, r; BUG_ON(align > sa_manager->align); BUG_ON(size > sa_manager->size); @@ -350,24 +349,20 @@ int radeon_sa_bo_new(struct radeon_device *rdev, /* see if we can skip over some allocations */ } while (radeon_sa_bo_next_hole(sa_manager, fences, tries)); + for (i = 0; i < RADEON_NUM_RINGS; ++i) + radeon_fence_ref(fences[i]); + spin_unlock(&sa_manager->wq.lock); r = radeon_fence_wait_any(rdev, fences, false); + for (i = 0; i < RADEON_NUM_RINGS; ++i) + radeon_fence_unref(&fences[i]); spin_lock(&sa_manager->wq.lock); /* if we have nothing to wait for block */ - if (r == -ENOENT && block) { - r = 0; - while (r == 0) { - if (radeon_sa_event(sa_manager, size, align)) - break; - error = msleep(&sa_manager->wq, &sa_manager->wq.lock, - PZERO | PCATCH, "samgr", 0); - if (error == ERESTART) - error = EINTR; /* XXX */ - r = -error; - } - - } else if (r == -ENOENT) { - r = -ENOMEM; + if (r == -ENOENT) { + r = wait_event_interruptible_locked( + sa_manager->wq, + radeon_sa_event(sa_manager, size, align) + ); } } while (!r); @@ -409,13 +404,15 @@ void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager, spin_lock(&sa_manager->wq.lock); list_for_each_entry(i, &sa_manager->olist, olist) { + uint64_t soffset = i->soffset + sa_manager->gpu_addr; + uint64_t eoffset = i->eoffset + sa_manager->gpu_addr; if (&i->olist == sa_manager->hole) { seq_printf(m, ">"); } else { seq_printf(m, " "); } - seq_printf(m, "[0x%08x 0x%08x] size %8d", - i->soffset, i->eoffset, i->eoffset - i->soffset); + seq_printf(m, "[0x%010llx 0x%010llx] size %8lld", + soffset, eoffset, eoffset - soffset); if (i->fence) { seq_printf(m, " protected by 0x%016llx on ring %d", i->fence->seq, i->fence->ring); diff --git a/sys/dev/pci/drm/radeon/radeon_semaphore.c b/sys/dev/pci/drm/radeon/radeon_semaphore.c index 7b07f15bc8c..e6d2f7bdcb1 100644 --- a/sys/dev/pci/drm/radeon/radeon_semaphore.c +++ b/sys/dev/pci/drm/radeon/radeon_semaphore.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_semaphore.c,v 1.5 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2011 Christian König. * All Rights Reserved. @@ -42,7 +41,7 @@ int radeon_semaphore_create(struct radeon_device *rdev, return -ENOMEM; } r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, - &(*semaphore)->sa_bo, 8, 8, true); + &(*semaphore)->sa_bo, 8, 8); if (r) { kfree(*semaphore); *semaphore = NULL; @@ -50,61 +49,44 @@ int radeon_semaphore_create(struct radeon_device *rdev, } (*semaphore)->waiters = 0; (*semaphore)->gpu_addr = radeon_sa_bo_gpu_addr((*semaphore)->sa_bo); - *((uint64_t*)radeon_sa_bo_cpu_addr((*semaphore)->sa_bo)) = 0; + + *((uint64_t *)radeon_sa_bo_cpu_addr((*semaphore)->sa_bo)) = 0; + return 0; } -void radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring, +bool radeon_semaphore_emit_signal(struct radeon_device *rdev, int ridx, struct radeon_semaphore *semaphore) { - trace_radeon_semaphore_signale(ring, semaphore); + struct radeon_ring *ring = &rdev->ring[ridx]; - --semaphore->waiters; - radeon_semaphore_ring_emit(rdev, ring, &rdev->ring[ring], semaphore, false); -} + trace_radeon_semaphore_signale(ridx, semaphore); -void radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring, - struct radeon_semaphore *semaphore) -{ - trace_radeon_semaphore_wait(ring, semaphore); + if (radeon_semaphore_ring_emit(rdev, ridx, ring, semaphore, false)) { + --semaphore->waiters; - ++semaphore->waiters; - radeon_semaphore_ring_emit(rdev, ring, &rdev->ring[ring], semaphore, true); + /* for debugging lockup only, used by sysfs debug files */ + ring->last_semaphore_signal_addr = semaphore->gpu_addr; + return true; + } + return false; } -/* caller must hold ring lock */ -int radeon_semaphore_sync_rings(struct radeon_device *rdev, - struct radeon_semaphore *semaphore, - int signaler, int waiter) +bool radeon_semaphore_emit_wait(struct radeon_device *rdev, int ridx, + struct radeon_semaphore *semaphore) { - int r; + struct radeon_ring *ring = &rdev->ring[ridx]; - /* no need to signal and wait on the same ring */ - if (signaler == waiter) { - return 0; - } + trace_radeon_semaphore_wait(ridx, semaphore); - /* prevent GPU deadlocks */ - if (!rdev->ring[signaler].ready) { - dev_err(rdev->dev, "Trying to sync to a disabled ring!"); - return -EINVAL; - } + if (radeon_semaphore_ring_emit(rdev, ridx, ring, semaphore, true)) { + ++semaphore->waiters; - r = radeon_ring_alloc(rdev, &rdev->ring[signaler], 8); - if (r) { - return r; + /* for debugging lockup only, used by sysfs debug files */ + ring->last_semaphore_wait_addr = semaphore->gpu_addr; + return true; } - radeon_semaphore_emit_signal(rdev, signaler, semaphore); - radeon_ring_commit(rdev, &rdev->ring[signaler]); - - /* we assume caller has already allocated space on waiters ring */ - radeon_semaphore_emit_wait(rdev, waiter, semaphore); - - /* for debugging lockup only, used by sysfs debug files */ - rdev->ring[signaler].last_semaphore_signal_addr = semaphore->gpu_addr; - rdev->ring[waiter].last_semaphore_wait_addr = semaphore->gpu_addr; - - return 0; + return false; } void radeon_semaphore_free(struct radeon_device *rdev, diff --git a/sys/dev/pci/drm/radeon/radeon_sync.c b/sys/dev/pci/drm/radeon/radeon_sync.c new file mode 100644 index 00000000000..668b45c0d9b --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_sync.c @@ -0,0 +1,220 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Christian König + */ + +#include +#include "radeon.h" +#include "radeon_trace.h" + +/** + * radeon_sync_create - zero init sync object + * + * @sync: sync object to initialize + * + * Just clear the sync object for now. + */ +void radeon_sync_create(struct radeon_sync *sync) +{ + unsigned i; + + for (i = 0; i < RADEON_NUM_SYNCS; ++i) + sync->semaphores[i] = NULL; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) + sync->sync_to[i] = NULL; + + sync->last_vm_update = NULL; +} + +/** + * radeon_sync_fence - use the semaphore to sync to a fence + * + * @sync: sync object to add fence to + * @fence: fence to sync to + * + * Sync to the fence using the semaphore objects + */ +void radeon_sync_fence(struct radeon_sync *sync, + struct radeon_fence *fence) +{ + struct radeon_fence *other; + + if (!fence) + return; + + other = sync->sync_to[fence->ring]; + sync->sync_to[fence->ring] = radeon_fence_later(fence, other); + + if (fence->is_vm_update) { + other = sync->last_vm_update; + sync->last_vm_update = radeon_fence_later(fence, other); + } +} + +/** + * radeon_sync_resv - use the semaphores to sync to a reservation object + * + * @sync: sync object to add fences from reservation object to + * @resv: reservation object with embedded fence + * @shared: true if we should only sync to the exclusive fence + * + * Sync to the fence using the semaphore objects + */ +int radeon_sync_resv(struct radeon_device *rdev, + struct radeon_sync *sync, + struct reservation_object *resv, + bool shared) +{ + struct reservation_object_list *flist; + struct fence *f; + struct radeon_fence *fence; + unsigned i; + int r = 0; + + /* always sync to the exclusive fence */ + f = reservation_object_get_excl(resv); + fence = f ? to_radeon_fence(f) : NULL; + if (fence && fence->rdev == rdev) + radeon_sync_fence(sync, fence); + else if (f) + r = fence_wait(f, true); + + flist = reservation_object_get_list(resv); + if (shared || !flist || r) + return r; + + for (i = 0; i < flist->shared_count; ++i) { + f = rcu_dereference_protected(flist->shared[i], + reservation_object_held(resv)); + fence = to_radeon_fence(f); + if (fence && fence->rdev == rdev) + radeon_sync_fence(sync, fence); + else + r = fence_wait(f, true); + + if (r) + break; + } + return r; +} + +/** + * radeon_sync_rings - sync ring to all registered fences + * + * @rdev: radeon_device pointer + * @sync: sync object to use + * @ring: ring that needs sync + * + * Ensure that all registered fences are signaled before letting + * the ring continue. The caller must hold the ring lock. + */ +int radeon_sync_rings(struct radeon_device *rdev, + struct radeon_sync *sync, + int ring) +{ + unsigned count = 0; + int i, r; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + struct radeon_fence *fence = sync->sync_to[i]; + struct radeon_semaphore *semaphore; + + /* check if we really need to sync */ + if (!radeon_fence_need_sync(fence, ring)) + continue; + + /* prevent GPU deadlocks */ + if (!rdev->ring[i].ready) { + dev_err(rdev->dev, "Syncing to a disabled ring!"); + return -EINVAL; + } + + if (count >= RADEON_NUM_SYNCS) { + /* not enough room, wait manually */ + r = radeon_fence_wait(fence, false); + if (r) + return r; + continue; + } + r = radeon_semaphore_create(rdev, &semaphore); + if (r) + return r; + + sync->semaphores[count++] = semaphore; + + /* allocate enough space for sync command */ + r = radeon_ring_alloc(rdev, &rdev->ring[i], 16); + if (r) + return r; + + /* emit the signal semaphore */ + if (!radeon_semaphore_emit_signal(rdev, i, semaphore)) { + /* signaling wasn't successful wait manually */ + radeon_ring_undo(&rdev->ring[i]); + r = radeon_fence_wait(fence, false); + if (r) + return r; + continue; + } + + /* we assume caller has already allocated space on waiters ring */ + if (!radeon_semaphore_emit_wait(rdev, ring, semaphore)) { + /* waiting wasn't successful wait manually */ + radeon_ring_undo(&rdev->ring[i]); + r = radeon_fence_wait(fence, false); + if (r) + return r; + continue; + } + + radeon_ring_commit(rdev, &rdev->ring[i], false); + radeon_fence_note_sync(fence, ring); + } + + return 0; +} + +/** + * radeon_sync_free - free the sync object + * + * @rdev: radeon_device pointer + * @sync: sync object to use + * @fence: fence to use for the free + * + * Free the sync object by freeing all semaphores in it. + */ +void radeon_sync_free(struct radeon_device *rdev, + struct radeon_sync *sync, + struct radeon_fence *fence) +{ + unsigned i; + + for (i = 0; i < RADEON_NUM_SYNCS; ++i) + radeon_semaphore_free(rdev, &sync->semaphores[i], fence); +} diff --git a/sys/dev/pci/drm/radeon/radeon_test.c b/sys/dev/pci/drm/radeon/radeon_test.c index 50a43b0d11f..4ab69640929 100644 --- a/sys/dev/pci/drm/radeon/radeon_test.c +++ b/sys/dev/pci/drm/radeon/radeon_test.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_test.c,v 1.6 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2009 VMware, Inc. * @@ -36,7 +35,6 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) { struct radeon_bo *vram_obj = NULL; struct radeon_bo **gtt_obj = NULL; - struct radeon_fence *fence = NULL; uint64_t gtt_addr, vram_addr; unsigned n, size; int i, r, ring; @@ -58,13 +56,7 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) /* Number of tests = * (Total GTT - IB pool - writeback page - ring buffers) / test size */ - n = rdev->mc.gtt_size - RADEON_IB_POOL_SIZE*64*1024; - for (i = 0; i < RADEON_NUM_RINGS; ++i) - n -= rdev->ring[i].ring_size; - if (rdev->wb.wb_obj) - n -= RADEON_GPU_PAGE_SIZE; - if (rdev->ih.ring_obj) - n -= rdev->ih.ring_size; + n = rdev->mc.gtt_size - rdev->gart_pin_size; n /= size; gtt_obj = kzalloc(n * sizeof(*gtt_obj), GFP_KERNEL); @@ -75,44 +67,46 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) } r = radeon_bo_create(rdev, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM, - NULL, &vram_obj); + 0, NULL, NULL, &vram_obj); if (r) { DRM_ERROR("Failed to create VRAM object\n"); goto out_cleanup; } r = radeon_bo_reserve(vram_obj, false); if (unlikely(r != 0)) - goto out_cleanup; + goto out_unref; r = radeon_bo_pin(vram_obj, RADEON_GEM_DOMAIN_VRAM, &vram_addr); if (r) { DRM_ERROR("Failed to pin VRAM object\n"); - goto out_cleanup; + goto out_unres; } for (i = 0; i < n; i++) { void *gtt_map, *vram_map; void **gtt_start, **gtt_end; void **vram_start, **vram_end; + struct radeon_fence *fence = NULL; r = radeon_bo_create(rdev, size, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_GTT, NULL, gtt_obj + i); + RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL, + gtt_obj + i); if (r) { DRM_ERROR("Failed to create GTT object %d\n", i); - goto out_cleanup; + goto out_lclean; } r = radeon_bo_reserve(gtt_obj[i], false); if (unlikely(r != 0)) - goto out_cleanup; + goto out_lclean_unref; r = radeon_bo_pin(gtt_obj[i], RADEON_GEM_DOMAIN_GTT, >t_addr); if (r) { DRM_ERROR("Failed to pin GTT object %d\n", i); - goto out_cleanup; + goto out_lclean_unres; } r = radeon_bo_kmap(gtt_obj[i], >t_map); if (r) { DRM_ERROR("Failed to map GTT object %d\n", i); - goto out_cleanup; + goto out_lclean_unpin; } for (gtt_start = gtt_map, gtt_end = gtt_map + size; @@ -123,18 +117,23 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) radeon_bo_kunmap(gtt_obj[i]); if (ring == R600_RING_TYPE_DMA_INDEX) - r = radeon_copy_dma(rdev, gtt_addr, vram_addr, size / RADEON_GPU_PAGE_SIZE, &fence); + fence = radeon_copy_dma(rdev, gtt_addr, vram_addr, + size / RADEON_GPU_PAGE_SIZE, + vram_obj->tbo.resv); else - r = radeon_copy_blit(rdev, gtt_addr, vram_addr, size / RADEON_GPU_PAGE_SIZE, &fence); - if (r) { + fence = radeon_copy_blit(rdev, gtt_addr, vram_addr, + size / RADEON_GPU_PAGE_SIZE, + vram_obj->tbo.resv); + if (IS_ERR(fence)) { DRM_ERROR("Failed GTT->VRAM copy %d\n", i); - goto out_cleanup; + r = PTR_ERR(fence); + goto out_lclean_unpin; } r = radeon_fence_wait(fence, false); if (r) { DRM_ERROR("Failed to wait for GTT->VRAM fence %d\n", i); - goto out_cleanup; + goto out_lclean_unpin; } radeon_fence_unref(&fence); @@ -142,7 +141,7 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) r = radeon_bo_kmap(vram_obj, &vram_map); if (r) { DRM_ERROR("Failed to map VRAM object after copy %d\n", i); - goto out_cleanup; + goto out_lclean_unpin; } for (gtt_start = gtt_map, gtt_end = gtt_map + size, @@ -161,7 +160,7 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) (vram_addr - rdev->mc.vram_start + (void*)gtt_start - gtt_map)); radeon_bo_kunmap(vram_obj); - goto out_cleanup; + goto out_lclean_unpin; } *vram_start = vram_start; } @@ -169,18 +168,23 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) radeon_bo_kunmap(vram_obj); if (ring == R600_RING_TYPE_DMA_INDEX) - r = radeon_copy_dma(rdev, vram_addr, gtt_addr, size / RADEON_GPU_PAGE_SIZE, &fence); + fence = radeon_copy_dma(rdev, vram_addr, gtt_addr, + size / RADEON_GPU_PAGE_SIZE, + vram_obj->tbo.resv); else - r = radeon_copy_blit(rdev, vram_addr, gtt_addr, size / RADEON_GPU_PAGE_SIZE, &fence); - if (r) { + fence = radeon_copy_blit(rdev, vram_addr, gtt_addr, + size / RADEON_GPU_PAGE_SIZE, + vram_obj->tbo.resv); + if (IS_ERR(fence)) { DRM_ERROR("Failed VRAM->GTT copy %d\n", i); - goto out_cleanup; + r = PTR_ERR(fence); + goto out_lclean_unpin; } r = radeon_fence_wait(fence, false); if (r) { DRM_ERROR("Failed to wait for VRAM->GTT fence %d\n", i); - goto out_cleanup; + goto out_lclean_unpin; } radeon_fence_unref(&fence); @@ -188,7 +192,7 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) r = radeon_bo_kmap(gtt_obj[i], >t_map); if (r) { DRM_ERROR("Failed to map GTT object after copy %d\n", i); - goto out_cleanup; + goto out_lclean_unpin; } for (gtt_start = gtt_map, gtt_end = gtt_map + size, @@ -207,7 +211,7 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) (gtt_addr - rdev->mc.gtt_start + (void*)vram_start - vram_map)); radeon_bo_kunmap(gtt_obj[i]); - goto out_cleanup; + goto out_lclean_unpin; } } @@ -215,31 +219,32 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag) DRM_INFO("Tested GTT->VRAM and VRAM->GTT copy for GTT offset 0x%llx\n", gtt_addr - rdev->mc.gtt_start); + continue; + +out_lclean_unpin: + radeon_bo_unpin(gtt_obj[i]); +out_lclean_unres: + radeon_bo_unreserve(gtt_obj[i]); +out_lclean_unref: + radeon_bo_unref(>t_obj[i]); +out_lclean: + for (--i; i >= 0; --i) { + radeon_bo_unpin(gtt_obj[i]); + radeon_bo_unreserve(gtt_obj[i]); + radeon_bo_unref(>t_obj[i]); + } + if (fence && !IS_ERR(fence)) + radeon_fence_unref(&fence); + break; } + radeon_bo_unpin(vram_obj); +out_unres: + radeon_bo_unreserve(vram_obj); +out_unref: + radeon_bo_unref(&vram_obj); out_cleanup: - if (vram_obj) { - if (radeon_bo_is_reserved(vram_obj)) { - radeon_bo_unpin(vram_obj); - radeon_bo_unreserve(vram_obj); - } - radeon_bo_unref(&vram_obj); - } - if (gtt_obj) { - for (i = 0; i < n; i++) { - if (gtt_obj[i]) { - if (radeon_bo_is_reserved(gtt_obj[i])) { - radeon_bo_unpin(gtt_obj[i]); - radeon_bo_unreserve(gtt_obj[i]); - } - radeon_bo_unref(>t_obj[i]); - } - } - kfree(gtt_obj); - } - if (fence) { - radeon_fence_unref(&fence); - } + kfree(gtt_obj); if (r) { printk(KERN_WARNING "Error while testing BO move.\n"); } @@ -253,6 +258,52 @@ void radeon_test_moves(struct radeon_device *rdev) radeon_do_test_moves(rdev, RADEON_TEST_COPY_BLIT); } +static int radeon_test_create_and_emit_fence(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_fence **fence) +{ + uint32_t handle = ring->idx ^ 0xdeafbeef; + int r; + + if (ring->idx == R600_RING_TYPE_UVD_INDEX) { + r = radeon_uvd_get_create_msg(rdev, ring->idx, handle, NULL); + if (r) { + DRM_ERROR("Failed to get dummy create msg\n"); + return r; + } + + r = radeon_uvd_get_destroy_msg(rdev, ring->idx, handle, fence); + if (r) { + DRM_ERROR("Failed to get dummy destroy msg\n"); + return r; + } + + } else if (ring->idx == TN_RING_TYPE_VCE1_INDEX || + ring->idx == TN_RING_TYPE_VCE2_INDEX) { + r = radeon_vce_get_create_msg(rdev, ring->idx, handle, NULL); + if (r) { + DRM_ERROR("Failed to get dummy create msg\n"); + return r; + } + + r = radeon_vce_get_destroy_msg(rdev, ring->idx, handle, fence); + if (r) { + DRM_ERROR("Failed to get dummy destroy msg\n"); + return r; + } + + } else { + r = radeon_ring_lock(rdev, ring, 64); + if (r) { + DRM_ERROR("Failed to lock ring A %d\n", ring->idx); + return r; + } + radeon_fence_emit(rdev, fence, ring->idx); + radeon_ring_unlock_commit(rdev, ring, false); + } + return 0; +} + void radeon_test_ring_sync(struct radeon_device *rdev, struct radeon_ring *ringA, struct radeon_ring *ringB) @@ -273,20 +324,23 @@ void radeon_test_ring_sync(struct radeon_device *rdev, goto out_cleanup; } radeon_semaphore_emit_wait(rdev, ringA->idx, semaphore); - r = radeon_fence_emit(rdev, &fence1, ringA->idx); + radeon_ring_unlock_commit(rdev, ringA, false); + + r = radeon_test_create_and_emit_fence(rdev, ringA, &fence1); + if (r) + goto out_cleanup; + + r = radeon_ring_lock(rdev, ringA, 64); if (r) { - DRM_ERROR("Failed to emit fence 1\n"); - radeon_ring_unlock_undo(rdev, ringA); + DRM_ERROR("Failed to lock ring A %d\n", ringA->idx); goto out_cleanup; } radeon_semaphore_emit_wait(rdev, ringA->idx, semaphore); - r = radeon_fence_emit(rdev, &fence2, ringA->idx); - if (r) { - DRM_ERROR("Failed to emit fence 2\n"); - radeon_ring_unlock_undo(rdev, ringA); + radeon_ring_unlock_commit(rdev, ringA, false); + + r = radeon_test_create_and_emit_fence(rdev, ringA, &fence2); + if (r) goto out_cleanup; - } - radeon_ring_unlock_commit(rdev, ringA); mdelay(1000); @@ -301,7 +355,7 @@ void radeon_test_ring_sync(struct radeon_device *rdev, goto out_cleanup; } radeon_semaphore_emit_signal(rdev, ringB->idx, semaphore); - radeon_ring_unlock_commit(rdev, ringB); + radeon_ring_unlock_commit(rdev, ringB, false); r = radeon_fence_wait(fence1, false); if (r) { @@ -322,7 +376,7 @@ void radeon_test_ring_sync(struct radeon_device *rdev, goto out_cleanup; } radeon_semaphore_emit_signal(rdev, ringB->idx, semaphore); - radeon_ring_unlock_commit(rdev, ringB); + radeon_ring_unlock_commit(rdev, ringB, false); r = radeon_fence_wait(fence2, false); if (r) { @@ -365,13 +419,11 @@ static void radeon_test_ring_sync2(struct radeon_device *rdev, goto out_cleanup; } radeon_semaphore_emit_wait(rdev, ringA->idx, semaphore); - r = radeon_fence_emit(rdev, &fenceA, ringA->idx); - if (r) { - DRM_ERROR("Failed to emit sync fence 1\n"); - radeon_ring_unlock_undo(rdev, ringA); + radeon_ring_unlock_commit(rdev, ringA, false); + + r = radeon_test_create_and_emit_fence(rdev, ringA, &fenceA); + if (r) goto out_cleanup; - } - radeon_ring_unlock_commit(rdev, ringA); r = radeon_ring_lock(rdev, ringB, 64); if (r) { @@ -379,13 +431,10 @@ static void radeon_test_ring_sync2(struct radeon_device *rdev, goto out_cleanup; } radeon_semaphore_emit_wait(rdev, ringB->idx, semaphore); - r = radeon_fence_emit(rdev, &fenceB, ringB->idx); - if (r) { - DRM_ERROR("Failed to create sync fence 2\n"); - radeon_ring_unlock_undo(rdev, ringB); + radeon_ring_unlock_commit(rdev, ringB, false); + r = radeon_test_create_and_emit_fence(rdev, ringB, &fenceB); + if (r) goto out_cleanup; - } - radeon_ring_unlock_commit(rdev, ringB); mdelay(1000); @@ -394,7 +443,7 @@ static void radeon_test_ring_sync2(struct radeon_device *rdev, goto out_cleanup; } if (radeon_fence_signaled(fenceB)) { - DRM_ERROR("Fence A signaled without waiting for semaphore.\n"); + DRM_ERROR("Fence B signaled without waiting for semaphore.\n"); goto out_cleanup; } @@ -404,7 +453,7 @@ static void radeon_test_ring_sync2(struct radeon_device *rdev, goto out_cleanup; } radeon_semaphore_emit_signal(rdev, ringC->idx, semaphore); - radeon_ring_unlock_commit(rdev, ringC); + radeon_ring_unlock_commit(rdev, ringC, false); for (i = 0; i < 30; ++i) { mdelay(100); @@ -430,7 +479,7 @@ static void radeon_test_ring_sync2(struct radeon_device *rdev, goto out_cleanup; } radeon_semaphore_emit_signal(rdev, ringC->idx, semaphore); - radeon_ring_unlock_commit(rdev, ringC); + radeon_ring_unlock_commit(rdev, ringC, false); mdelay(1000); @@ -458,6 +507,16 @@ out_cleanup: printk(KERN_WARNING "Error while testing ring sync (%d).\n", r); } +static bool radeon_test_sync_possible(struct radeon_ring *ringA, + struct radeon_ring *ringB) +{ + if (ringA->idx == TN_RING_TYPE_VCE2_INDEX && + ringB->idx == TN_RING_TYPE_VCE1_INDEX) + return false; + + return true; +} + void radeon_test_syncing(struct radeon_device *rdev) { int i, j, k; @@ -472,6 +531,9 @@ void radeon_test_syncing(struct radeon_device *rdev) if (!ringB->ready) continue; + if (!radeon_test_sync_possible(ringA, ringB)) + continue; + DRM_INFO("Testing syncing between rings %d and %d...\n", i, j); radeon_test_ring_sync(rdev, ringA, ringB); @@ -483,6 +545,12 @@ void radeon_test_syncing(struct radeon_device *rdev) if (!ringC->ready) continue; + if (!radeon_test_sync_possible(ringA, ringC)) + continue; + + if (!radeon_test_sync_possible(ringB, ringC)) + continue; + DRM_INFO("Testing syncing between rings %d, %d and %d...\n", i, j, k); radeon_test_ring_sync2(rdev, ringA, ringB, ringC); diff --git a/sys/dev/pci/drm/radeon/radeon_trace.h b/sys/dev/pci/drm/radeon/radeon_trace.h index 4377842f744..5bb2cd121c4 100644 --- a/sys/dev/pci/drm/radeon/radeon_trace.h +++ b/sys/dev/pci/drm/radeon/radeon_trace.h @@ -1,5 +1,3 @@ -/* $OpenBSD: radeon_trace.h,v 1.4 2018/04/20 16:09:37 deraadt Exp $ */ - #if !defined(_RADEON_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) #define _RADEON_TRACE_H_ @@ -7,7 +5,6 @@ #undef TRACE_SYSTEM #define TRACE_SYSTEM radeon -#define TRACE_SYSTEM_STRING __stringify(TRACE_SYSTEM) #define TRACE_INCLUDE_FILE radeon_trace TRACE_EVENT(radeon_bo_create, @@ -25,51 +22,142 @@ TRACE_EVENT(radeon_bo_create, TP_printk("bo=%p, pages=%u", __entry->bo, __entry->pages) ); +TRACE_EVENT(radeon_cs, + TP_PROTO(struct radeon_cs_parser *p), + TP_ARGS(p), + TP_STRUCT__entry( + __field(u32, ring) + __field(u32, dw) + __field(u32, fences) + ), + + TP_fast_assign( + __entry->ring = p->ring; + __entry->dw = p->chunk_ib->length_dw; + __entry->fences = radeon_fence_count_emitted( + p->rdev, p->ring); + ), + TP_printk("ring=%u, dw=%u, fences=%u", + __entry->ring, __entry->dw, + __entry->fences) +); + +TRACE_EVENT(radeon_vm_grab_id, + TP_PROTO(unsigned vmid, int ring), + TP_ARGS(vmid, ring), + TP_STRUCT__entry( + __field(u32, vmid) + __field(u32, ring) + ), + + TP_fast_assign( + __entry->vmid = vmid; + __entry->ring = ring; + ), + TP_printk("vmid=%u, ring=%u", __entry->vmid, __entry->ring) +); + +TRACE_EVENT(radeon_vm_bo_update, + TP_PROTO(struct radeon_bo_va *bo_va), + TP_ARGS(bo_va), + TP_STRUCT__entry( + __field(u64, soffset) + __field(u64, eoffset) + __field(u32, flags) + ), + + TP_fast_assign( + __entry->soffset = bo_va->it.start; + __entry->eoffset = bo_va->it.last + 1; + __entry->flags = bo_va->flags; + ), + TP_printk("soffs=%010llx, eoffs=%010llx, flags=%08x", + __entry->soffset, __entry->eoffset, __entry->flags) +); + +TRACE_EVENT(radeon_vm_set_page, + TP_PROTO(uint64_t pe, uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags), + TP_ARGS(pe, addr, count, incr, flags), + TP_STRUCT__entry( + __field(u64, pe) + __field(u64, addr) + __field(u32, count) + __field(u32, incr) + __field(u32, flags) + ), + + TP_fast_assign( + __entry->pe = pe; + __entry->addr = addr; + __entry->count = count; + __entry->incr = incr; + __entry->flags = flags; + ), + TP_printk("pe=%010Lx, addr=%010Lx, incr=%u, flags=%08x, count=%u", + __entry->pe, __entry->addr, __entry->incr, + __entry->flags, __entry->count) +); + +TRACE_EVENT(radeon_vm_flush, + TP_PROTO(uint64_t pd_addr, unsigned ring, unsigned id), + TP_ARGS(pd_addr, ring, id), + TP_STRUCT__entry( + __field(u64, pd_addr) + __field(u32, ring) + __field(u32, id) + ), + + TP_fast_assign( + __entry->pd_addr = pd_addr; + __entry->ring = ring; + __entry->id = id; + ), + TP_printk("pd_addr=%010Lx, ring=%u, id=%u", + __entry->pd_addr, __entry->ring, __entry->id) +); + DECLARE_EVENT_CLASS(radeon_fence_request, - TP_PROTO(struct drm_device *dev, u32 seqno), + TP_PROTO(struct drm_device *dev, int ring, u32 seqno), - TP_ARGS(dev, seqno), + TP_ARGS(dev, ring, seqno), TP_STRUCT__entry( __field(u32, dev) + __field(int, ring) __field(u32, seqno) ), TP_fast_assign( __entry->dev = dev->primary->index; + __entry->ring = ring; __entry->seqno = seqno; ), - TP_printk("dev=%u, seqno=%u", __entry->dev, __entry->seqno) + TP_printk("dev=%u, ring=%d, seqno=%u", + __entry->dev, __entry->ring, __entry->seqno) ); DEFINE_EVENT(radeon_fence_request, radeon_fence_emit, - TP_PROTO(struct drm_device *dev, u32 seqno), - - TP_ARGS(dev, seqno) -); - -DEFINE_EVENT(radeon_fence_request, radeon_fence_retire, - - TP_PROTO(struct drm_device *dev, u32 seqno), + TP_PROTO(struct drm_device *dev, int ring, u32 seqno), - TP_ARGS(dev, seqno) + TP_ARGS(dev, ring, seqno) ); DEFINE_EVENT(radeon_fence_request, radeon_fence_wait_begin, - TP_PROTO(struct drm_device *dev, u32 seqno), + TP_PROTO(struct drm_device *dev, int ring, u32 seqno), - TP_ARGS(dev, seqno) + TP_ARGS(dev, ring, seqno) ); DEFINE_EVENT(radeon_fence_request, radeon_fence_wait_end, - TP_PROTO(struct drm_device *dev, u32 seqno), + TP_PROTO(struct drm_device *dev, int ring, u32 seqno), - TP_ARGS(dev, seqno) + TP_ARGS(dev, ring, seqno) ); DECLARE_EVENT_CLASS(radeon_semaphore_request, diff --git a/sys/dev/pci/drm/radeon/radeon_trace_points.c b/sys/dev/pci/drm/radeon/radeon_trace_points.c index 4bef278ba12..440e092f4a6 100644 --- a/sys/dev/pci/drm/radeon/radeon_trace_points.c +++ b/sys/dev/pci/drm/radeon/radeon_trace_points.c @@ -1,5 +1,3 @@ -/* $OpenBSD: radeon_trace_points.c,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ - /* Copyright Red Hat Inc 2010. * Author : Dave Airlie */ diff --git a/sys/dev/pci/drm/radeon/radeon_ttm.c b/sys/dev/pci/drm/radeon/radeon_ttm.c index e06d4dc256b..0ce9fed2931 100644 --- a/sys/dev/pci/drm/radeon/radeon_ttm.c +++ b/sys/dev/pci/drm/radeon/radeon_ttm.c @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_ttm.c,v 1.14 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2009 Jerome Glisse. * All Rights Reserved. @@ -30,12 +29,12 @@ * Thomas Hellstrom * Dave Airlie */ +#include #include #include #include #include #include -#include #include #include "radeon_reg.h" #include "radeon.h" @@ -43,6 +42,7 @@ #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) static int radeon_ttm_debugfs_init(struct radeon_device *rdev); +static void radeon_ttm_debugfs_fini(struct radeon_device *rdev); static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) { @@ -140,13 +140,11 @@ static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; #if __OS_HAS_AGP if (rdev->flags & RADEON_IS_AGP) { -#ifdef notyet - if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) { + if (!rdev->ddev->agp) { DRM_ERROR("AGP is not enabled for memory type %u\n", (unsigned)type); return -EINVAL; } -#endif if (!rdev->ddev->agp->cant_use_aperture) man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_FLAG_UNCACHED | @@ -174,12 +172,15 @@ static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, static void radeon_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *placement) { + static struct ttm_place placements = { + .fpfn = 0, + .lpfn = 0, + .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM + }; + struct radeon_bo *rbo; - static u32 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; if (!radeon_ttm_bo_is_radeon_bo(bo)) { - placement->fpfn = 0; - placement->lpfn = 0; placement->placement = &placements; placement->busy_placement = &placements; placement->num_placement = 1; @@ -189,9 +190,32 @@ static void radeon_evict_flags(struct ttm_buffer_object *bo, rbo = container_of(bo, struct radeon_bo, tbo); switch (bo->mem.mem_type) { case TTM_PL_VRAM: - if (rbo->rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready == false) + if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); - else + else if (rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size && + bo->mem.start < (rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT)) { + unsigned fpfn = rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; + int i; + + /* Try evicting to the CPU inaccessible part of VRAM + * first, but only set GTT as busy placement, so this + * BO will be evicted to GTT rather than causing other + * BOs to be evicted from VRAM + */ + radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM | + RADEON_GEM_DOMAIN_GTT); + rbo->placement.num_busy_placement = 0; + for (i = 0; i < rbo->placement.num_placement; i++) { + if (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) { + if (rbo->placements[i].fpfn < fpfn) + rbo->placements[i].fpfn = fpfn; + } else { + rbo->placement.busy_placement = + &rbo->placements[i]; + rbo->placement.num_busy_placement = 1; + } + } + } else radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); break; case TTM_PL_TT: @@ -203,7 +227,11 @@ static void radeon_evict_flags(struct ttm_buffer_object *bo, static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp) { - return 0; + struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo); + + if (radeon_ttm_tt_has_userptr(bo->ttm)) + return -EPERM; + return drm_vma_node_verify_access(&rbo->gem_base.vma_node, filp); } static void radeon_move_null(struct ttm_buffer_object *bo, @@ -224,12 +252,13 @@ static int radeon_move_blit(struct ttm_buffer_object *bo, struct radeon_device *rdev; uint64_t old_start, new_start; struct radeon_fence *fence; + unsigned num_pages; int r, ridx; rdev = radeon_get_rdev(bo->bdev); ridx = radeon_copy_ring_index(rdev); - old_start = old_mem->start << PAGE_SHIFT; - new_start = new_mem->start << PAGE_SHIFT; + old_start = (u64)old_mem->start << PAGE_SHIFT; + new_start = (u64)new_mem->start << PAGE_SHIFT; switch (old_mem->mem_type) { case TTM_PL_VRAM: @@ -260,13 +289,12 @@ static int radeon_move_blit(struct ttm_buffer_object *bo, BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); - /* sync other rings */ - fence = bo->sync_obj; - r = radeon_copy(rdev, old_start, new_start, - new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ - &fence); - /* FIXME: handle copy error */ - r = ttm_bo_move_accel_cleanup(bo, (void *)fence, + num_pages = new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); + fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->resv); + if (IS_ERR(fence)) + return PTR_ERR(fence); + + r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, no_wait_gpu, new_mem); radeon_fence_unref(&fence); return r; @@ -280,20 +308,20 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo, struct radeon_device *rdev; struct ttm_mem_reg *old_mem = &bo->mem; struct ttm_mem_reg tmp_mem; - u32 placements; + struct ttm_place placements; struct ttm_placement placement; int r; rdev = radeon_get_rdev(bo->bdev); tmp_mem = *new_mem; tmp_mem.mm_node = NULL; - placement.fpfn = 0; - placement.lpfn = 0; placement.num_placement = 1; placement.placement = &placements; placement.num_busy_placement = 1; placement.busy_placement = &placements; - placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; + placements.fpfn = 0; + placements.lpfn = 0; + placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_gpu); if (unlikely(r)) { @@ -328,19 +356,19 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo, struct ttm_mem_reg *old_mem = &bo->mem; struct ttm_mem_reg tmp_mem; struct ttm_placement placement; - u32 placements; + struct ttm_place placements; int r; rdev = radeon_get_rdev(bo->bdev); tmp_mem = *new_mem; tmp_mem.mm_node = NULL; - placement.fpfn = 0; - placement.lpfn = 0; placement.num_placement = 1; placement.placement = &placements; placement.num_busy_placement = 1; placement.busy_placement = &placements; - placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; + placements.fpfn = 0; + placements.lpfn = 0; + placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_gpu); if (unlikely(r)) { @@ -402,8 +430,14 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, if (r) { memcpy: r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); + if (r) { + return r; + } } - return r; + + /* update statistics */ + atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); + return 0; } static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) @@ -473,55 +507,143 @@ static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_re { } -static int radeon_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible) -{ - return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible); -} - -static int radeon_sync_obj_flush(void *sync_obj) -{ - return 0; -} - -static void radeon_sync_obj_unref(void **sync_obj) -{ - radeon_fence_unref((struct radeon_fence **)sync_obj); -} - -static void *radeon_sync_obj_ref(void *sync_obj) -{ - return radeon_fence_ref((struct radeon_fence *)sync_obj); -} - -static bool radeon_sync_obj_signaled(void *sync_obj) -{ - return radeon_fence_signaled((struct radeon_fence *)sync_obj); -} - /* * TTM backend functions. */ struct radeon_ttm_tt { struct ttm_dma_tt ttm; struct radeon_device *rdev; + u64 offset; + + uint64_t userptr; + struct mm_struct *usermm; + uint32_t userflags; + bus_dmamap_t map; bus_dma_segment_t *segs; - u64 offset; }; +/* prepare the sg table with the user pages */ +static int radeon_ttm_tt_pin_userptr(struct ttm_tt *ttm) +{ + STUB(); + return -ENOSYS; +#ifdef notyet + struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); + struct radeon_ttm_tt *gtt = (void *)ttm; + unsigned pinned = 0, nents; + int r; + + int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); + enum dma_data_direction direction = write ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + + if (current->mm != gtt->usermm) + return -EPERM; + + if (gtt->userflags & RADEON_GEM_USERPTR_ANONONLY) { + /* check that we only pin down anonymous memory + to prevent problems with writeback */ + unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE; + struct vm_area_struct *vma; + vma = find_vma(gtt->usermm, gtt->userptr); + if (!vma || vma->vm_file || vma->vm_end < end) + return -EPERM; + } + + do { + unsigned num_pages = ttm->num_pages - pinned; + uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; + struct vm_page **pages = ttm->pages + pinned; + + r = get_user_pages(current, current->mm, userptr, num_pages, + write, 0, pages, NULL); + if (r < 0) + goto release_pages; + + pinned += r; + + } while (pinned < ttm->num_pages); + + r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, + ttm->num_pages << PAGE_SHIFT, + GFP_KERNEL); + if (r) + goto release_sg; + + r = -ENOMEM; + nents = dma_map_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction); + if (nents != ttm->sg->nents) + goto release_sg; + + drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, + gtt->ttm.dma_address, ttm->num_pages); + + return 0; + +release_sg: + kfree(ttm->sg); + +release_pages: + release_pages(ttm->pages, pinned, 0); + return r; +#endif +} + +static void radeon_ttm_tt_unpin_userptr(struct ttm_tt *ttm) +{ + STUB(); +#ifdef notyet + struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); + struct radeon_ttm_tt *gtt = (void *)ttm; + struct sg_page_iter sg_iter; + + int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); + enum dma_data_direction direction = write ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + + /* double check that we don't free the table twice */ + if (!ttm->sg->sgl) + return; + + /* free the sg table and pages again */ + dma_unmap_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction); + + for_each_sg_page(ttm->sg->sgl, &sg_iter, ttm->sg->nents, 0) { + struct vm_page *page = sg_page_iter_page(&sg_iter); + if (!(gtt->userflags & RADEON_GEM_USERPTR_READONLY)) + set_page_dirty(page); + + mark_page_accessed(page); + page_cache_release(page); + } + + sg_free_table(ttm->sg); +#endif +} + static int radeon_ttm_backend_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) { struct radeon_ttm_tt *gtt = (void*)ttm; + uint32_t flags = RADEON_GART_PAGE_VALID | RADEON_GART_PAGE_READ | + RADEON_GART_PAGE_WRITE; int r; + if (gtt->userptr) { + radeon_ttm_tt_pin_userptr(ttm); + flags &= ~RADEON_GART_PAGE_WRITE; + } + gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); if (!ttm->num_pages) { WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", ttm->num_pages, bo_mem, ttm); } - r = radeon_gart_bind(gtt->rdev, gtt->offset, - ttm->num_pages, ttm->pages, gtt->ttm.dma_address); + if (ttm->caching_state == tt_cached) + flags |= RADEON_GART_PAGE_SNOOP; + r = radeon_gart_bind(gtt->rdev, gtt->offset, ttm->num_pages, + ttm->pages, gtt->ttm.dma_address, flags); if (r) { DRM_ERROR("failed to bind %lu pages at 0x%08X\n", ttm->num_pages, (unsigned)gtt->offset); @@ -535,6 +657,10 @@ static int radeon_ttm_backend_unbind(struct ttm_tt *ttm) struct radeon_ttm_tt *gtt = (void *)ttm; radeon_gart_unbind(gtt->rdev, gtt->offset, ttm->num_pages); + + if (gtt->userptr) + radeon_ttm_tt_unpin_userptr(ttm); + return 0; } @@ -594,17 +720,35 @@ static struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev, return >t->ttm.ttm; } +static struct radeon_ttm_tt *radeon_ttm_tt_to_gtt(struct ttm_tt *ttm) +{ + if (!ttm || ttm->func != &radeon_backend_func) + return NULL; + return (struct radeon_ttm_tt *)ttm; +} + static int radeon_ttm_tt_populate(struct ttm_tt *ttm) { + struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); struct radeon_device *rdev; - struct radeon_ttm_tt *gtt = (void *)ttm; unsigned i; - int r, seg; + int r; + int seg; bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); if (ttm->state != tt_unpopulated) return 0; + if (gtt && gtt->userptr) { + ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); + if (!ttm->sg) + return -ENOMEM; + + ttm->page_flags |= TTM_PAGE_FLAG_SG; + ttm->state = tt_unbound; + return 0; + } + if (slave && ttm->sg) { #ifdef notyet drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, @@ -632,6 +776,22 @@ static int radeon_ttm_tt_populate(struct ttm_tt *ttm) return r; } +#ifdef __linux__ + for (i = 0; i < ttm->num_pages; i++) { + gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i], + 0, PAGE_SIZE, + PCI_DMA_BIDIRECTIONAL); + if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) { + while (i--) { + pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], + PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + gtt->ttm.dma_address[i] = 0; + } + ttm_pool_unpopulate(ttm); + return -EFAULT; + } + } +#else for (i = 0; i < ttm->num_pages; i++) { gtt->segs[i].ds_addr = VM_PAGE_TO_PHYS(ttm->pages[i]); gtt->segs[i].ds_len = PAGE_SIZE; @@ -643,6 +803,7 @@ static int radeon_ttm_tt_populate(struct ttm_tt *ttm) ttm_pool_unpopulate(ttm); return -EFAULT; } +#endif for (seg = 0, i = 0; seg < gtt->map->dm_nsegs; seg++) { bus_addr_t addr = gtt->map->dm_segs[seg].ds_addr; @@ -661,10 +822,16 @@ static int radeon_ttm_tt_populate(struct ttm_tt *ttm) static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) { struct radeon_device *rdev; - struct radeon_ttm_tt *gtt = (void *)ttm; + struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); unsigned i; bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); + if (gtt && gtt->userptr) { + kfree(ttm->sg); + ttm->page_flags &= ~TTM_PAGE_FLAG_SG; + return; + } + if (slave) return; @@ -683,13 +850,61 @@ static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) } #endif +#ifdef __linux__ + for (i = 0; i < ttm->num_pages; i++) { + if (gtt->ttm.dma_address[i]) { + pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], + PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + } + } +#else bus_dmamap_unload(rdev->dmat, gtt->map); for (i = 0; i < ttm->num_pages; i++) gtt->ttm.dma_address[i] = 0; +#endif + ttm_pool_unpopulate(ttm); } +int radeon_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, + uint32_t flags) +{ + STUB(); + return -ENOSYS; +#ifdef notyet + struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); + + if (gtt == NULL) + return -EINVAL; + + gtt->userptr = addr; + gtt->usermm = current->mm; + gtt->userflags = flags; + return 0; +#endif +} + +bool radeon_ttm_tt_has_userptr(struct ttm_tt *ttm) +{ + struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); + + if (gtt == NULL) + return false; + + return !!gtt->userptr; +} + +bool radeon_ttm_tt_is_readonly(struct ttm_tt *ttm) +{ + struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); + + if (gtt == NULL) + return false; + + return !!(gtt->userflags & RADEON_GEM_USERPTR_READONLY); +} + static struct ttm_bo_driver radeon_bo_driver = { .ttm_tt_create = &radeon_ttm_tt_create, .ttm_tt_populate = &radeon_ttm_tt_populate, @@ -699,11 +914,6 @@ static struct ttm_bo_driver radeon_bo_driver = { .evict_flags = &radeon_evict_flags, .move = &radeon_bo_move, .verify_access = &radeon_verify_access, - .sync_obj_signaled = &radeon_sync_obj_signaled, - .sync_obj_wait = &radeon_sync_obj_wait, - .sync_obj_flush = &radeon_sync_obj_flush, - .sync_obj_unref = &radeon_sync_obj_unref, - .sync_obj_ref = &radeon_sync_obj_ref, .move_notify = &radeon_bo_move_notify, .fault_reserve_notify = &radeon_bo_fault_reserve_notify, .io_mem_reserve = &radeon_ttm_io_mem_reserve, @@ -719,10 +929,21 @@ int radeon_ttm_init(struct radeon_device *rdev) return r; } /* No others user of address space so set it to 0 */ +#ifdef notyet r = ttm_bo_device_init(&rdev->mman.bdev, rdev->mman.bo_global_ref.ref.object, - &radeon_bo_driver, DRM_FILE_PAGE_OFFSET, + &radeon_bo_driver, + rdev->ddev->anon_inode->i_mapping, + DRM_FILE_PAGE_OFFSET, rdev->need_dma32); +#else + r = ttm_bo_device_init(&rdev->mman.bdev, + rdev->mman.bo_global_ref.ref.object, + &radeon_bo_driver, + /*rdev->ddev->anon_inode->i_mapping*/ NULL, + DRM_FILE_PAGE_OFFSET, + rdev->need_dma32); +#endif if (r) { DRM_ERROR("failed initializing buffer object driver(%d).\n", r); return r; @@ -742,11 +963,11 @@ int radeon_ttm_init(struct radeon_device *rdev) #ifdef __sparc64__ r = radeon_bo_create(rdev, rdev->fb_offset, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_VRAM, + RADEON_GEM_DOMAIN_VRAM, 0, NULL, NULL, &rdev->stollen_vga_memory); #else r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_VRAM, + RADEON_GEM_DOMAIN_VRAM, 0, NULL, NULL, &rdev->stollen_vga_memory); #endif if (r) { @@ -771,9 +992,6 @@ int radeon_ttm_init(struct radeon_device *rdev) } DRM_INFO("radeon: %uM of GTT memory ready.\n", (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); -#ifdef notyet - rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping; -#endif r = radeon_ttm_debugfs_init(rdev); if (r) { @@ -789,6 +1007,7 @@ void radeon_ttm_fini(struct radeon_device *rdev) if (!rdev->mman.initialized) return; + radeon_ttm_debugfs_fini(rdev); if (rdev->stollen_vga_memory) { r = radeon_bo_reserve(rdev->stollen_vga_memory, false); if (r == 0) { @@ -820,6 +1039,56 @@ void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) man->size = size >> PAGE_SHIFT; } +#ifdef __linux__ +static struct vm_operations_struct radeon_ttm_vm_ops; +static const struct vm_operations_struct *ttm_vm_ops = NULL; + +static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct ttm_buffer_object *bo; + struct radeon_device *rdev; + int r; + + bo = (struct ttm_buffer_object *)vma->vm_private_data; + if (bo == NULL) { + return VM_FAULT_NOPAGE; + } + rdev = radeon_get_rdev(bo->bdev); + down_read(&rdev->pm.mclk_lock); + r = ttm_vm_ops->fault(vma, vmf); + up_read(&rdev->pm.mclk_lock); + return r; +} + +int radeon_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct drm_file *file_priv; + struct radeon_device *rdev; + int r; + + if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) { + return -EINVAL; + } + + file_priv = filp->private_data; + rdev = file_priv->minor->dev->dev_private; + if (rdev == NULL) { + return -EINVAL; + } + r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); + if (unlikely(r != 0)) { + return r; + } + if (unlikely(ttm_vm_ops == NULL)) { + ttm_vm_ops = vma->vm_ops; + radeon_ttm_vm_ops = *ttm_vm_ops; + radeon_ttm_vm_ops.fault = &radeon_ttm_fault; + } + vma->vm_ops = &radeon_ttm_vm_ops; + return 0; +} +#else + static struct uvm_pagerops radeon_ttm_vm_ops; static const struct uvm_pagerops *ttm_vm_ops = NULL; @@ -869,17 +1138,17 @@ radeon_mmap(struct drm_device *dev, voff_t off, vsize_t size) uobj->pgops = &radeon_ttm_vm_ops; return uobj; } - - -#define RADEON_DEBUGFS_MEM_TYPES 2 +#endif #if defined(CONFIG_DEBUG_FS) + static int radeon_mm_dump_table(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *)m->private; - struct drm_mm *mm = (struct drm_mm *)node->info_ent->data; + unsigned ttm_pl = *(int *)node->info_ent->data; struct drm_device *dev = node->minor->dev; struct radeon_device *rdev = dev->dev_private; + struct drm_mm *mm = (struct drm_mm *)rdev->mman.bdev.man[ttm_pl].priv; int ret; struct ttm_bo_global *glob = rdev->mman.bdev.glob; @@ -888,46 +1157,169 @@ static int radeon_mm_dump_table(struct seq_file *m, void *data) spin_unlock(&glob->lru_lock); return ret; } + +static int ttm_pl_vram = TTM_PL_VRAM; +static int ttm_pl_tt = TTM_PL_TT; + +static struct drm_info_list radeon_ttm_debugfs_list[] = { + {"radeon_vram_mm", radeon_mm_dump_table, 0, &ttm_pl_vram}, + {"radeon_gtt_mm", radeon_mm_dump_table, 0, &ttm_pl_tt}, + {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL}, +#ifdef CONFIG_SWIOTLB + {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL} +#endif +}; + +static int radeon_ttm_vram_open(struct inode *inode, struct file *filep) +{ + struct radeon_device *rdev = inode->i_private; + i_size_write(inode, rdev->mc.mc_vram_size); + filep->private_data = inode->i_private; + return 0; +} + +static ssize_t radeon_ttm_vram_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct radeon_device *rdev = f->private_data; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + while (size) { + unsigned long flags; + uint32_t value; + + if (*pos >= rdev->mc.mc_vram_size) + return result; + + spin_lock_irqsave(&rdev->mmio_idx_lock, flags); + WREG32(RADEON_MM_INDEX, ((uint32_t)*pos) | 0x80000000); + if (rdev->family >= CHIP_CEDAR) + WREG32(EVERGREEN_MM_INDEX_HI, *pos >> 31); + value = RREG32(RADEON_MM_DATA); + spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); + + r = put_user(value, (uint32_t *)buf); + if (r) + return r; + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + return result; +} + +static const struct file_operations radeon_ttm_vram_fops = { + .owner = THIS_MODULE, + .open = radeon_ttm_vram_open, + .read = radeon_ttm_vram_read, + .llseek = default_llseek +}; + +static int radeon_ttm_gtt_open(struct inode *inode, struct file *filep) +{ + struct radeon_device *rdev = inode->i_private; + i_size_write(inode, rdev->mc.gtt_size); + filep->private_data = inode->i_private; + return 0; +} + +static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct radeon_device *rdev = f->private_data; + ssize_t result = 0; + int r; + + while (size) { + loff_t p = *pos / PAGE_SIZE; + unsigned off = *pos & PAGE_MASK; + size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); + struct vm_page *page; + void *ptr; + + if (p >= rdev->gart.num_cpu_pages) + return result; + + page = rdev->gart.pages[p]; + if (page) { + ptr = kmap(page); + ptr += off; + + r = copy_to_user(buf, ptr, cur_size); + kunmap(rdev->gart.pages[p]); + } else + r = clear_user(buf, cur_size); + + if (r) + return -EFAULT; + + result += cur_size; + buf += cur_size; + *pos += cur_size; + size -= cur_size; + } + + return result; +} + +static const struct file_operations radeon_ttm_gtt_fops = { + .owner = THIS_MODULE, + .open = radeon_ttm_gtt_open, + .read = radeon_ttm_gtt_read, + .llseek = default_llseek +}; + #endif static int radeon_ttm_debugfs_init(struct radeon_device *rdev) { #if defined(CONFIG_DEBUG_FS) - static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES+2]; - static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES+2][32]; - unsigned i; + unsigned count; - for (i = 0; i < RADEON_DEBUGFS_MEM_TYPES; i++) { - if (i == 0) - sprintf(radeon_mem_types_names[i], "radeon_vram_mm"); - else - sprintf(radeon_mem_types_names[i], "radeon_gtt_mm"); - radeon_mem_types_list[i].name = radeon_mem_types_names[i]; - radeon_mem_types_list[i].show = &radeon_mm_dump_table; - radeon_mem_types_list[i].driver_features = 0; - if (i == 0) - radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_VRAM].priv; - else - radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_TT].priv; + struct drm_minor *minor = rdev->ddev->primary; + struct dentry *ent, *root = minor->debugfs_root; + + ent = debugfs_create_file("radeon_vram", S_IFREG | S_IRUGO, root, + rdev, &radeon_ttm_vram_fops); + if (IS_ERR(ent)) + return PTR_ERR(ent); + rdev->mman.vram = ent; + + ent = debugfs_create_file("radeon_gtt", S_IFREG | S_IRUGO, root, + rdev, &radeon_ttm_gtt_fops); + if (IS_ERR(ent)) + return PTR_ERR(ent); + rdev->mman.gtt = ent; + + count = ARRAY_SIZE(radeon_ttm_debugfs_list); - } - /* Add ttm page pool to debugfs */ - sprintf(radeon_mem_types_names[i], "ttm_page_pool"); - radeon_mem_types_list[i].name = radeon_mem_types_names[i]; - radeon_mem_types_list[i].show = &ttm_page_alloc_debugfs; - radeon_mem_types_list[i].driver_features = 0; - radeon_mem_types_list[i++].data = NULL; #ifdef CONFIG_SWIOTLB - if (swiotlb_nr_tbl()) { - sprintf(radeon_mem_types_names[i], "ttm_dma_page_pool"); - radeon_mem_types_list[i].name = radeon_mem_types_names[i]; - radeon_mem_types_list[i].show = &ttm_dma_page_alloc_debugfs; - radeon_mem_types_list[i].driver_features = 0; - radeon_mem_types_list[i++].data = NULL; - } + if (!swiotlb_nr_tbl()) + --count; #endif - return radeon_debugfs_add_files(rdev, radeon_mem_types_list, i); -#endif + return radeon_debugfs_add_files(rdev, radeon_ttm_debugfs_list, count); +#else + return 0; +#endif +} + +static void radeon_ttm_debugfs_fini(struct radeon_device *rdev) +{ +#if defined(CONFIG_DEBUG_FS) + + debugfs_remove(rdev->mman.vram); + rdev->mman.vram = NULL; + + debugfs_remove(rdev->mman.gtt); + rdev->mman.gtt = NULL; +#endif } diff --git a/sys/dev/pci/drm/radeon/radeon_ucode.c b/sys/dev/pci/drm/radeon/radeon_ucode.c new file mode 100644 index 00000000000..ea8fa92f88e --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_ucode.c @@ -0,0 +1,174 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "radeon.h" +#include "radeon_ucode.h" + +static void radeon_ucode_print_common_hdr(const struct common_firmware_header *hdr) +{ + DRM_DEBUG("size_bytes: %u\n", le32_to_cpu(hdr->size_bytes)); + DRM_DEBUG("header_size_bytes: %u\n", le32_to_cpu(hdr->header_size_bytes)); + DRM_DEBUG("header_version_major: %u\n", le16_to_cpu(hdr->header_version_major)); + DRM_DEBUG("header_version_minor: %u\n", le16_to_cpu(hdr->header_version_minor)); + DRM_DEBUG("ip_version_major: %u\n", le16_to_cpu(hdr->ip_version_major)); + DRM_DEBUG("ip_version_minor: %u\n", le16_to_cpu(hdr->ip_version_minor)); + DRM_DEBUG("ucode_version: 0x%08x\n", le32_to_cpu(hdr->ucode_version)); + DRM_DEBUG("ucode_size_bytes: %u\n", le32_to_cpu(hdr->ucode_size_bytes)); + DRM_DEBUG("ucode_array_offset_bytes: %u\n", + le32_to_cpu(hdr->ucode_array_offset_bytes)); + DRM_DEBUG("crc32: 0x%08x\n", le32_to_cpu(hdr->crc32)); +} + +void radeon_ucode_print_mc_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("MC\n"); + radeon_ucode_print_common_hdr(hdr); + + if (version_major == 1) { +#ifdef DRMDEBUG + const struct mc_firmware_header_v1_0 *mc_hdr = + container_of(hdr, struct mc_firmware_header_v1_0, header); +#endif + + DRM_DEBUG("io_debug_size_bytes: %u\n", + le32_to_cpu(mc_hdr->io_debug_size_bytes)); + DRM_DEBUG("io_debug_array_offset_bytes: %u\n", + le32_to_cpu(mc_hdr->io_debug_array_offset_bytes)); + } else { + DRM_ERROR("Unknown MC ucode version: %u.%u\n", version_major, version_minor); + } +} + +void radeon_ucode_print_smc_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("SMC\n"); + radeon_ucode_print_common_hdr(hdr); + + if (version_major == 1) { +#ifdef DRMDEBUG + const struct smc_firmware_header_v1_0 *smc_hdr = + container_of(hdr, struct smc_firmware_header_v1_0, header); +#endif + + DRM_DEBUG("ucode_start_addr: %u\n", le32_to_cpu(smc_hdr->ucode_start_addr)); + } else { + DRM_ERROR("Unknown SMC ucode version: %u.%u\n", version_major, version_minor); + } +} + +void radeon_ucode_print_gfx_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("GFX\n"); + radeon_ucode_print_common_hdr(hdr); + + if (version_major == 1) { +#ifdef DRMDEBUG + const struct gfx_firmware_header_v1_0 *gfx_hdr = + container_of(hdr, struct gfx_firmware_header_v1_0, header); +#endif + + DRM_DEBUG("ucode_feature_version: %u\n", + le32_to_cpu(gfx_hdr->ucode_feature_version)); + DRM_DEBUG("jt_offset: %u\n", le32_to_cpu(gfx_hdr->jt_offset)); + DRM_DEBUG("jt_size: %u\n", le32_to_cpu(gfx_hdr->jt_size)); + } else { + DRM_ERROR("Unknown GFX ucode version: %u.%u\n", version_major, version_minor); + } +} + +void radeon_ucode_print_rlc_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("RLC\n"); + radeon_ucode_print_common_hdr(hdr); + + if (version_major == 1) { +#ifdef DRMDEBUG + const struct rlc_firmware_header_v1_0 *rlc_hdr = + container_of(hdr, struct rlc_firmware_header_v1_0, header); +#endif + + DRM_DEBUG("ucode_feature_version: %u\n", + le32_to_cpu(rlc_hdr->ucode_feature_version)); + DRM_DEBUG("save_and_restore_offset: %u\n", + le32_to_cpu(rlc_hdr->save_and_restore_offset)); + DRM_DEBUG("clear_state_descriptor_offset: %u\n", + le32_to_cpu(rlc_hdr->clear_state_descriptor_offset)); + DRM_DEBUG("avail_scratch_ram_locations: %u\n", + le32_to_cpu(rlc_hdr->avail_scratch_ram_locations)); + DRM_DEBUG("master_pkt_description_offset: %u\n", + le32_to_cpu(rlc_hdr->master_pkt_description_offset)); + } else { + DRM_ERROR("Unknown RLC ucode version: %u.%u\n", version_major, version_minor); + } +} + +void radeon_ucode_print_sdma_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("SDMA\n"); + radeon_ucode_print_common_hdr(hdr); + + if (version_major == 1) { +#ifdef DRMDEBUG + const struct sdma_firmware_header_v1_0 *sdma_hdr = + container_of(hdr, struct sdma_firmware_header_v1_0, header); +#endif + + DRM_DEBUG("ucode_feature_version: %u\n", + le32_to_cpu(sdma_hdr->ucode_feature_version)); + DRM_DEBUG("ucode_change_version: %u\n", + le32_to_cpu(sdma_hdr->ucode_change_version)); + DRM_DEBUG("jt_offset: %u\n", le32_to_cpu(sdma_hdr->jt_offset)); + DRM_DEBUG("jt_size: %u\n", le32_to_cpu(sdma_hdr->jt_size)); + } else { + DRM_ERROR("Unknown SDMA ucode version: %u.%u\n", + version_major, version_minor); + } +} + +int radeon_ucode_validate(const struct firmware *fw) +{ + const struct common_firmware_header *hdr = + (const struct common_firmware_header *)fw->data; + + if (fw->size == le32_to_cpu(hdr->size_bytes)) + return 0; + + return -EINVAL; +} + diff --git a/sys/dev/pci/drm/radeon/radeon_ucode.h b/sys/dev/pci/drm/radeon/radeon_ucode.h new file mode 100644 index 00000000000..dc4576e4d8a --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_ucode.h @@ -0,0 +1,227 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __RADEON_UCODE_H__ +#define __RADEON_UCODE_H__ + +/* CP */ +#define R600_PFP_UCODE_SIZE 576 +#define R600_PM4_UCODE_SIZE 1792 +#define R700_PFP_UCODE_SIZE 848 +#define R700_PM4_UCODE_SIZE 1360 +#define EVERGREEN_PFP_UCODE_SIZE 1120 +#define EVERGREEN_PM4_UCODE_SIZE 1376 +#define CAYMAN_PFP_UCODE_SIZE 2176 +#define CAYMAN_PM4_UCODE_SIZE 2176 +#define SI_PFP_UCODE_SIZE 2144 +#define SI_PM4_UCODE_SIZE 2144 +#define SI_CE_UCODE_SIZE 2144 +#define CIK_PFP_UCODE_SIZE 2144 +#define CIK_ME_UCODE_SIZE 2144 +#define CIK_CE_UCODE_SIZE 2144 + +/* MEC */ +#define CIK_MEC_UCODE_SIZE 4192 + +/* RLC */ +#define R600_RLC_UCODE_SIZE 768 +#define R700_RLC_UCODE_SIZE 1024 +#define EVERGREEN_RLC_UCODE_SIZE 768 +#define CAYMAN_RLC_UCODE_SIZE 1024 +#define ARUBA_RLC_UCODE_SIZE 1536 +#define SI_RLC_UCODE_SIZE 2048 +#define BONAIRE_RLC_UCODE_SIZE 2048 +#define KB_RLC_UCODE_SIZE 2560 +#define KV_RLC_UCODE_SIZE 2560 +#define ML_RLC_UCODE_SIZE 2560 + +/* MC */ +#define BTC_MC_UCODE_SIZE 6024 +#define CAYMAN_MC_UCODE_SIZE 6037 +#define SI_MC_UCODE_SIZE 7769 +#define TAHITI_MC_UCODE_SIZE 7808 +#define PITCAIRN_MC_UCODE_SIZE 7775 +#define VERDE_MC_UCODE_SIZE 7875 +#define OLAND_MC_UCODE_SIZE 7863 +#define BONAIRE_MC_UCODE_SIZE 7866 +#define BONAIRE_MC2_UCODE_SIZE 7948 +#define HAWAII_MC_UCODE_SIZE 7933 +#define HAWAII_MC2_UCODE_SIZE 8091 + +/* SDMA */ +#define CIK_SDMA_UCODE_SIZE 1050 +#define CIK_SDMA_UCODE_VERSION 64 + +/* SMC */ +#define RV770_SMC_UCODE_START 0x0100 +#define RV770_SMC_UCODE_SIZE 0x410d +#define RV770_SMC_INT_VECTOR_START 0xffc0 +#define RV770_SMC_INT_VECTOR_SIZE 0x0040 + +#define RV730_SMC_UCODE_START 0x0100 +#define RV730_SMC_UCODE_SIZE 0x412c +#define RV730_SMC_INT_VECTOR_START 0xffc0 +#define RV730_SMC_INT_VECTOR_SIZE 0x0040 + +#define RV710_SMC_UCODE_START 0x0100 +#define RV710_SMC_UCODE_SIZE 0x3f1f +#define RV710_SMC_INT_VECTOR_START 0xffc0 +#define RV710_SMC_INT_VECTOR_SIZE 0x0040 + +#define RV740_SMC_UCODE_START 0x0100 +#define RV740_SMC_UCODE_SIZE 0x41c5 +#define RV740_SMC_INT_VECTOR_START 0xffc0 +#define RV740_SMC_INT_VECTOR_SIZE 0x0040 + +#define CEDAR_SMC_UCODE_START 0x0100 +#define CEDAR_SMC_UCODE_SIZE 0x5d50 +#define CEDAR_SMC_INT_VECTOR_START 0xffc0 +#define CEDAR_SMC_INT_VECTOR_SIZE 0x0040 + +#define REDWOOD_SMC_UCODE_START 0x0100 +#define REDWOOD_SMC_UCODE_SIZE 0x5f0a +#define REDWOOD_SMC_INT_VECTOR_START 0xffc0 +#define REDWOOD_SMC_INT_VECTOR_SIZE 0x0040 + +#define JUNIPER_SMC_UCODE_START 0x0100 +#define JUNIPER_SMC_UCODE_SIZE 0x5f1f +#define JUNIPER_SMC_INT_VECTOR_START 0xffc0 +#define JUNIPER_SMC_INT_VECTOR_SIZE 0x0040 + +#define CYPRESS_SMC_UCODE_START 0x0100 +#define CYPRESS_SMC_UCODE_SIZE 0x61f7 +#define CYPRESS_SMC_INT_VECTOR_START 0xffc0 +#define CYPRESS_SMC_INT_VECTOR_SIZE 0x0040 + +#define BARTS_SMC_UCODE_START 0x0100 +#define BARTS_SMC_UCODE_SIZE 0x6107 +#define BARTS_SMC_INT_VECTOR_START 0xffc0 +#define BARTS_SMC_INT_VECTOR_SIZE 0x0040 + +#define TURKS_SMC_UCODE_START 0x0100 +#define TURKS_SMC_UCODE_SIZE 0x605b +#define TURKS_SMC_INT_VECTOR_START 0xffc0 +#define TURKS_SMC_INT_VECTOR_SIZE 0x0040 + +#define CAICOS_SMC_UCODE_START 0x0100 +#define CAICOS_SMC_UCODE_SIZE 0x5fbd +#define CAICOS_SMC_INT_VECTOR_START 0xffc0 +#define CAICOS_SMC_INT_VECTOR_SIZE 0x0040 + +#define CAYMAN_SMC_UCODE_START 0x0100 +#define CAYMAN_SMC_UCODE_SIZE 0x79ec +#define CAYMAN_SMC_INT_VECTOR_START 0xffc0 +#define CAYMAN_SMC_INT_VECTOR_SIZE 0x0040 + +#define TAHITI_SMC_UCODE_START 0x10000 +#define TAHITI_SMC_UCODE_SIZE 0xf458 + +#define PITCAIRN_SMC_UCODE_START 0x10000 +#define PITCAIRN_SMC_UCODE_SIZE 0xe9f4 + +#define VERDE_SMC_UCODE_START 0x10000 +#define VERDE_SMC_UCODE_SIZE 0xebe4 + +#define OLAND_SMC_UCODE_START 0x10000 +#define OLAND_SMC_UCODE_SIZE 0xe7b4 + +#define HAINAN_SMC_UCODE_START 0x10000 +#define HAINAN_SMC_UCODE_SIZE 0xe67C + +#define BONAIRE_SMC_UCODE_START 0x20000 +#define BONAIRE_SMC_UCODE_SIZE 0x1FDEC + +#define HAWAII_SMC_UCODE_START 0x20000 +#define HAWAII_SMC_UCODE_SIZE 0x1FDEC + +struct common_firmware_header { + uint32_t size_bytes; /* size of the entire header+image(s) in bytes */ + uint32_t header_size_bytes; /* size of just the header in bytes */ + uint16_t header_version_major; /* header version */ + uint16_t header_version_minor; /* header version */ + uint16_t ip_version_major; /* IP version */ + uint16_t ip_version_minor; /* IP version */ + uint32_t ucode_version; + uint32_t ucode_size_bytes; /* size of ucode in bytes */ + uint32_t ucode_array_offset_bytes; /* payload offset from the start of the header */ + uint32_t crc32; /* crc32 checksum of the payload */ +}; + +/* version_major=1, version_minor=0 */ +struct mc_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t io_debug_size_bytes; /* size of debug array in dwords */ + uint32_t io_debug_array_offset_bytes; /* payload offset from the start of the header */ +}; + +/* version_major=1, version_minor=0 */ +struct smc_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ucode_start_addr; +}; + +/* version_major=1, version_minor=0 */ +struct gfx_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ucode_feature_version; + uint32_t jt_offset; /* jt location */ + uint32_t jt_size; /* size of jt */ +}; + +/* version_major=1, version_minor=0 */ +struct rlc_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ucode_feature_version; + uint32_t save_and_restore_offset; + uint32_t clear_state_descriptor_offset; + uint32_t avail_scratch_ram_locations; + uint32_t master_pkt_description_offset; +}; + +/* version_major=1, version_minor=0 */ +struct sdma_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ucode_feature_version; + uint32_t ucode_change_version; + uint32_t jt_offset; /* jt location */ + uint32_t jt_size; /* size of jt */ +}; + +/* header is fixed size */ +union radeon_firmware_header { + struct common_firmware_header common; + struct mc_firmware_header_v1_0 mc; + struct smc_firmware_header_v1_0 smc; + struct gfx_firmware_header_v1_0 gfx; + struct rlc_firmware_header_v1_0 rlc; + struct sdma_firmware_header_v1_0 sdma; + uint8_t raw[0x100]; +}; + +void radeon_ucode_print_mc_hdr(const struct common_firmware_header *hdr); +void radeon_ucode_print_smc_hdr(const struct common_firmware_header *hdr); +void radeon_ucode_print_gfx_hdr(const struct common_firmware_header *hdr); +void radeon_ucode_print_rlc_hdr(const struct common_firmware_header *hdr); +void radeon_ucode_print_sdma_hdr(const struct common_firmware_header *hdr); +int radeon_ucode_validate(const struct firmware *fw); + +#endif diff --git a/sys/dev/pci/drm/radeon/radeon_uvd.c b/sys/dev/pci/drm/radeon/radeon_uvd.c new file mode 100644 index 00000000000..04cbe5dd8d8 --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_uvd.c @@ -0,0 +1,1001 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Christian König + */ + +#include +#include + +#include "radeon.h" +#include "r600d.h" + +/* 1 second timeout */ +#define UVD_IDLE_TIMEOUT_MS 1000 + +/* Firmware Names */ +#define FIRMWARE_R600 "radeon/R600_uvd.bin" +#define FIRMWARE_RS780 "radeon/RS780_uvd.bin" +#define FIRMWARE_RV770 "radeon/RV770_uvd.bin" +#define FIRMWARE_RV710 "radeon/RV710_uvd.bin" +#define FIRMWARE_CYPRESS "radeon/CYPRESS_uvd.bin" +#define FIRMWARE_SUMO "radeon/SUMO_uvd.bin" +#define FIRMWARE_TAHITI "radeon/TAHITI_uvd.bin" +#define FIRMWARE_BONAIRE "radeon/BONAIRE_uvd.bin" + +MODULE_FIRMWARE(FIRMWARE_R600); +MODULE_FIRMWARE(FIRMWARE_RS780); +MODULE_FIRMWARE(FIRMWARE_RV770); +MODULE_FIRMWARE(FIRMWARE_RV710); +MODULE_FIRMWARE(FIRMWARE_CYPRESS); +MODULE_FIRMWARE(FIRMWARE_SUMO); +MODULE_FIRMWARE(FIRMWARE_TAHITI); +MODULE_FIRMWARE(FIRMWARE_BONAIRE); + +static void radeon_uvd_idle_work_handler(struct work_struct *work); + +int radeon_uvd_init(struct radeon_device *rdev) +{ + unsigned long bo_size; + const char *fw_name; + int i, r; + + INIT_DELAYED_WORK(&rdev->uvd.idle_work, radeon_uvd_idle_work_handler); + + switch (rdev->family) { + case CHIP_RV610: + case CHIP_RV630: + case CHIP_RV670: + case CHIP_RV620: + case CHIP_RV635: + fw_name = FIRMWARE_R600; + break; + + case CHIP_RS780: + case CHIP_RS880: + fw_name = FIRMWARE_RS780; + break; + + case CHIP_RV770: + fw_name = FIRMWARE_RV770; + break; + + case CHIP_RV710: + case CHIP_RV730: + case CHIP_RV740: + fw_name = FIRMWARE_RV710; + break; + + case CHIP_CYPRESS: + case CHIP_HEMLOCK: + case CHIP_JUNIPER: + case CHIP_REDWOOD: + case CHIP_CEDAR: + fw_name = FIRMWARE_CYPRESS; + break; + + case CHIP_SUMO: + case CHIP_SUMO2: + case CHIP_PALM: + case CHIP_CAYMAN: + case CHIP_BARTS: + case CHIP_TURKS: + case CHIP_CAICOS: + fw_name = FIRMWARE_SUMO; + break; + + case CHIP_TAHITI: + case CHIP_VERDE: + case CHIP_PITCAIRN: + case CHIP_ARUBA: + case CHIP_OLAND: + fw_name = FIRMWARE_TAHITI; + break; + + case CHIP_BONAIRE: + case CHIP_KABINI: + case CHIP_KAVERI: + case CHIP_HAWAII: + case CHIP_MULLINS: + fw_name = FIRMWARE_BONAIRE; + break; + + default: + return -EINVAL; + } + + r = request_firmware(&rdev->uvd_fw, fw_name, rdev->dev); + if (r) { + dev_err(rdev->dev, "radeon_uvd: Can't load firmware \"%s\"\n", + fw_name); + return r; + } + + bo_size = RADEON_GPU_PAGE_ALIGN(rdev->uvd_fw->size + 8) + + RADEON_UVD_STACK_SIZE + RADEON_UVD_HEAP_SIZE + + RADEON_GPU_PAGE_SIZE; + r = radeon_bo_create(rdev, bo_size, PAGE_SIZE, true, + RADEON_GEM_DOMAIN_VRAM, 0, NULL, + NULL, &rdev->uvd.vcpu_bo); + if (r) { + dev_err(rdev->dev, "(%d) failed to allocate UVD bo\n", r); + return r; + } + + r = radeon_bo_reserve(rdev->uvd.vcpu_bo, false); + if (r) { + radeon_bo_unref(&rdev->uvd.vcpu_bo); + dev_err(rdev->dev, "(%d) failed to reserve UVD bo\n", r); + return r; + } + + r = radeon_bo_pin(rdev->uvd.vcpu_bo, RADEON_GEM_DOMAIN_VRAM, + &rdev->uvd.gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->uvd.vcpu_bo); + radeon_bo_unref(&rdev->uvd.vcpu_bo); + dev_err(rdev->dev, "(%d) UVD bo pin failed\n", r); + return r; + } + + r = radeon_bo_kmap(rdev->uvd.vcpu_bo, &rdev->uvd.cpu_addr); + if (r) { + dev_err(rdev->dev, "(%d) UVD map failed\n", r); + return r; + } + + radeon_bo_unreserve(rdev->uvd.vcpu_bo); + + for (i = 0; i < RADEON_MAX_UVD_HANDLES; ++i) { + atomic_set(&rdev->uvd.handles[i], 0); + rdev->uvd.filp[i] = NULL; + rdev->uvd.img_size[i] = 0; + } + + return 0; +} + +void radeon_uvd_fini(struct radeon_device *rdev) +{ + int r; + + if (rdev->uvd.vcpu_bo == NULL) + return; + + r = radeon_bo_reserve(rdev->uvd.vcpu_bo, false); + if (!r) { + radeon_bo_kunmap(rdev->uvd.vcpu_bo); + radeon_bo_unpin(rdev->uvd.vcpu_bo); + radeon_bo_unreserve(rdev->uvd.vcpu_bo); + } + + radeon_bo_unref(&rdev->uvd.vcpu_bo); + + radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX]); + + release_firmware(rdev->uvd_fw); +} + +int radeon_uvd_suspend(struct radeon_device *rdev) +{ + int i, r; + + if (rdev->uvd.vcpu_bo == NULL) + return 0; + + for (i = 0; i < RADEON_MAX_UVD_HANDLES; ++i) { + uint32_t handle = atomic_read(&rdev->uvd.handles[i]); + if (handle != 0) { + struct radeon_fence *fence; + + radeon_uvd_note_usage(rdev); + + r = radeon_uvd_get_destroy_msg(rdev, + R600_RING_TYPE_UVD_INDEX, handle, &fence); + if (r) { + DRM_ERROR("Error destroying UVD (%d)!\n", r); + continue; + } + + radeon_fence_wait(fence, false); + radeon_fence_unref(&fence); + + rdev->uvd.filp[i] = NULL; + atomic_set(&rdev->uvd.handles[i], 0); + } + } + + return 0; +} + +int radeon_uvd_resume(struct radeon_device *rdev) +{ + unsigned size; + void *ptr; + + if (rdev->uvd.vcpu_bo == NULL) + return -EINVAL; + + memcpy(rdev->uvd.cpu_addr, rdev->uvd_fw->data, rdev->uvd_fw->size); + + size = radeon_bo_size(rdev->uvd.vcpu_bo); + size -= rdev->uvd_fw->size; + + ptr = rdev->uvd.cpu_addr; + ptr += rdev->uvd_fw->size; + + memset(ptr, 0, size); + + return 0; +} + +void radeon_uvd_force_into_uvd_segment(struct radeon_bo *rbo, + uint32_t allowed_domains) +{ + int i; + + for (i = 0; i < rbo->placement.num_placement; ++i) { + rbo->placements[i].fpfn = 0 >> PAGE_SHIFT; + rbo->placements[i].lpfn = (256 * 1024 * 1024) >> PAGE_SHIFT; + } + + /* If it must be in VRAM it must be in the first segment as well */ + if (allowed_domains == RADEON_GEM_DOMAIN_VRAM) + return; + + /* abort if we already have more than one placement */ + if (rbo->placement.num_placement > 1) + return; + + /* add another 256MB segment */ + rbo->placements[1] = rbo->placements[0]; + rbo->placements[1].fpfn += (256 * 1024 * 1024) >> PAGE_SHIFT; + rbo->placements[1].lpfn += (256 * 1024 * 1024) >> PAGE_SHIFT; + rbo->placement.num_placement++; + rbo->placement.num_busy_placement++; +} + +void radeon_uvd_free_handles(struct radeon_device *rdev, struct drm_file *filp) +{ + int i, r; + for (i = 0; i < RADEON_MAX_UVD_HANDLES; ++i) { + uint32_t handle = atomic_read(&rdev->uvd.handles[i]); + if (handle != 0 && rdev->uvd.filp[i] == filp) { + struct radeon_fence *fence; + + radeon_uvd_note_usage(rdev); + + r = radeon_uvd_get_destroy_msg(rdev, + R600_RING_TYPE_UVD_INDEX, handle, &fence); + if (r) { + DRM_ERROR("Error destroying UVD (%d)!\n", r); + continue; + } + + radeon_fence_wait(fence, false); + radeon_fence_unref(&fence); + + rdev->uvd.filp[i] = NULL; + atomic_set(&rdev->uvd.handles[i], 0); + } + } +} + +static int radeon_uvd_cs_msg_decode(uint32_t *msg, unsigned buf_sizes[]) +{ + unsigned stream_type = msg[4]; + unsigned width = msg[6]; + unsigned height = msg[7]; + unsigned dpb_size = msg[9]; + unsigned pitch = msg[28]; + + unsigned width_in_mb = width / 16; + unsigned height_in_mb = roundup2(height / 16, 2); + + unsigned image_size, tmp, min_dpb_size; + + image_size = width * height; + image_size += image_size / 2; + image_size = roundup2(image_size, 1024); + + switch (stream_type) { + case 0: /* H264 */ + + /* reference picture buffer */ + min_dpb_size = image_size * 17; + + /* macroblock context buffer */ + min_dpb_size += width_in_mb * height_in_mb * 17 * 192; + + /* IT surface buffer */ + min_dpb_size += width_in_mb * height_in_mb * 32; + break; + + case 1: /* VC1 */ + + /* reference picture buffer */ + min_dpb_size = image_size * 3; + + /* CONTEXT_BUFFER */ + min_dpb_size += width_in_mb * height_in_mb * 128; + + /* IT surface buffer */ + min_dpb_size += width_in_mb * 64; + + /* DB surface buffer */ + min_dpb_size += width_in_mb * 128; + + /* BP */ + tmp = max(width_in_mb, height_in_mb); + min_dpb_size += roundup2(tmp * 7 * 16, 64); + break; + + case 3: /* MPEG2 */ + + /* reference picture buffer */ + min_dpb_size = image_size * 3; + break; + + case 4: /* MPEG4 */ + + /* reference picture buffer */ + min_dpb_size = image_size * 3; + + /* CM */ + min_dpb_size += width_in_mb * height_in_mb * 64; + + /* IT surface buffer */ + min_dpb_size += roundup2(width_in_mb * height_in_mb * 32, 64); + break; + + default: + DRM_ERROR("UVD codec not handled %d!\n", stream_type); + return -EINVAL; + } + + if (width > pitch) { + DRM_ERROR("Invalid UVD decoding target pitch!\n"); + return -EINVAL; + } + + if (dpb_size < min_dpb_size) { + DRM_ERROR("Invalid dpb_size in UVD message (%d / %d)!\n", + dpb_size, min_dpb_size); + return -EINVAL; + } + + buf_sizes[0x1] = dpb_size; + buf_sizes[0x2] = image_size; + return 0; +} + +static int radeon_uvd_validate_codec(struct radeon_cs_parser *p, + unsigned stream_type) +{ + switch (stream_type) { + case 0: /* H264 */ + case 1: /* VC1 */ + /* always supported */ + return 0; + + case 3: /* MPEG2 */ + case 4: /* MPEG4 */ + /* only since UVD 3 */ + if (p->rdev->family >= CHIP_PALM) + return 0; + + /* fall through */ + default: + DRM_ERROR("UVD codec not supported by hardware %d!\n", + stream_type); + return -EINVAL; + } +} + +static int radeon_uvd_cs_msg(struct radeon_cs_parser *p, struct radeon_bo *bo, + unsigned offset, unsigned buf_sizes[]) +{ + int32_t *msg, msg_type, handle; + unsigned img_size = 0; + struct fence *f; + void *ptr; + + int i, r; + + if (offset & 0x3F) { + DRM_ERROR("UVD messages must be 64 byte aligned!\n"); + return -EINVAL; + } + + f = reservation_object_get_excl(bo->tbo.resv); + if (f) { + r = radeon_fence_wait((struct radeon_fence *)f, false); + if (r) { + DRM_ERROR("Failed waiting for UVD message (%d)!\n", r); + return r; + } + } + + r = radeon_bo_kmap(bo, &ptr); + if (r) { + DRM_ERROR("Failed mapping the UVD message (%d)!\n", r); + return r; + } + + msg = ptr + offset; + + msg_type = msg[1]; + handle = msg[2]; + + if (handle == 0) { + DRM_ERROR("Invalid UVD handle!\n"); + return -EINVAL; + } + + switch (msg_type) { + case 0: + /* it's a create msg, calc image size (width * height) */ + img_size = msg[7] * msg[8]; + + r = radeon_uvd_validate_codec(p, msg[4]); + radeon_bo_kunmap(bo); + if (r) + return r; + + /* try to alloc a new handle */ + for (i = 0; i < RADEON_MAX_UVD_HANDLES; ++i) { + if (atomic_read(&p->rdev->uvd.handles[i]) == handle) { + DRM_ERROR("Handle 0x%x already in use!\n", handle); + return -EINVAL; + } + + if (!atomic_cmpxchg(&p->rdev->uvd.handles[i], 0, handle)) { + p->rdev->uvd.filp[i] = p->filp; + p->rdev->uvd.img_size[i] = img_size; + return 0; + } + } + + DRM_ERROR("No more free UVD handles!\n"); + return -EINVAL; + + case 1: + /* it's a decode msg, validate codec and calc buffer sizes */ + r = radeon_uvd_validate_codec(p, msg[4]); + if (!r) + r = radeon_uvd_cs_msg_decode(msg, buf_sizes); + radeon_bo_kunmap(bo); + if (r) + return r; + + /* validate the handle */ + for (i = 0; i < RADEON_MAX_UVD_HANDLES; ++i) { + if (atomic_read(&p->rdev->uvd.handles[i]) == handle) { + if (p->rdev->uvd.filp[i] != p->filp) { + DRM_ERROR("UVD handle collision detected!\n"); + return -EINVAL; + } + return 0; + } + } + + DRM_ERROR("Invalid UVD handle 0x%x!\n", handle); + return -ENOENT; + + case 2: + /* it's a destroy msg, free the handle */ + for (i = 0; i < RADEON_MAX_UVD_HANDLES; ++i) + atomic_cmpxchg(&p->rdev->uvd.handles[i], handle, 0); + radeon_bo_kunmap(bo); + return 0; + + default: + + DRM_ERROR("Illegal UVD message type (%d)!\n", msg_type); + return -EINVAL; + } + + BUG(); + return -EINVAL; +} + +static int radeon_uvd_cs_reloc(struct radeon_cs_parser *p, + int data0, int data1, + unsigned buf_sizes[], bool *has_msg_cmd) +{ + struct radeon_cs_chunk *relocs_chunk; + struct radeon_bo_list *reloc; + unsigned idx, cmd, offset; + uint64_t start, end; + int r; + + relocs_chunk = p->chunk_relocs; + offset = radeon_get_ib_value(p, data0); + idx = radeon_get_ib_value(p, data1); + if (idx >= relocs_chunk->length_dw) { + DRM_ERROR("Relocs at %d after relocations chunk end %d !\n", + idx, relocs_chunk->length_dw); + return -EINVAL; + } + + reloc = &p->relocs[(idx / 4)]; + start = reloc->gpu_offset; + end = start + radeon_bo_size(reloc->robj); + start += offset; + + p->ib.ptr[data0] = start & 0xFFFFFFFF; + p->ib.ptr[data1] = start >> 32; + + cmd = radeon_get_ib_value(p, p->idx) >> 1; + + if (cmd < 0x4) { + if (end <= start) { + DRM_ERROR("invalid reloc offset %X!\n", offset); + return -EINVAL; + } + if ((end - start) < buf_sizes[cmd]) { + DRM_ERROR("buffer (%d) to small (%d / %d)!\n", cmd, + (unsigned)(end - start), buf_sizes[cmd]); + return -EINVAL; + } + + } else if (cmd != 0x100) { + DRM_ERROR("invalid UVD command %X!\n", cmd); + return -EINVAL; + } + + if ((start >> 28) != ((end - 1) >> 28)) { + DRM_ERROR("reloc %llX-%llX crossing 256MB boundary!\n", + start, end); + return -EINVAL; + } + + /* TODO: is this still necessary on NI+ ? */ + if ((cmd == 0 || cmd == 0x3) && + (start >> 28) != (p->rdev->uvd.gpu_addr >> 28)) { + DRM_ERROR("msg/fb buffer %llX-%llX out of 256MB segment!\n", + start, end); + return -EINVAL; + } + + if (cmd == 0) { + if (*has_msg_cmd) { + DRM_ERROR("More than one message in a UVD-IB!\n"); + return -EINVAL; + } + *has_msg_cmd = true; + r = radeon_uvd_cs_msg(p, reloc->robj, offset, buf_sizes); + if (r) + return r; + } else if (!*has_msg_cmd) { + DRM_ERROR("Message needed before other commands are send!\n"); + return -EINVAL; + } + + return 0; +} + +static int radeon_uvd_cs_reg(struct radeon_cs_parser *p, + struct radeon_cs_packet *pkt, + int *data0, int *data1, + unsigned buf_sizes[], + bool *has_msg_cmd) +{ + int i, r; + + p->idx++; + for (i = 0; i <= pkt->count; ++i) { + switch (pkt->reg + i*4) { + case UVD_GPCOM_VCPU_DATA0: + *data0 = p->idx; + break; + case UVD_GPCOM_VCPU_DATA1: + *data1 = p->idx; + break; + case UVD_GPCOM_VCPU_CMD: + r = radeon_uvd_cs_reloc(p, *data0, *data1, + buf_sizes, has_msg_cmd); + if (r) + return r; + break; + case UVD_ENGINE_CNTL: + break; + default: + DRM_ERROR("Invalid reg 0x%X!\n", + pkt->reg + i*4); + return -EINVAL; + } + p->idx++; + } + return 0; +} + +int radeon_uvd_cs_parse(struct radeon_cs_parser *p) +{ + struct radeon_cs_packet pkt; + int r, data0 = 0, data1 = 0; + + /* does the IB has a msg command */ + bool has_msg_cmd = false; + + /* minimum buffer sizes */ + unsigned buf_sizes[] = { + [0x00000000] = 2048, + [0x00000001] = 32 * 1024 * 1024, + [0x00000002] = 2048 * 1152 * 3, + [0x00000003] = 2048, + }; + + if (p->chunk_ib->length_dw % 16) { + DRM_ERROR("UVD IB length (%d) not 16 dwords aligned!\n", + p->chunk_ib->length_dw); + return -EINVAL; + } + + if (p->chunk_relocs == NULL) { + DRM_ERROR("No relocation chunk !\n"); + return -EINVAL; + } + + + do { + r = radeon_cs_packet_parse(p, &pkt, p->idx); + if (r) + return r; + switch (pkt.type) { + case RADEON_PACKET_TYPE0: + r = radeon_uvd_cs_reg(p, &pkt, &data0, &data1, + buf_sizes, &has_msg_cmd); + if (r) + return r; + break; + case RADEON_PACKET_TYPE2: + p->idx += pkt.count + 2; + break; + default: + DRM_ERROR("Unknown packet type %d !\n", pkt.type); + return -EINVAL; + } + } while (p->idx < p->chunk_ib->length_dw); + + if (!has_msg_cmd) { + DRM_ERROR("UVD-IBs need a msg command!\n"); + return -EINVAL; + } + + return 0; +} + +static int radeon_uvd_send_msg(struct radeon_device *rdev, + int ring, uint64_t addr, + struct radeon_fence **fence) +{ + struct radeon_ib ib; + int i, r; + + r = radeon_ib_get(rdev, ring, &ib, NULL, 64); + if (r) + return r; + + ib.ptr[0] = PACKET0(UVD_GPCOM_VCPU_DATA0, 0); + ib.ptr[1] = addr; + ib.ptr[2] = PACKET0(UVD_GPCOM_VCPU_DATA1, 0); + ib.ptr[3] = addr >> 32; + ib.ptr[4] = PACKET0(UVD_GPCOM_VCPU_CMD, 0); + ib.ptr[5] = 0; + for (i = 6; i < 16; ++i) + ib.ptr[i] = PACKET2(0); + ib.length_dw = 16; + + r = radeon_ib_schedule(rdev, &ib, NULL, false); + + if (fence) + *fence = radeon_fence_ref(ib.fence); + + radeon_ib_free(rdev, &ib); + return r; +} + +/* multiple fence commands without any stream commands in between can + crash the vcpu so just try to emmit a dummy create/destroy msg to + avoid this */ +int radeon_uvd_get_create_msg(struct radeon_device *rdev, int ring, + uint32_t handle, struct radeon_fence **fence) +{ + /* we use the last page of the vcpu bo for the UVD message */ + uint64_t offs = radeon_bo_size(rdev->uvd.vcpu_bo) - + RADEON_GPU_PAGE_SIZE; + + uint32_t *msg = rdev->uvd.cpu_addr + offs; + uint64_t addr = rdev->uvd.gpu_addr + offs; + + int r, i; + + r = radeon_bo_reserve(rdev->uvd.vcpu_bo, true); + if (r) + return r; + + /* stitch together an UVD create msg */ + msg[0] = cpu_to_le32(0x00000de4); + msg[1] = cpu_to_le32(0x00000000); + msg[2] = cpu_to_le32(handle); + msg[3] = cpu_to_le32(0x00000000); + msg[4] = cpu_to_le32(0x00000000); + msg[5] = cpu_to_le32(0x00000000); + msg[6] = cpu_to_le32(0x00000000); + msg[7] = cpu_to_le32(0x00000780); + msg[8] = cpu_to_le32(0x00000440); + msg[9] = cpu_to_le32(0x00000000); + msg[10] = cpu_to_le32(0x01b37000); + for (i = 11; i < 1024; ++i) + msg[i] = cpu_to_le32(0x0); + + r = radeon_uvd_send_msg(rdev, ring, addr, fence); + radeon_bo_unreserve(rdev->uvd.vcpu_bo); + return r; +} + +int radeon_uvd_get_destroy_msg(struct radeon_device *rdev, int ring, + uint32_t handle, struct radeon_fence **fence) +{ + /* we use the last page of the vcpu bo for the UVD message */ + uint64_t offs = radeon_bo_size(rdev->uvd.vcpu_bo) - + RADEON_GPU_PAGE_SIZE; + + uint32_t *msg = rdev->uvd.cpu_addr + offs; + uint64_t addr = rdev->uvd.gpu_addr + offs; + + int r, i; + + r = radeon_bo_reserve(rdev->uvd.vcpu_bo, true); + if (r) + return r; + + /* stitch together an UVD destroy msg */ + msg[0] = cpu_to_le32(0x00000de4); + msg[1] = cpu_to_le32(0x00000002); + msg[2] = cpu_to_le32(handle); + msg[3] = cpu_to_le32(0x00000000); + for (i = 4; i < 1024; ++i) + msg[i] = cpu_to_le32(0x0); + + r = radeon_uvd_send_msg(rdev, ring, addr, fence); + radeon_bo_unreserve(rdev->uvd.vcpu_bo); + return r; +} + +/** + * radeon_uvd_count_handles - count number of open streams + * + * @rdev: radeon_device pointer + * @sd: number of SD streams + * @hd: number of HD streams + * + * Count the number of open SD/HD streams as a hint for power mangement + */ +static void radeon_uvd_count_handles(struct radeon_device *rdev, + unsigned *sd, unsigned *hd) +{ + unsigned i; + + *sd = 0; + *hd = 0; + + for (i = 0; i < RADEON_MAX_UVD_HANDLES; ++i) { + if (!atomic_read(&rdev->uvd.handles[i])) + continue; + + if (rdev->uvd.img_size[i] >= 720*576) + ++(*hd); + else + ++(*sd); + } +} + +static void radeon_uvd_idle_work_handler(struct work_struct *work) +{ + struct radeon_device *rdev = + container_of(work, struct radeon_device, uvd.idle_work.work); + + if (radeon_fence_count_emitted(rdev, R600_RING_TYPE_UVD_INDEX) == 0) { + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + radeon_uvd_count_handles(rdev, &rdev->pm.dpm.sd, + &rdev->pm.dpm.hd); + radeon_dpm_enable_uvd(rdev, false); + } else { + radeon_set_uvd_clocks(rdev, 0, 0); + } + } else { + schedule_delayed_work(&rdev->uvd.idle_work, + msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS)); + } +} + +void radeon_uvd_note_usage(struct radeon_device *rdev) +{ + bool streams_changed = false; + bool set_clocks = !cancel_delayed_work_sync(&rdev->uvd.idle_work); + set_clocks &= schedule_delayed_work(&rdev->uvd.idle_work, + msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS)); + + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + unsigned hd = 0, sd = 0; + radeon_uvd_count_handles(rdev, &sd, &hd); + if ((rdev->pm.dpm.sd != sd) || + (rdev->pm.dpm.hd != hd)) { + rdev->pm.dpm.sd = sd; + rdev->pm.dpm.hd = hd; + /* disable this for now */ + /*streams_changed = true;*/ + } + } + + if (set_clocks || streams_changed) { + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + radeon_dpm_enable_uvd(rdev, true); + } else { + radeon_set_uvd_clocks(rdev, 53300, 40000); + } + } +} + +static unsigned radeon_uvd_calc_upll_post_div(unsigned vco_freq, + unsigned target_freq, + unsigned pd_min, + unsigned pd_even) +{ + unsigned post_div = vco_freq / target_freq; + + /* adjust to post divider minimum value */ + if (post_div < pd_min) + post_div = pd_min; + + /* we alway need a frequency less than or equal the target */ + if ((vco_freq / post_div) > target_freq) + post_div += 1; + + /* post dividers above a certain value must be even */ + if (post_div > pd_even && post_div % 2) + post_div += 1; + + return post_div; +} + +/** + * radeon_uvd_calc_upll_dividers - calc UPLL clock dividers + * + * @rdev: radeon_device pointer + * @vclk: wanted VCLK + * @dclk: wanted DCLK + * @vco_min: minimum VCO frequency + * @vco_max: maximum VCO frequency + * @fb_factor: factor to multiply vco freq with + * @fb_mask: limit and bitmask for feedback divider + * @pd_min: post divider minimum + * @pd_max: post divider maximum + * @pd_even: post divider must be even above this value + * @optimal_fb_div: resulting feedback divider + * @optimal_vclk_div: resulting vclk post divider + * @optimal_dclk_div: resulting dclk post divider + * + * Calculate dividers for UVDs UPLL (R6xx-SI, except APUs). + * Returns zero on success -EINVAL on error. + */ +int radeon_uvd_calc_upll_dividers(struct radeon_device *rdev, + unsigned vclk, unsigned dclk, + unsigned vco_min, unsigned vco_max, + unsigned fb_factor, unsigned fb_mask, + unsigned pd_min, unsigned pd_max, + unsigned pd_even, + unsigned *optimal_fb_div, + unsigned *optimal_vclk_div, + unsigned *optimal_dclk_div) +{ + unsigned vco_freq, ref_freq = rdev->clock.spll.reference_freq; + + /* start off with something large */ + unsigned optimal_score = ~0; + + /* loop through vco from low to high */ + vco_min = max(max(vco_min, vclk), dclk); + for (vco_freq = vco_min; vco_freq <= vco_max; vco_freq += 100) { + + uint64_t fb_div = (uint64_t)vco_freq * fb_factor; + unsigned vclk_div, dclk_div, score; + + do_div(fb_div, ref_freq); + + /* fb div out of range ? */ + if (fb_div > fb_mask) + break; /* it can oly get worse */ + + fb_div &= fb_mask; + + /* calc vclk divider with current vco freq */ + vclk_div = radeon_uvd_calc_upll_post_div(vco_freq, vclk, + pd_min, pd_even); + if (vclk_div > pd_max) + break; /* vco is too big, it has to stop */ + + /* calc dclk divider with current vco freq */ + dclk_div = radeon_uvd_calc_upll_post_div(vco_freq, dclk, + pd_min, pd_even); + if (dclk_div > pd_max) + break; /* vco is too big, it has to stop */ + + /* calc score with current vco freq */ + score = vclk - (vco_freq / vclk_div) + dclk - (vco_freq / dclk_div); + + /* determine if this vco setting is better than current optimal settings */ + if (score < optimal_score) { + *optimal_fb_div = fb_div; + *optimal_vclk_div = vclk_div; + *optimal_dclk_div = dclk_div; + optimal_score = score; + if (optimal_score == 0) + break; /* it can't get better than this */ + } + } + + /* did we found a valid setup ? */ + if (optimal_score == ~0) + return -EINVAL; + + return 0; +} + +int radeon_uvd_send_upll_ctlreq(struct radeon_device *rdev, + unsigned cg_upll_func_cntl) +{ + unsigned i; + + /* make sure UPLL_CTLREQ is deasserted */ + WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK); + + mdelay(10); + + /* assert UPLL_CTLREQ */ + WREG32_P(cg_upll_func_cntl, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK); + + /* wait for CTLACK and CTLACK2 to get asserted */ + for (i = 0; i < 100; ++i) { + uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK; + if ((RREG32(cg_upll_func_cntl) & mask) == mask) + break; + mdelay(10); + } + + /* deassert UPLL_CTLREQ */ + WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK); + + if (i == 100) { + DRM_ERROR("Timeout setting UVD clocks!\n"); + return -ETIMEDOUT; + } + + return 0; +} diff --git a/sys/dev/pci/drm/radeon/radeon_vce.c b/sys/dev/pci/drm/radeon/radeon_vce.c new file mode 100644 index 00000000000..94eb667044a --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_vce.c @@ -0,0 +1,864 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * Authors: Christian König + */ + +#include +#include + +#include "radeon.h" +#include "radeon_asic.h" +#include "sid.h" + +/* 1 second timeout */ +#define VCE_IDLE_TIMEOUT_MS 1000 + +/* Firmware Names */ +#define FIRMWARE_TAHITI "radeon/TAHITI_vce.bin" +#define FIRMWARE_BONAIRE "radeon/BONAIRE_vce.bin" + +MODULE_FIRMWARE(FIRMWARE_TAHITI); +MODULE_FIRMWARE(FIRMWARE_BONAIRE); + +static void radeon_vce_idle_work_handler(struct work_struct *work); + +/** + * radeon_vce_init - allocate memory, load vce firmware + * + * @rdev: radeon_device pointer + * + * First step to get VCE online, allocate memory and load the firmware + */ +int radeon_vce_init(struct radeon_device *rdev) +{ + static const char *fw_version = "[ATI LIB=VCEFW,"; + static const char *fb_version = "[ATI LIB=VCEFWSTATS,"; + unsigned long size; + const char *fw_name, *c; + uint8_t start, mid, end; + int i, r; + + INIT_DELAYED_WORK(&rdev->vce.idle_work, radeon_vce_idle_work_handler); + + switch (rdev->family) { + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_VERDE: + case CHIP_OLAND: + case CHIP_ARUBA: + fw_name = FIRMWARE_TAHITI; + break; + + case CHIP_BONAIRE: + case CHIP_KAVERI: + case CHIP_KABINI: + case CHIP_HAWAII: + case CHIP_MULLINS: + fw_name = FIRMWARE_BONAIRE; + break; + + default: + return -EINVAL; + } + + r = request_firmware(&rdev->vce_fw, fw_name, rdev->dev); + if (r) { + dev_err(rdev->dev, "radeon_vce: Can't load firmware \"%s\"\n", + fw_name); + return r; + } + + /* search for firmware version */ + + size = rdev->vce_fw->size - strlen(fw_version) - 9; + c = rdev->vce_fw->data; + for (;size > 0; --size, ++c) + if (strncmp(c, fw_version, strlen(fw_version)) == 0) + break; + + if (size == 0) + return -EINVAL; + + c += strlen(fw_version); +#ifdef notyet + if (sscanf(c, "%2hhd.%2hhd.%2hhd]", &start, &mid, &end) != 3) + return -EINVAL; +#else +#if 0 + int x; + printf("\n%s A\n", __func__); + for (x = 0; x < 16; x++) { + printf("%02x ", c[x]); + } +#endif + if (c[2] != '.') { + printf("%s %s bad start value\n", rdev->self.dv_xname, __func__); + return -EINVAL; + } + start = (10 * (c[0] - '0')) + (c[1] - '0'); + c += 3; + + if (c[1] != '.') { + printf("%s %s bad mid value\n", rdev->self.dv_xname, __func__); + return -EINVAL; + } + mid = c[0] - '0'; + c += 2; + + if (c[1] != ']') { + printf("%s %s bad end value\n", rdev->self.dv_xname, __func__); + return -EINVAL; + } + end = c[0] - '0'; +#endif + + /* search for feedback version */ + + size = rdev->vce_fw->size - strlen(fb_version) - 3; + c = rdev->vce_fw->data; + for (;size > 0; --size, ++c) + if (strncmp(c, fb_version, strlen(fb_version)) == 0) + break; + + if (size == 0) + return -EINVAL; + + c += strlen(fb_version); +#ifdef notyet + if (sscanf(c, "%2u]", &rdev->vce.fb_version) != 1) + return -EINVAL; +#else +#if 0 + printf("\n%s B\n", __func__); + for (x = 0; x < 16; x++) { + printf("%02x ", c[x]); + } + printf("\n"); +#endif + if (c[2] != ']') { + printf("%s %s bad fb_version value\n", rdev->self.dv_xname, __func__); + return -EINVAL; + } + rdev->vce.fb_version = (10 * (c[0] - '0')) + (c[1] - '0'); +#endif + + DRM_INFO("Found VCE firmware/feedback version %hhd.%hhd.%hhd / %d!\n", + start, mid, end, rdev->vce.fb_version); + + rdev->vce.fw_version = (start << 24) | (mid << 16) | (end << 8); + + /* we can only work with this fw version for now */ + if ((rdev->vce.fw_version != ((40 << 24) | (2 << 16) | (2 << 8))) && + (rdev->vce.fw_version != ((50 << 24) | (0 << 16) | (1 << 8))) && + (rdev->vce.fw_version != ((50 << 24) | (1 << 16) | (2 << 8)))) + return -EINVAL; + + /* allocate firmware, stack and heap BO */ + + if (rdev->family < CHIP_BONAIRE) + size = vce_v1_0_bo_size(rdev); + else + size = vce_v2_0_bo_size(rdev); + r = radeon_bo_create(rdev, size, PAGE_SIZE, true, + RADEON_GEM_DOMAIN_VRAM, 0, NULL, NULL, + &rdev->vce.vcpu_bo); + if (r) { + dev_err(rdev->dev, "(%d) failed to allocate VCE bo\n", r); + return r; + } + + r = radeon_bo_reserve(rdev->vce.vcpu_bo, false); + if (r) { + radeon_bo_unref(&rdev->vce.vcpu_bo); + dev_err(rdev->dev, "(%d) failed to reserve VCE bo\n", r); + return r; + } + + r = radeon_bo_pin(rdev->vce.vcpu_bo, RADEON_GEM_DOMAIN_VRAM, + &rdev->vce.gpu_addr); + radeon_bo_unreserve(rdev->vce.vcpu_bo); + if (r) { + radeon_bo_unref(&rdev->vce.vcpu_bo); + dev_err(rdev->dev, "(%d) VCE bo pin failed\n", r); + return r; + } + + for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) { + atomic_set(&rdev->vce.handles[i], 0); + rdev->vce.filp[i] = NULL; + } + + return 0; +} + +/** + * radeon_vce_fini - free memory + * + * @rdev: radeon_device pointer + * + * Last step on VCE teardown, free firmware memory + */ +void radeon_vce_fini(struct radeon_device *rdev) +{ + if (rdev->vce.vcpu_bo == NULL) + return; + + radeon_bo_unref(&rdev->vce.vcpu_bo); + + release_firmware(rdev->vce_fw); +} + +/** + * radeon_vce_suspend - unpin VCE fw memory + * + * @rdev: radeon_device pointer + * + */ +int radeon_vce_suspend(struct radeon_device *rdev) +{ + int i; + + if (rdev->vce.vcpu_bo == NULL) + return 0; + + for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) + if (atomic_read(&rdev->vce.handles[i])) + break; + + if (i == RADEON_MAX_VCE_HANDLES) + return 0; + + /* TODO: suspending running encoding sessions isn't supported */ + return -EINVAL; +} + +/** + * radeon_vce_resume - pin VCE fw memory + * + * @rdev: radeon_device pointer + * + */ +int radeon_vce_resume(struct radeon_device *rdev) +{ + void *cpu_addr; + int r; + + if (rdev->vce.vcpu_bo == NULL) + return -EINVAL; + + r = radeon_bo_reserve(rdev->vce.vcpu_bo, false); + if (r) { + dev_err(rdev->dev, "(%d) failed to reserve VCE bo\n", r); + return r; + } + + r = radeon_bo_kmap(rdev->vce.vcpu_bo, &cpu_addr); + if (r) { + radeon_bo_unreserve(rdev->vce.vcpu_bo); + dev_err(rdev->dev, "(%d) VCE map failed\n", r); + return r; + } + + memset(cpu_addr, 0, radeon_bo_size(rdev->vce.vcpu_bo)); + if (rdev->family < CHIP_BONAIRE) + r = vce_v1_0_load_fw(rdev, cpu_addr); + else + memcpy(cpu_addr, rdev->vce_fw->data, rdev->vce_fw->size); + + radeon_bo_kunmap(rdev->vce.vcpu_bo); + + radeon_bo_unreserve(rdev->vce.vcpu_bo); + + return r; +} + +/** + * radeon_vce_idle_work_handler - power off VCE + * + * @work: pointer to work structure + * + * power of VCE when it's not used any more + */ +static void radeon_vce_idle_work_handler(struct work_struct *work) +{ + struct radeon_device *rdev = + container_of(work, struct radeon_device, vce.idle_work.work); + + if ((radeon_fence_count_emitted(rdev, TN_RING_TYPE_VCE1_INDEX) == 0) && + (radeon_fence_count_emitted(rdev, TN_RING_TYPE_VCE2_INDEX) == 0)) { + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + radeon_dpm_enable_vce(rdev, false); + } else { + radeon_set_vce_clocks(rdev, 0, 0); + } + } else { + schedule_delayed_work(&rdev->vce.idle_work, + msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS)); + } +} + +/** + * radeon_vce_note_usage - power up VCE + * + * @rdev: radeon_device pointer + * + * Make sure VCE is powerd up when we want to use it + */ +void radeon_vce_note_usage(struct radeon_device *rdev) +{ + bool streams_changed = false; + bool set_clocks = !cancel_delayed_work_sync(&rdev->vce.idle_work); + set_clocks &= schedule_delayed_work(&rdev->vce.idle_work, + msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS)); + + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + /* XXX figure out if the streams changed */ + streams_changed = false; + } + + if (set_clocks || streams_changed) { + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + radeon_dpm_enable_vce(rdev, true); + } else { + radeon_set_vce_clocks(rdev, 53300, 40000); + } + } +} + +/** + * radeon_vce_free_handles - free still open VCE handles + * + * @rdev: radeon_device pointer + * @filp: drm file pointer + * + * Close all VCE handles still open by this file pointer + */ +void radeon_vce_free_handles(struct radeon_device *rdev, struct drm_file *filp) +{ + int i, r; + for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) { + uint32_t handle = atomic_read(&rdev->vce.handles[i]); + if (!handle || rdev->vce.filp[i] != filp) + continue; + + radeon_vce_note_usage(rdev); + + r = radeon_vce_get_destroy_msg(rdev, TN_RING_TYPE_VCE1_INDEX, + handle, NULL); + if (r) + DRM_ERROR("Error destroying VCE handle (%d)!\n", r); + + rdev->vce.filp[i] = NULL; + atomic_set(&rdev->vce.handles[i], 0); + } +} + +/** + * radeon_vce_get_create_msg - generate a VCE create msg + * + * @rdev: radeon_device pointer + * @ring: ring we should submit the msg to + * @handle: VCE session handle to use + * @fence: optional fence to return + * + * Open up a stream for HW test + */ +int radeon_vce_get_create_msg(struct radeon_device *rdev, int ring, + uint32_t handle, struct radeon_fence **fence) +{ + const unsigned ib_size_dw = 1024; + struct radeon_ib ib; + uint64_t dummy; + int i, r; + + r = radeon_ib_get(rdev, ring, &ib, NULL, ib_size_dw * 4); + if (r) { + DRM_ERROR("radeon: failed to get ib (%d).\n", r); + return r; + } + + dummy = ib.gpu_addr + 1024; + + /* stitch together an VCE create msg */ + ib.length_dw = 0; + ib.ptr[ib.length_dw++] = cpu_to_le32(0x0000000c); /* len */ + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001); /* session cmd */ + ib.ptr[ib.length_dw++] = cpu_to_le32(handle); + + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000030); /* len */ + ib.ptr[ib.length_dw++] = cpu_to_le32(0x01000001); /* create cmd */ + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000000); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000042); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x0000000a); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000080); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000060); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000100); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000100); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x0000000c); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000000); + + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000014); /* len */ + ib.ptr[ib.length_dw++] = cpu_to_le32(0x05000005); /* feedback buffer */ + ib.ptr[ib.length_dw++] = cpu_to_le32(upper_32_bits(dummy)); + ib.ptr[ib.length_dw++] = cpu_to_le32(dummy); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001); + + for (i = ib.length_dw; i < ib_size_dw; ++i) + ib.ptr[i] = cpu_to_le32(0x0); + + r = radeon_ib_schedule(rdev, &ib, NULL, false); + if (r) { + DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); + } + + if (fence) + *fence = radeon_fence_ref(ib.fence); + + radeon_ib_free(rdev, &ib); + + return r; +} + +/** + * radeon_vce_get_destroy_msg - generate a VCE destroy msg + * + * @rdev: radeon_device pointer + * @ring: ring we should submit the msg to + * @handle: VCE session handle to use + * @fence: optional fence to return + * + * Close up a stream for HW test or if userspace failed to do so + */ +int radeon_vce_get_destroy_msg(struct radeon_device *rdev, int ring, + uint32_t handle, struct radeon_fence **fence) +{ + const unsigned ib_size_dw = 1024; + struct radeon_ib ib; + uint64_t dummy; + int i, r; + + r = radeon_ib_get(rdev, ring, &ib, NULL, ib_size_dw * 4); + if (r) { + DRM_ERROR("radeon: failed to get ib (%d).\n", r); + return r; + } + + dummy = ib.gpu_addr + 1024; + + /* stitch together an VCE destroy msg */ + ib.length_dw = 0; + ib.ptr[ib.length_dw++] = cpu_to_le32(0x0000000c); /* len */ + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001); /* session cmd */ + ib.ptr[ib.length_dw++] = cpu_to_le32(handle); + + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000014); /* len */ + ib.ptr[ib.length_dw++] = cpu_to_le32(0x05000005); /* feedback buffer */ + ib.ptr[ib.length_dw++] = cpu_to_le32(upper_32_bits(dummy)); + ib.ptr[ib.length_dw++] = cpu_to_le32(dummy); + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001); + + ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000008); /* len */ + ib.ptr[ib.length_dw++] = cpu_to_le32(0x02000001); /* destroy cmd */ + + for (i = ib.length_dw; i < ib_size_dw; ++i) + ib.ptr[i] = cpu_to_le32(0x0); + + r = radeon_ib_schedule(rdev, &ib, NULL, false); + if (r) { + DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); + } + + if (fence) + *fence = radeon_fence_ref(ib.fence); + + radeon_ib_free(rdev, &ib); + + return r; +} + +/** + * radeon_vce_cs_reloc - command submission relocation + * + * @p: parser context + * @lo: address of lower dword + * @hi: address of higher dword + * @size: size of checker for relocation buffer + * + * Patch relocation inside command stream with real buffer address + */ +int radeon_vce_cs_reloc(struct radeon_cs_parser *p, int lo, int hi, + unsigned size) +{ + struct radeon_cs_chunk *relocs_chunk; + struct radeon_bo_list *reloc; + uint64_t start, end, offset; + unsigned idx; + + relocs_chunk = p->chunk_relocs; + offset = radeon_get_ib_value(p, lo); + idx = radeon_get_ib_value(p, hi); + + if (idx >= relocs_chunk->length_dw) { + DRM_ERROR("Relocs at %d after relocations chunk end %d !\n", + idx, relocs_chunk->length_dw); + return -EINVAL; + } + + reloc = &p->relocs[(idx / 4)]; + start = reloc->gpu_offset; + end = start + radeon_bo_size(reloc->robj); + start += offset; + + p->ib.ptr[lo] = start & 0xFFFFFFFF; + p->ib.ptr[hi] = start >> 32; + + if (end <= start) { + DRM_ERROR("invalid reloc offset %llX!\n", offset); + return -EINVAL; + } + if ((end - start) < size) { + DRM_ERROR("buffer to small (%d / %d)!\n", + (unsigned)(end - start), size); + return -EINVAL; + } + + return 0; +} + +/** + * radeon_vce_validate_handle - validate stream handle + * + * @p: parser context + * @handle: handle to validate + * @allocated: allocated a new handle? + * + * Validates the handle and return the found session index or -EINVAL + * we we don't have another free session index. + */ +static int radeon_vce_validate_handle(struct radeon_cs_parser *p, + uint32_t handle, bool *allocated) +{ + unsigned i; + + *allocated = false; + + /* validate the handle */ + for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) { + if (atomic_read(&p->rdev->vce.handles[i]) == handle) { + if (p->rdev->vce.filp[i] != p->filp) { + DRM_ERROR("VCE handle collision detected!\n"); + return -EINVAL; + } + return i; + } + } + + /* handle not found try to alloc a new one */ + for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) { + if (!atomic_cmpxchg(&p->rdev->vce.handles[i], 0, handle)) { + p->rdev->vce.filp[i] = p->filp; + p->rdev->vce.img_size[i] = 0; + *allocated = true; + return i; + } + } + + DRM_ERROR("No more free VCE handles!\n"); + return -EINVAL; +} + +/** + * radeon_vce_cs_parse - parse and validate the command stream + * + * @p: parser context + * + */ +int radeon_vce_cs_parse(struct radeon_cs_parser *p) +{ + int session_idx = -1; + bool destroyed = false, created = false, allocated = false; + uint32_t tmp, handle = 0; + uint32_t *size = &tmp; + int i, r = 0; + + while (p->idx < p->chunk_ib->length_dw) { + uint32_t len = radeon_get_ib_value(p, p->idx); + uint32_t cmd = radeon_get_ib_value(p, p->idx + 1); + + if ((len < 8) || (len & 3)) { + DRM_ERROR("invalid VCE command length (%d)!\n", len); + r = -EINVAL; + goto out; + } + + if (destroyed) { + DRM_ERROR("No other command allowed after destroy!\n"); + r = -EINVAL; + goto out; + } + + switch (cmd) { + case 0x00000001: // session + handle = radeon_get_ib_value(p, p->idx + 2); + session_idx = radeon_vce_validate_handle(p, handle, + &allocated); + if (session_idx < 0) + return session_idx; + size = &p->rdev->vce.img_size[session_idx]; + break; + + case 0x00000002: // task info + break; + + case 0x01000001: // create + created = true; + if (!allocated) { + DRM_ERROR("Handle already in use!\n"); + r = -EINVAL; + goto out; + } + + *size = radeon_get_ib_value(p, p->idx + 8) * + radeon_get_ib_value(p, p->idx + 10) * + 8 * 3 / 2; + break; + + case 0x04000001: // config extension + case 0x04000002: // pic control + case 0x04000005: // rate control + case 0x04000007: // motion estimation + case 0x04000008: // rdo + case 0x04000009: // vui + break; + + case 0x03000001: // encode + r = radeon_vce_cs_reloc(p, p->idx + 10, p->idx + 9, + *size); + if (r) + goto out; + + r = radeon_vce_cs_reloc(p, p->idx + 12, p->idx + 11, + *size / 3); + if (r) + goto out; + break; + + case 0x02000001: // destroy + destroyed = true; + break; + + case 0x05000001: // context buffer + r = radeon_vce_cs_reloc(p, p->idx + 3, p->idx + 2, + *size * 2); + if (r) + goto out; + break; + + case 0x05000004: // video bitstream buffer + tmp = radeon_get_ib_value(p, p->idx + 4); + r = radeon_vce_cs_reloc(p, p->idx + 3, p->idx + 2, + tmp); + if (r) + goto out; + break; + + case 0x05000005: // feedback buffer + r = radeon_vce_cs_reloc(p, p->idx + 3, p->idx + 2, + 4096); + if (r) + goto out; + break; + + default: + DRM_ERROR("invalid VCE command (0x%x)!\n", cmd); + r = -EINVAL; + goto out; + } + + if (session_idx == -1) { + DRM_ERROR("no session command at start of IB\n"); + r = -EINVAL; + goto out; + } + + p->idx += len / 4; + } + + if (allocated && !created) { + DRM_ERROR("New session without create command!\n"); + r = -ENOENT; + } + +out: + if ((!r && destroyed) || (r && allocated)) { + /* + * IB contains a destroy msg or we have allocated an + * handle and got an error, anyway free the handle + */ + for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) + atomic_cmpxchg(&p->rdev->vce.handles[i], handle, 0); + } + + return r; +} + +/** + * radeon_vce_semaphore_emit - emit a semaphore command + * + * @rdev: radeon_device pointer + * @ring: engine to use + * @semaphore: address of semaphore + * @emit_wait: true=emit wait, false=emit signal + * + */ +bool radeon_vce_semaphore_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait) +{ + uint64_t addr = semaphore->gpu_addr; + + radeon_ring_write(ring, cpu_to_le32(VCE_CMD_SEMAPHORE)); + radeon_ring_write(ring, cpu_to_le32((addr >> 3) & 0x000FFFFF)); + radeon_ring_write(ring, cpu_to_le32((addr >> 23) & 0x000FFFFF)); + radeon_ring_write(ring, cpu_to_le32(0x01003000 | (emit_wait ? 1 : 0))); + if (!emit_wait) + radeon_ring_write(ring, cpu_to_le32(VCE_CMD_END)); + + return true; +} + +/** + * radeon_vce_ib_execute - execute indirect buffer + * + * @rdev: radeon_device pointer + * @ib: the IB to execute + * + */ +void radeon_vce_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) +{ + struct radeon_ring *ring = &rdev->ring[ib->ring]; + radeon_ring_write(ring, cpu_to_le32(VCE_CMD_IB)); + radeon_ring_write(ring, cpu_to_le32(ib->gpu_addr)); + radeon_ring_write(ring, cpu_to_le32(upper_32_bits(ib->gpu_addr))); + radeon_ring_write(ring, cpu_to_le32(ib->length_dw)); +} + +/** + * radeon_vce_fence_emit - add a fence command to the ring + * + * @rdev: radeon_device pointer + * @fence: the fence + * + */ +void radeon_vce_fence_emit(struct radeon_device *rdev, + struct radeon_fence *fence) +{ + struct radeon_ring *ring = &rdev->ring[fence->ring]; + uint64_t addr = rdev->fence_drv[fence->ring].gpu_addr; + + radeon_ring_write(ring, cpu_to_le32(VCE_CMD_FENCE)); + radeon_ring_write(ring, cpu_to_le32(addr)); + radeon_ring_write(ring, cpu_to_le32(upper_32_bits(addr))); + radeon_ring_write(ring, cpu_to_le32(fence->seq)); + radeon_ring_write(ring, cpu_to_le32(VCE_CMD_TRAP)); + radeon_ring_write(ring, cpu_to_le32(VCE_CMD_END)); +} + +/** + * radeon_vce_ring_test - test if VCE ring is working + * + * @rdev: radeon_device pointer + * @ring: the engine to test on + * + */ +int radeon_vce_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) +{ + uint32_t rptr = vce_v1_0_get_rptr(rdev, ring); + unsigned i; + int r; + + r = radeon_ring_lock(rdev, ring, 16); + if (r) { + DRM_ERROR("radeon: vce failed to lock ring %d (%d).\n", + ring->idx, r); + return r; + } + radeon_ring_write(ring, cpu_to_le32(VCE_CMD_END)); + radeon_ring_unlock_commit(rdev, ring, false); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (vce_v1_0_get_rptr(rdev, ring) != rptr) + break; + DRM_UDELAY(1); + } + + if (i < rdev->usec_timeout) { + DRM_INFO("ring test on %d succeeded in %d usecs\n", + ring->idx, i); + } else { + DRM_ERROR("radeon: ring %d test failed\n", + ring->idx); + r = -ETIMEDOUT; + } + + return r; +} + +/** + * radeon_vce_ib_test - test if VCE IBs are working + * + * @rdev: radeon_device pointer + * @ring: the engine to test on + * + */ +int radeon_vce_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) +{ + struct radeon_fence *fence = NULL; + int r; + + r = radeon_vce_get_create_msg(rdev, ring->idx, 1, NULL); + if (r) { + DRM_ERROR("radeon: failed to get create msg (%d).\n", r); + goto error; + } + + r = radeon_vce_get_destroy_msg(rdev, ring->idx, 1, &fence); + if (r) { + DRM_ERROR("radeon: failed to get destroy ib (%d).\n", r); + goto error; + } + + r = radeon_fence_wait(fence, false); + if (r) { + DRM_ERROR("radeon: fence wait failed (%d).\n", r); + } else { + DRM_INFO("ib test on ring %d succeeded\n", ring->idx); + } +error: + radeon_fence_unref(&fence); + return r; +} diff --git a/sys/dev/pci/drm/radeon/radeon_vm.c b/sys/dev/pci/drm/radeon/radeon_vm.c new file mode 100644 index 00000000000..6629c071b7f --- /dev/null +++ b/sys/dev/pci/drm/radeon/radeon_vm.c @@ -0,0 +1,1265 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ +#include +#include +#include "radeon.h" +#include "radeon_trace.h" + +/* + * GPUVM + * GPUVM is similar to the legacy gart on older asics, however + * rather than there being a single global gart table + * for the entire GPU, there are multiple VM page tables active + * at any given time. The VM page tables can contain a mix + * vram pages and system memory pages and system memory pages + * can be mapped as snooped (cached system pages) or unsnooped + * (uncached system pages). + * Each VM has an ID associated with it and there is a page table + * associated with each VMID. When execting a command buffer, + * the kernel tells the the ring what VMID to use for that command + * buffer. VMIDs are allocated dynamically as commands are submitted. + * The userspace drivers maintain their own address space and the kernel + * sets up their pages tables accordingly when they submit their + * command buffers and a VMID is assigned. + * Cayman/Trinity support up to 8 active VMs at any given time; + * SI supports 16. + */ + +/** + * radeon_vm_num_pde - return the number of page directory entries + * + * @rdev: radeon_device pointer + * + * Calculate the number of page directory entries (cayman+). + */ +static unsigned radeon_vm_num_pdes(struct radeon_device *rdev) +{ + return rdev->vm_manager.max_pfn >> radeon_vm_block_size; +} + +/** + * radeon_vm_directory_size - returns the size of the page directory in bytes + * + * @rdev: radeon_device pointer + * + * Calculate the size of the page directory in bytes (cayman+). + */ +static unsigned radeon_vm_directory_size(struct radeon_device *rdev) +{ + return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8); +} + +/** + * radeon_vm_manager_init - init the vm manager + * + * @rdev: radeon_device pointer + * + * Init the vm manager (cayman+). + * Returns 0 for success, error for failure. + */ +int radeon_vm_manager_init(struct radeon_device *rdev) +{ + int r; + + if (!rdev->vm_manager.enabled) { + r = radeon_asic_vm_init(rdev); + if (r) + return r; + + rdev->vm_manager.enabled = true; + } + return 0; +} + +/** + * radeon_vm_manager_fini - tear down the vm manager + * + * @rdev: radeon_device pointer + * + * Tear down the VM manager (cayman+). + */ +void radeon_vm_manager_fini(struct radeon_device *rdev) +{ + int i; + + if (!rdev->vm_manager.enabled) + return; + + for (i = 0; i < RADEON_NUM_VM; ++i) + radeon_fence_unref(&rdev->vm_manager.active[i]); + radeon_asic_vm_fini(rdev); + rdev->vm_manager.enabled = false; +} + +/** + * radeon_vm_get_bos - add the vm BOs to a validation list + * + * @vm: vm providing the BOs + * @head: head of validation list + * + * Add the page directory to the list of BOs to + * validate for command submission (cayman+). + */ +struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev, + struct radeon_vm *vm, + struct list_head *head) +{ + struct radeon_bo_list *list; + unsigned i, idx; + + list = drm_malloc_ab(vm->max_pde_used + 2, + sizeof(struct radeon_bo_list)); + if (!list) + return NULL; + + /* add the vm page table to the list */ + list[0].robj = vm->page_directory; + list[0].prefered_domains = RADEON_GEM_DOMAIN_VRAM; + list[0].allowed_domains = RADEON_GEM_DOMAIN_VRAM; + list[0].tv.bo = &vm->page_directory->tbo; + list[0].tv.shared = true; + list[0].tiling_flags = 0; + list_add(&list[0].tv.head, head); + + for (i = 0, idx = 1; i <= vm->max_pde_used; i++) { + if (!vm->page_tables[i].bo) + continue; + + list[idx].robj = vm->page_tables[i].bo; + list[idx].prefered_domains = RADEON_GEM_DOMAIN_VRAM; + list[idx].allowed_domains = RADEON_GEM_DOMAIN_VRAM; + list[idx].tv.bo = &list[idx].robj->tbo; + list[idx].tv.shared = true; + list[idx].tiling_flags = 0; + list_add(&list[idx++].tv.head, head); + } + + return list; +} + +/** + * radeon_vm_grab_id - allocate the next free VMID + * + * @rdev: radeon_device pointer + * @vm: vm to allocate id for + * @ring: ring we want to submit job to + * + * Allocate an id for the vm (cayman+). + * Returns the fence we need to sync to (if any). + * + * Global and local mutex must be locked! + */ +struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, + struct radeon_vm *vm, int ring) +{ + struct radeon_fence *best[RADEON_NUM_RINGS] = {}; + struct radeon_vm_id *vm_id = &vm->ids[ring]; + + unsigned choices[2] = {}; + unsigned i; + + /* check if the id is still valid */ + if (vm_id->id && vm_id->last_id_use && + vm_id->last_id_use == rdev->vm_manager.active[vm_id->id]) + return NULL; + + /* we definately need to flush */ + vm_id->pd_gpu_addr = ~0ll; + + /* skip over VMID 0, since it is the system VM */ + for (i = 1; i < rdev->vm_manager.nvm; ++i) { + struct radeon_fence *fence = rdev->vm_manager.active[i]; + + if (fence == NULL) { + /* found a free one */ + vm_id->id = i; + trace_radeon_vm_grab_id(i, ring); + return NULL; + } + + if (radeon_fence_is_earlier(fence, best[fence->ring])) { + best[fence->ring] = fence; + choices[fence->ring == ring ? 0 : 1] = i; + } + } + + for (i = 0; i < 2; ++i) { + if (choices[i]) { + vm_id->id = choices[i]; + trace_radeon_vm_grab_id(choices[i], ring); + return rdev->vm_manager.active[choices[i]]; + } + } + + /* should never happen */ + BUG(); + return NULL; +} + +/** + * radeon_vm_flush - hardware flush the vm + * + * @rdev: radeon_device pointer + * @vm: vm we want to flush + * @ring: ring to use for flush + * @updates: last vm update that is waited for + * + * Flush the vm (cayman+). + * + * Global and local mutex must be locked! + */ +void radeon_vm_flush(struct radeon_device *rdev, + struct radeon_vm *vm, + int ring, struct radeon_fence *updates) +{ + uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory); + struct radeon_vm_id *vm_id = &vm->ids[ring]; + + if (pd_addr != vm_id->pd_gpu_addr || !vm_id->flushed_updates || + radeon_fence_is_earlier(vm_id->flushed_updates, updates)) { + + trace_radeon_vm_flush(pd_addr, ring, vm->ids[ring].id); + radeon_fence_unref(&vm_id->flushed_updates); + vm_id->flushed_updates = radeon_fence_ref(updates); + vm_id->pd_gpu_addr = pd_addr; + radeon_ring_vm_flush(rdev, &rdev->ring[ring], + vm_id->id, vm_id->pd_gpu_addr); + + } +} + +/** + * radeon_vm_fence - remember fence for vm + * + * @rdev: radeon_device pointer + * @vm: vm we want to fence + * @fence: fence to remember + * + * Fence the vm (cayman+). + * Set the fence used to protect page table and id. + * + * Global and local mutex must be locked! + */ +void radeon_vm_fence(struct radeon_device *rdev, + struct radeon_vm *vm, + struct radeon_fence *fence) +{ + unsigned vm_id = vm->ids[fence->ring].id; + + radeon_fence_unref(&rdev->vm_manager.active[vm_id]); + rdev->vm_manager.active[vm_id] = radeon_fence_ref(fence); + + radeon_fence_unref(&vm->ids[fence->ring].last_id_use); + vm->ids[fence->ring].last_id_use = radeon_fence_ref(fence); +} + +/** + * radeon_vm_bo_find - find the bo_va for a specific vm & bo + * + * @vm: requested vm + * @bo: requested buffer object + * + * Find @bo inside the requested vm (cayman+). + * Search inside the @bos vm list for the requested vm + * Returns the found bo_va or NULL if none is found + * + * Object has to be reserved! + */ +struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm, + struct radeon_bo *bo) +{ + struct radeon_bo_va *bo_va; + + list_for_each_entry(bo_va, &bo->va, bo_list) { + if (bo_va->vm == vm) { + return bo_va; + } + } + return NULL; +} + +/** + * radeon_vm_bo_add - add a bo to a specific vm + * + * @rdev: radeon_device pointer + * @vm: requested vm + * @bo: radeon buffer object + * + * Add @bo into the requested vm (cayman+). + * Add @bo to the list of bos associated with the vm + * Returns newly added bo_va or NULL for failure + * + * Object has to be reserved! + */ +struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev, + struct radeon_vm *vm, + struct radeon_bo *bo) +{ + struct radeon_bo_va *bo_va; + + bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL); + if (bo_va == NULL) { + return NULL; + } + bo_va->vm = vm; + bo_va->bo = bo; + bo_va->it.start = 0; + bo_va->it.last = 0; + bo_va->flags = 0; + bo_va->ref_count = 1; + INIT_LIST_HEAD(&bo_va->bo_list); + INIT_LIST_HEAD(&bo_va->vm_status); + + mutex_lock(&vm->mutex); + list_add_tail(&bo_va->bo_list, &bo->va); + mutex_unlock(&vm->mutex); + + return bo_va; +} + +/** + * radeon_vm_set_pages - helper to call the right asic function + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: hw access flags + * + * Traces the parameters and calls the right asic functions + * to setup the page table using the DMA. + */ +static void radeon_vm_set_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags) +{ + trace_radeon_vm_set_page(pe, addr, count, incr, flags); + + if ((flags & R600_PTE_GART_MASK) == R600_PTE_GART_MASK) { + uint64_t src = rdev->gart.table_addr + (addr >> 12) * 8; + radeon_asic_vm_copy_pages(rdev, ib, pe, src, count); + + } else if ((flags & R600_PTE_SYSTEM) || (count < 3)) { + radeon_asic_vm_write_pages(rdev, ib, pe, addr, + count, incr, flags); + + } else { + radeon_asic_vm_set_pages(rdev, ib, pe, addr, + count, incr, flags); + } +} + +/** + * radeon_vm_clear_bo - initially clear the page dir/table + * + * @rdev: radeon_device pointer + * @bo: bo to clear + */ +static int radeon_vm_clear_bo(struct radeon_device *rdev, + struct radeon_bo *bo) +{ + struct radeon_ib ib; + unsigned entries; + uint64_t addr; + int r; + + r = radeon_bo_reserve(bo, false); + if (r) + return r; + + r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); + if (r) + goto error_unreserve; + + addr = radeon_bo_gpu_offset(bo); + entries = radeon_bo_size(bo) / 8; + + r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, 256); + if (r) + goto error_unreserve; + + ib.length_dw = 0; + + radeon_vm_set_pages(rdev, &ib, addr, 0, entries, 0, 0); + radeon_asic_vm_pad_ib(rdev, &ib); + WARN_ON(ib.length_dw > 64); + + r = radeon_ib_schedule(rdev, &ib, NULL, false); + if (r) + goto error_free; + + ib.fence->is_vm_update = true; + radeon_bo_fence(bo, ib.fence, false); + +error_free: + radeon_ib_free(rdev, &ib); + +error_unreserve: + radeon_bo_unreserve(bo); + return r; +} + +/** + * radeon_vm_bo_set_addr - set bos virtual address inside a vm + * + * @rdev: radeon_device pointer + * @bo_va: bo_va to store the address + * @soffset: requested offset of the buffer in the VM address space + * @flags: attributes of pages (read/write/valid/etc.) + * + * Set offset of @bo_va (cayman+). + * Validate and set the offset requested within the vm address space. + * Returns 0 for success, error for failure. + * + * Object has to be reserved and gets unreserved by this function! + */ +int radeon_vm_bo_set_addr(struct radeon_device *rdev, + struct radeon_bo_va *bo_va, + uint64_t soffset, + uint32_t flags) +{ + uint64_t size = radeon_bo_size(bo_va->bo); + struct radeon_vm *vm = bo_va->vm; + unsigned last_pfn, pt_idx; + uint64_t eoffset; + int r; + + if (soffset) { + /* make sure object fit at this offset */ + eoffset = soffset + size - 1; + if (soffset >= eoffset) { + r = -EINVAL; + goto error_unreserve; + } + + last_pfn = eoffset / RADEON_GPU_PAGE_SIZE; + if (last_pfn >= rdev->vm_manager.max_pfn) { + dev_err(rdev->dev, "va above limit (0x%08X >= 0x%08X)\n", + last_pfn, rdev->vm_manager.max_pfn); + r = -EINVAL; + goto error_unreserve; + } + + } else { + eoffset = last_pfn = 0; + } + + mutex_lock(&vm->mutex); + soffset /= RADEON_GPU_PAGE_SIZE; + eoffset /= RADEON_GPU_PAGE_SIZE; + if (soffset || eoffset) { + struct interval_tree_node *it; + it = interval_tree_iter_first(&vm->va, soffset, eoffset); + if (it && it != &bo_va->it) { + struct radeon_bo_va *tmp; + tmp = container_of(it, struct radeon_bo_va, it); + /* bo and tmp overlap, invalid offset */ + dev_err(rdev->dev, "bo %p va 0x%010llx conflict with " + "(bo %p 0x%010lx 0x%010lx)\n", bo_va->bo, + soffset, tmp->bo, tmp->it.start, tmp->it.last); + mutex_unlock(&vm->mutex); + r = -EINVAL; + goto error_unreserve; + } + } + + if (bo_va->it.start || bo_va->it.last) { + /* add a clone of the bo_va to clear the old address */ + struct radeon_bo_va *tmp; + tmp = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL); + if (!tmp) { + mutex_unlock(&vm->mutex); + r = -ENOMEM; + goto error_unreserve; + } + tmp->it.start = bo_va->it.start; + tmp->it.last = bo_va->it.last; + tmp->vm = vm; + tmp->bo = radeon_bo_ref(bo_va->bo); + + interval_tree_remove(&bo_va->it, &vm->va); + spin_lock(&vm->status_lock); + bo_va->it.start = 0; + bo_va->it.last = 0; + list_del_init(&bo_va->vm_status); + list_add(&tmp->vm_status, &vm->freed); + spin_unlock(&vm->status_lock); + } + + if (soffset || eoffset) { + spin_lock(&vm->status_lock); + bo_va->it.start = soffset; + bo_va->it.last = eoffset; + list_add(&bo_va->vm_status, &vm->cleared); + spin_unlock(&vm->status_lock); + interval_tree_insert(&bo_va->it, &vm->va); + } + + bo_va->flags = flags; + + soffset >>= radeon_vm_block_size; + eoffset >>= radeon_vm_block_size; + + BUG_ON(eoffset >= radeon_vm_num_pdes(rdev)); + + if (eoffset > vm->max_pde_used) + vm->max_pde_used = eoffset; + + radeon_bo_unreserve(bo_va->bo); + + /* walk over the address space and allocate the page tables */ + for (pt_idx = soffset; pt_idx <= eoffset; ++pt_idx) { + struct radeon_bo *pt; + + if (vm->page_tables[pt_idx].bo) + continue; + + /* drop mutex to allocate and clear page table */ + mutex_unlock(&vm->mutex); + + r = radeon_bo_create(rdev, RADEON_VM_PTE_COUNT * 8, + RADEON_GPU_PAGE_SIZE, true, + RADEON_GEM_DOMAIN_VRAM, 0, + NULL, NULL, &pt); + if (r) + return r; + + r = radeon_vm_clear_bo(rdev, pt); + if (r) { + radeon_bo_unref(&pt); + return r; + } + + /* aquire mutex again */ + mutex_lock(&vm->mutex); + if (vm->page_tables[pt_idx].bo) { + /* someone else allocated the pt in the meantime */ + mutex_unlock(&vm->mutex); + radeon_bo_unref(&pt); + mutex_lock(&vm->mutex); + continue; + } + + vm->page_tables[pt_idx].addr = 0; + vm->page_tables[pt_idx].bo = pt; + } + + mutex_unlock(&vm->mutex); + return 0; + +error_unreserve: + radeon_bo_unreserve(bo_va->bo); + return r; +} + +/** + * radeon_vm_map_gart - get the physical address of a gart page + * + * @rdev: radeon_device pointer + * @addr: the unmapped addr + * + * Look up the physical address of the page that the pte resolves + * to (cayman+). + * Returns the physical address of the page. + */ +uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr) +{ + uint64_t result; + + /* page table offset */ + result = rdev->gart.pages_entry[addr >> RADEON_GPU_PAGE_SHIFT]; + result &= ~RADEON_GPU_PAGE_MASK; + + return result; +} + +/** + * radeon_vm_page_flags - translate page flags to what the hw uses + * + * @flags: flags comming from userspace + * + * Translate the flags the userspace ABI uses to hw flags. + */ +static uint32_t radeon_vm_page_flags(uint32_t flags) +{ + uint32_t hw_flags = 0; + hw_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0; + hw_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0; + hw_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0; + if (flags & RADEON_VM_PAGE_SYSTEM) { + hw_flags |= R600_PTE_SYSTEM; + hw_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0; + } + return hw_flags; +} + +/** + * radeon_vm_update_pdes - make sure that page directory is valid + * + * @rdev: radeon_device pointer + * @vm: requested vm + * @start: start of GPU address range + * @end: end of GPU address range + * + * Allocates new page tables if necessary + * and updates the page directory (cayman+). + * Returns 0 for success, error for failure. + * + * Global and local mutex must be locked! + */ +int radeon_vm_update_page_directory(struct radeon_device *rdev, + struct radeon_vm *vm) +{ + struct radeon_bo *pd = vm->page_directory; + uint64_t pd_addr = radeon_bo_gpu_offset(pd); + uint32_t incr = RADEON_VM_PTE_COUNT * 8; + uint64_t last_pde = ~0, last_pt = ~0; + unsigned count = 0, pt_idx, ndw; + struct radeon_ib ib; + int r; + + /* padding, etc. */ + ndw = 64; + + /* assume the worst case */ + ndw += vm->max_pde_used * 6; + + /* update too big for an IB */ + if (ndw > 0xfffff) + return -ENOMEM; + + r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4); + if (r) + return r; + ib.length_dw = 0; + + /* walk over the address space and update the page directory */ + for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) { + struct radeon_bo *bo = vm->page_tables[pt_idx].bo; + uint64_t pde, pt; + + if (bo == NULL) + continue; + + pt = radeon_bo_gpu_offset(bo); + if (vm->page_tables[pt_idx].addr == pt) + continue; + vm->page_tables[pt_idx].addr = pt; + + pde = pd_addr + pt_idx * 8; + if (((last_pde + 8 * count) != pde) || + ((last_pt + incr * count) != pt)) { + + if (count) { + radeon_vm_set_pages(rdev, &ib, last_pde, + last_pt, count, incr, + R600_PTE_VALID); + } + + count = 1; + last_pde = pde; + last_pt = pt; + } else { + ++count; + } + } + + if (count) + radeon_vm_set_pages(rdev, &ib, last_pde, last_pt, count, + incr, R600_PTE_VALID); + + if (ib.length_dw != 0) { + radeon_asic_vm_pad_ib(rdev, &ib); + + radeon_sync_resv(rdev, &ib.sync, pd->tbo.resv, true); + WARN_ON(ib.length_dw > ndw); + r = radeon_ib_schedule(rdev, &ib, NULL, false); + if (r) { + radeon_ib_free(rdev, &ib); + return r; + } + ib.fence->is_vm_update = true; + radeon_bo_fence(pd, ib.fence, false); + } + radeon_ib_free(rdev, &ib); + + return 0; +} + +/** + * radeon_vm_frag_ptes - add fragment information to PTEs + * + * @rdev: radeon_device pointer + * @ib: IB for the update + * @pe_start: first PTE to handle + * @pe_end: last PTE to handle + * @addr: addr those PTEs should point to + * @flags: hw mapping flags + * + * Global and local mutex must be locked! + */ +static void radeon_vm_frag_ptes(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe_start, uint64_t pe_end, + uint64_t addr, uint32_t flags) +{ + /** + * The MC L1 TLB supports variable sized pages, based on a fragment + * field in the PTE. When this field is set to a non-zero value, page + * granularity is increased from 4KB to (1 << (12 + frag)). The PTE + * flags are considered valid for all PTEs within the fragment range + * and corresponding mappings are assumed to be physically contiguous. + * + * The L1 TLB can store a single PTE for the whole fragment, + * significantly increasing the space available for translation + * caching. This leads to large improvements in throughput when the + * TLB is under pressure. + * + * The L2 TLB distributes small and large fragments into two + * asymmetric partitions. The large fragment cache is significantly + * larger. Thus, we try to use large fragments wherever possible. + * Userspace can support this by aligning virtual base address and + * allocation size to the fragment size. + */ + + /* NI is optimized for 256KB fragments, SI and newer for 64KB */ + uint64_t frag_flags = ((rdev->family == CHIP_CAYMAN) || + (rdev->family == CHIP_ARUBA)) ? + R600_PTE_FRAG_256KB : R600_PTE_FRAG_64KB; + uint64_t frag_align = ((rdev->family == CHIP_CAYMAN) || + (rdev->family == CHIP_ARUBA)) ? 0x200 : 0x80; + + uint64_t frag_start = roundup2(pe_start, frag_align); + uint64_t frag_end = pe_end & ~(frag_align - 1); + + unsigned count; + + /* system pages are non continuously */ + if ((flags & R600_PTE_SYSTEM) || !(flags & R600_PTE_VALID) || + (frag_start >= frag_end)) { + + count = (pe_end - pe_start) / 8; + radeon_vm_set_pages(rdev, ib, pe_start, addr, count, + RADEON_GPU_PAGE_SIZE, flags); + return; + } + + /* handle the 4K area at the beginning */ + if (pe_start != frag_start) { + count = (frag_start - pe_start) / 8; + radeon_vm_set_pages(rdev, ib, pe_start, addr, count, + RADEON_GPU_PAGE_SIZE, flags); + addr += RADEON_GPU_PAGE_SIZE * count; + } + + /* handle the area in the middle */ + count = (frag_end - frag_start) / 8; + radeon_vm_set_pages(rdev, ib, frag_start, addr, count, + RADEON_GPU_PAGE_SIZE, flags | frag_flags); + + /* handle the 4K area at the end */ + if (frag_end != pe_end) { + addr += RADEON_GPU_PAGE_SIZE * count; + count = (pe_end - frag_end) / 8; + radeon_vm_set_pages(rdev, ib, frag_end, addr, count, + RADEON_GPU_PAGE_SIZE, flags); + } +} + +/** + * radeon_vm_update_ptes - make sure that page tables are valid + * + * @rdev: radeon_device pointer + * @vm: requested vm + * @start: start of GPU address range + * @end: end of GPU address range + * @dst: destination address to map to + * @flags: mapping flags + * + * Update the page tables in the range @start - @end (cayman+). + * + * Global and local mutex must be locked! + */ +static int radeon_vm_update_ptes(struct radeon_device *rdev, + struct radeon_vm *vm, + struct radeon_ib *ib, + uint64_t start, uint64_t end, + uint64_t dst, uint32_t flags) +{ + uint64_t mask = RADEON_VM_PTE_COUNT - 1; + uint64_t last_pte = ~0, last_dst = ~0; + unsigned count = 0; + uint64_t addr; + + /* walk over the address space and update the page tables */ + for (addr = start; addr < end; ) { + uint64_t pt_idx = addr >> radeon_vm_block_size; + struct radeon_bo *pt = vm->page_tables[pt_idx].bo; + unsigned nptes; + uint64_t pte; + int r; + + radeon_sync_resv(rdev, &ib->sync, pt->tbo.resv, true); + r = reservation_object_reserve_shared(pt->tbo.resv); + if (r) + return r; + + if ((addr & ~mask) == (end & ~mask)) + nptes = end - addr; + else + nptes = RADEON_VM_PTE_COUNT - (addr & mask); + + pte = radeon_bo_gpu_offset(pt); + pte += (addr & mask) * 8; + + if ((last_pte + 8 * count) != pte) { + + if (count) { + radeon_vm_frag_ptes(rdev, ib, last_pte, + last_pte + 8 * count, + last_dst, flags); + } + + count = nptes; + last_pte = pte; + last_dst = dst; + } else { + count += nptes; + } + + addr += nptes; + dst += nptes * RADEON_GPU_PAGE_SIZE; + } + + if (count) { + radeon_vm_frag_ptes(rdev, ib, last_pte, + last_pte + 8 * count, + last_dst, flags); + } + + return 0; +} + +/** + * radeon_vm_fence_pts - fence page tables after an update + * + * @vm: requested vm + * @start: start of GPU address range + * @end: end of GPU address range + * @fence: fence to use + * + * Fence the page tables in the range @start - @end (cayman+). + * + * Global and local mutex must be locked! + */ +static void radeon_vm_fence_pts(struct radeon_vm *vm, + uint64_t start, uint64_t end, + struct radeon_fence *fence) +{ + unsigned i; + + start >>= radeon_vm_block_size; + end = (end - 1) >> radeon_vm_block_size; + + for (i = start; i <= end; ++i) + radeon_bo_fence(vm->page_tables[i].bo, fence, true); +} + +/** + * radeon_vm_bo_update - map a bo into the vm page table + * + * @rdev: radeon_device pointer + * @vm: requested vm + * @bo: radeon buffer object + * @mem: ttm mem + * + * Fill in the page table entries for @bo (cayman+). + * Returns 0 for success, -EINVAL for failure. + * + * Object have to be reserved and mutex must be locked! + */ +int radeon_vm_bo_update(struct radeon_device *rdev, + struct radeon_bo_va *bo_va, + struct ttm_mem_reg *mem) +{ + struct radeon_vm *vm = bo_va->vm; + struct radeon_ib ib; + unsigned nptes, ncmds, ndw; + uint64_t addr; + uint32_t flags; + int r; + + if (!bo_va->it.start) { + dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n", + bo_va->bo, vm); + return -EINVAL; + } + + spin_lock(&vm->status_lock); + if (mem) { + if (list_empty(&bo_va->vm_status)) { + spin_unlock(&vm->status_lock); + return 0; + } + list_del_init(&bo_va->vm_status); + } else { + list_del(&bo_va->vm_status); + list_add(&bo_va->vm_status, &vm->cleared); + } + spin_unlock(&vm->status_lock); + + bo_va->flags &= ~RADEON_VM_PAGE_VALID; + bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM; + bo_va->flags &= ~RADEON_VM_PAGE_SNOOPED; + if (bo_va->bo && radeon_ttm_tt_is_readonly(bo_va->bo->tbo.ttm)) + bo_va->flags &= ~RADEON_VM_PAGE_WRITEABLE; + + if (mem) { + addr = mem->start << PAGE_SHIFT; + if (mem->mem_type != TTM_PL_SYSTEM) { + bo_va->flags |= RADEON_VM_PAGE_VALID; + } + if (mem->mem_type == TTM_PL_TT) { + bo_va->flags |= RADEON_VM_PAGE_SYSTEM; + if (!(bo_va->bo->flags & (RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC))) + bo_va->flags |= RADEON_VM_PAGE_SNOOPED; + + } else { + addr += rdev->vm_manager.vram_base_offset; + } + } else { + addr = 0; + } + + trace_radeon_vm_bo_update(bo_va); + + nptes = bo_va->it.last - bo_va->it.start + 1; + + /* reserve space for one command every (1 << BLOCK_SIZE) entries + or 2k dwords (whatever is smaller) */ + ncmds = (nptes >> min(radeon_vm_block_size, 11)) + 1; + + /* padding, etc. */ + ndw = 64; + + flags = radeon_vm_page_flags(bo_va->flags); + if ((flags & R600_PTE_GART_MASK) == R600_PTE_GART_MASK) { + /* only copy commands needed */ + ndw += ncmds * 7; + + } else if (flags & R600_PTE_SYSTEM) { + /* header for write data commands */ + ndw += ncmds * 4; + + /* body of write data command */ + ndw += nptes * 2; + + } else { + /* set page commands needed */ + ndw += ncmds * 10; + + /* two extra commands for begin/end of fragment */ + ndw += 2 * 10; + } + + /* update too big for an IB */ + if (ndw > 0xfffff) + return -ENOMEM; + + r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4); + if (r) + return r; + ib.length_dw = 0; + + if (!(bo_va->flags & RADEON_VM_PAGE_VALID)) { + unsigned i; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) + radeon_sync_fence(&ib.sync, vm->ids[i].last_id_use); + } + + r = radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start, + bo_va->it.last + 1, addr, + radeon_vm_page_flags(bo_va->flags)); + if (r) { + radeon_ib_free(rdev, &ib); + return r; + } + + radeon_asic_vm_pad_ib(rdev, &ib); + WARN_ON(ib.length_dw > ndw); + + r = radeon_ib_schedule(rdev, &ib, NULL, false); + if (r) { + radeon_ib_free(rdev, &ib); + return r; + } + ib.fence->is_vm_update = true; + radeon_vm_fence_pts(vm, bo_va->it.start, bo_va->it.last + 1, ib.fence); + radeon_fence_unref(&bo_va->last_pt_update); + bo_va->last_pt_update = radeon_fence_ref(ib.fence); + radeon_ib_free(rdev, &ib); + + return 0; +} + +/** + * radeon_vm_clear_freed - clear freed BOs in the PT + * + * @rdev: radeon_device pointer + * @vm: requested vm + * + * Make sure all freed BOs are cleared in the PT. + * Returns 0 for success. + * + * PTs have to be reserved and mutex must be locked! + */ +int radeon_vm_clear_freed(struct radeon_device *rdev, + struct radeon_vm *vm) +{ + struct radeon_bo_va *bo_va; + int r = 0; + + spin_lock(&vm->status_lock); + while (!list_empty(&vm->freed)) { + bo_va = list_first_entry(&vm->freed, + struct radeon_bo_va, vm_status); + spin_unlock(&vm->status_lock); + + r = radeon_vm_bo_update(rdev, bo_va, NULL); + radeon_bo_unref(&bo_va->bo); + radeon_fence_unref(&bo_va->last_pt_update); + spin_lock(&vm->status_lock); + list_del(&bo_va->vm_status); + kfree(bo_va); + if (r) + break; + + } + spin_unlock(&vm->status_lock); + return r; + +} + +/** + * radeon_vm_clear_invalids - clear invalidated BOs in the PT + * + * @rdev: radeon_device pointer + * @vm: requested vm + * + * Make sure all invalidated BOs are cleared in the PT. + * Returns 0 for success. + * + * PTs have to be reserved and mutex must be locked! + */ +int radeon_vm_clear_invalids(struct radeon_device *rdev, + struct radeon_vm *vm) +{ + struct radeon_bo_va *bo_va; + int r; + + spin_lock(&vm->status_lock); + while (!list_empty(&vm->invalidated)) { + bo_va = list_first_entry(&vm->invalidated, + struct radeon_bo_va, vm_status); + spin_unlock(&vm->status_lock); + + r = radeon_vm_bo_update(rdev, bo_va, NULL); + if (r) + return r; + + spin_lock(&vm->status_lock); + } + spin_unlock(&vm->status_lock); + + return 0; +} + +/** + * radeon_vm_bo_rmv - remove a bo to a specific vm + * + * @rdev: radeon_device pointer + * @bo_va: requested bo_va + * + * Remove @bo_va->bo from the requested vm (cayman+). + * + * Object have to be reserved! + */ +void radeon_vm_bo_rmv(struct radeon_device *rdev, + struct radeon_bo_va *bo_va) +{ + struct radeon_vm *vm = bo_va->vm; + + list_del(&bo_va->bo_list); + + mutex_lock(&vm->mutex); + if (bo_va->it.start || bo_va->it.last) + interval_tree_remove(&bo_va->it, &vm->va); + + spin_lock(&vm->status_lock); + list_del(&bo_va->vm_status); + if (bo_va->it.start || bo_va->it.last) { + bo_va->bo = radeon_bo_ref(bo_va->bo); + list_add(&bo_va->vm_status, &vm->freed); + } else { + radeon_fence_unref(&bo_va->last_pt_update); + kfree(bo_va); + } + spin_unlock(&vm->status_lock); + + mutex_unlock(&vm->mutex); +} + +/** + * radeon_vm_bo_invalidate - mark the bo as invalid + * + * @rdev: radeon_device pointer + * @vm: requested vm + * @bo: radeon buffer object + * + * Mark @bo as invalid (cayman+). + */ +void radeon_vm_bo_invalidate(struct radeon_device *rdev, + struct radeon_bo *bo) +{ + struct radeon_bo_va *bo_va; + + list_for_each_entry(bo_va, &bo->va, bo_list) { + spin_lock(&bo_va->vm->status_lock); + if (list_empty(&bo_va->vm_status) && + (bo_va->it.start || bo_va->it.last)) + list_add(&bo_va->vm_status, &bo_va->vm->invalidated); + spin_unlock(&bo_va->vm->status_lock); + } +} + +/** + * radeon_vm_init - initialize a vm instance + * + * @rdev: radeon_device pointer + * @vm: requested vm + * + * Init @vm fields (cayman+). + */ +int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) +{ + const unsigned align = min(RADEON_VM_PTB_ALIGN_SIZE, + RADEON_VM_PTE_COUNT * 8); + unsigned pd_size, pd_entries, pts_size; + int i, r; + + vm->ib_bo_va = NULL; + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + vm->ids[i].id = 0; + vm->ids[i].flushed_updates = NULL; + vm->ids[i].last_id_use = NULL; + } + rw_init(&vm->mutex, "vmlk"); + vm->va = RB_ROOT; + mtx_init(&vm->status_lock, IPL_TTY); + INIT_LIST_HEAD(&vm->invalidated); + INIT_LIST_HEAD(&vm->freed); + INIT_LIST_HEAD(&vm->cleared); + + pd_size = radeon_vm_directory_size(rdev); + pd_entries = radeon_vm_num_pdes(rdev); + + /* allocate page table array */ + pts_size = pd_entries * sizeof(struct radeon_vm_pt); + vm->page_tables = kzalloc(pts_size, GFP_KERNEL); + if (vm->page_tables == NULL) { + DRM_ERROR("Cannot allocate memory for page table array\n"); + return -ENOMEM; + } + + r = radeon_bo_create(rdev, pd_size, align, true, + RADEON_GEM_DOMAIN_VRAM, 0, NULL, + NULL, &vm->page_directory); + if (r) + return r; + + r = radeon_vm_clear_bo(rdev, vm->page_directory); + if (r) { + radeon_bo_unref(&vm->page_directory); + vm->page_directory = NULL; + return r; + } + + return 0; +} + +/** + * radeon_vm_fini - tear down a vm instance + * + * @rdev: radeon_device pointer + * @vm: requested vm + * + * Tear down @vm (cayman+). + * Unbind the VM and remove all bos from the vm bo list + */ +void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) +{ + struct radeon_bo_va *bo_va, *tmp; + int i, r; + + if (!RB_EMPTY_ROOT(&vm->va)) { + dev_err(rdev->dev, "still active bo inside vm\n"); + } + rbtree_postorder_for_each_entry_safe(bo_va, tmp, &vm->va, it.rb) { + interval_tree_remove(&bo_va->it, &vm->va); + r = radeon_bo_reserve(bo_va->bo, false); + if (!r) { + list_del_init(&bo_va->bo_list); + radeon_bo_unreserve(bo_va->bo); + radeon_fence_unref(&bo_va->last_pt_update); + kfree(bo_va); + } + } + list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) { + radeon_bo_unref(&bo_va->bo); + radeon_fence_unref(&bo_va->last_pt_update); + kfree(bo_va); + } + + for (i = 0; i < radeon_vm_num_pdes(rdev); i++) + radeon_bo_unref(&vm->page_tables[i].bo); + kfree(vm->page_tables); + + radeon_bo_unref(&vm->page_directory); + + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + radeon_fence_unref(&vm->ids[i].flushed_updates); + radeon_fence_unref(&vm->ids[i].last_id_use); + } + + mutex_destroy(&vm->mutex); +} diff --git a/sys/dev/pci/drm/radeon/reg_srcs/cayman b/sys/dev/pci/drm/radeon/reg_srcs/cayman index af7a941adf0..d46b58d078a 100644 --- a/sys/dev/pci/drm/radeon/reg_srcs/cayman +++ b/sys/dev/pci/drm/radeon/reg_srcs/cayman @@ -21,7 +21,7 @@ cayman 0x9400 0x000089AC VGT_COMPUTE_THREAD_GOURP_SIZE 0x000089B0 VGT_HS_OFFCHIP_PARAM 0x00008A14 PA_CL_ENHANCE -0x00008A60 PA_SC_LINE_STIPPLE_VALUE +0x00008A60 PA_SU_LINE_STIPPLE_VALUE 0x00008B10 PA_SC_LINE_STIPPLE_STATE 0x00008BF0 PA_SC_ENHANCE 0x00008D8C SQ_DYN_GPR_CNTL_PS_FLUSH_REQ diff --git a/sys/dev/pci/drm/radeon/reg_srcs/evergreen b/sys/dev/pci/drm/radeon/reg_srcs/evergreen index e19ef0e6ff2..57745c8761c 100644 --- a/sys/dev/pci/drm/radeon/reg_srcs/evergreen +++ b/sys/dev/pci/drm/radeon/reg_srcs/evergreen @@ -22,7 +22,7 @@ evergreen 0x9400 0x000089A4 VGT_COMPUTE_START_Z 0x000089AC VGT_COMPUTE_THREAD_GOURP_SIZE 0x00008A14 PA_CL_ENHANCE -0x00008A60 PA_SC_LINE_STIPPLE_VALUE +0x00008A60 PA_SU_LINE_STIPPLE_VALUE 0x00008B10 PA_SC_LINE_STIPPLE_STATE 0x00008BF0 PA_SC_ENHANCE 0x00008D8C SQ_DYN_GPR_CNTL_PS_FLUSH_REQ diff --git a/sys/dev/pci/drm/radeon/rs100d.h b/sys/dev/pci/drm/radeon/rs100d.h index 3d3a29de44f..48a913a06cf 100644 --- a/sys/dev/pci/drm/radeon/rs100d.h +++ b/sys/dev/pci/drm/radeon/rs100d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: rs100d.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/rs400.c b/sys/dev/pci/drm/radeon/rs400.c index 1a32211497b..6e2eb342d07 100644 --- a/sys/dev/pci/drm/radeon/rs400.c +++ b/sys/dev/pci/drm/radeon/rs400.c @@ -1,4 +1,3 @@ -/* $OpenBSD: rs400.c,v 1.9 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -108,7 +107,6 @@ int rs400_gart_enable(struct radeon_device *rdev) uint32_t size_reg; uint32_t tmp; - radeon_gart_restore(rdev); tmp = RREG32_MC(RS690_AIC_CTRL_SCRATCH); tmp |= RS690_DIS_OUT_OF_PCI_GART_ACCESS; WREG32_MC(RS690_AIC_CTRL_SCRATCH, tmp); @@ -208,24 +206,30 @@ void rs400_gart_fini(struct radeon_device *rdev) radeon_gart_table_ram_free(rdev); } +#define RS400_PTE_UNSNOOPED (1 << 0) #define RS400_PTE_WRITEABLE (1 << 2) #define RS400_PTE_READABLE (1 << 3) -int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr) +uint64_t rs400_gart_get_page_entry(uint64_t addr, uint32_t flags) { uint32_t entry; - u32 *gtt = rdev->gart.ptr; - - if (i < 0 || i > rdev->gart.num_gpu_pages) { - return -EINVAL; - } entry = (lower_32_bits(addr) & ~PAGE_MASK) | - ((upper_32_bits(addr) & 0xff) << 4) | - RS400_PTE_WRITEABLE | RS400_PTE_READABLE; - entry = cpu_to_le32(entry); - gtt[i] = entry; - return 0; + ((upper_32_bits(addr) & 0xff) << 4); + if (flags & RADEON_GART_PAGE_READ) + entry |= RS400_PTE_READABLE; + if (flags & RADEON_GART_PAGE_WRITE) + entry |= RS400_PTE_WRITEABLE; + if (!(flags & RADEON_GART_PAGE_SNOOP)) + entry |= RS400_PTE_UNSNOOPED; + return entry; +} + +void rs400_gart_set_page(struct radeon_device *rdev, unsigned i, + uint64_t entry) +{ + u32 *gtt = rdev->gart.ptr; + gtt[i] = cpu_to_le32(lower_32_bits(entry)); } int rs400_mc_wait_for_idle(struct radeon_device *rdev) @@ -273,19 +277,26 @@ static void rs400_mc_init(struct radeon_device *rdev) uint32_t rs400_mc_rreg(struct radeon_device *rdev, uint32_t reg) { + unsigned long flags; uint32_t r; + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(RS480_NB_MC_INDEX, reg & 0xff); r = RREG32(RS480_NB_MC_DATA); WREG32(RS480_NB_MC_INDEX, 0xff); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); return r; } void rs400_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v) { + unsigned long flags; + + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(RS480_NB_MC_INDEX, ((reg) & 0xff) | RS480_NB_MC_IND_WR_EN); WREG32(RS480_NB_MC_DATA, (v)); WREG32(RS480_NB_MC_INDEX, 0xff); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); } #if defined(CONFIG_DEBUG_FS) @@ -476,6 +487,7 @@ int rs400_resume(struct radeon_device *rdev) int rs400_suspend(struct radeon_device *rdev) { + radeon_pm_suspend(rdev); r100_cp_disable(rdev); radeon_wb_disable(rdev); r100_irq_disable(rdev); @@ -485,6 +497,7 @@ int rs400_suspend(struct radeon_device *rdev) void rs400_fini(struct radeon_device *rdev) { + radeon_pm_fini(rdev); r100_cp_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); @@ -552,6 +565,9 @@ int rs400_init(struct radeon_device *rdev) return r; r300_set_reg_safe(rdev); + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->accel_working = true; r = rs400_startup(rdev); if (r) { diff --git a/sys/dev/pci/drm/radeon/rs400d.h b/sys/dev/pci/drm/radeon/rs400d.h index d2f0156120e..6d8bac58ced 100644 --- a/sys/dev/pci/drm/radeon/rs400d.h +++ b/sys/dev/pci/drm/radeon/rs400d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: rs400d.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/rs600.c b/sys/dev/pci/drm/radeon/rs600.c index f531bd17c56..c6c7e2213a8 100644 --- a/sys/dev/pci/drm/radeon/rs600.c +++ b/sys/dev/pci/drm/radeon/rs600.c @@ -1,4 +1,3 @@ -/* $OpenBSD: rs600.c,v 1.10 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -39,6 +38,7 @@ #include #include "radeon.h" #include "radeon_asic.h" +#include "radeon_audio.h" #include "atom.h" #include "rs600d.h" @@ -110,19 +110,7 @@ void avivo_wait_for_vblank(struct radeon_device *rdev, int crtc) } } -void rs600_pre_page_flip(struct radeon_device *rdev, int crtc) -{ - /* enable the pflip int */ - radeon_irq_kms_pflip_irq_get(rdev, crtc); -} - -void rs600_post_page_flip(struct radeon_device *rdev, int crtc) -{ - /* disable the pflip int */ - radeon_irq_kms_pflip_irq_put(rdev, crtc); -} - -u32 rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) +void rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset); @@ -149,9 +137,79 @@ u32 rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) /* Unlock the lock, so double-buffering can take place inside vblank */ tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK; WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); +} + +bool rs600_page_flip_pending(struct radeon_device *rdev, int crtc_id) +{ + struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; /* Return current update_pending status: */ - return RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING; + return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & + AVIVO_D1GRPH_SURFACE_UPDATE_PENDING); +} + +void avivo_program_fmt(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + int bpc = 0; + u32 tmp = 0; + enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; + + if (connector) { + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + bpc = radeon_get_monitor_bpc(connector); + dither = radeon_connector->dither; + } + + /* LVDS FMT is set up by atom */ + if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) + return; + + if (bpc == 0) + return; + + switch (bpc) { + case 6: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= AVIVO_TMDS_BIT_DEPTH_CONTROL_SPATIAL_DITHER_EN; + else + tmp |= AVIVO_TMDS_BIT_DEPTH_CONTROL_TRUNCATE_EN; + break; + case 8: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= (AVIVO_TMDS_BIT_DEPTH_CONTROL_SPATIAL_DITHER_EN | + AVIVO_TMDS_BIT_DEPTH_CONTROL_SPATIAL_DITHER_DEPTH); + else + tmp |= (AVIVO_TMDS_BIT_DEPTH_CONTROL_TRUNCATE_EN | + AVIVO_TMDS_BIT_DEPTH_CONTROL_TRUNCATE_DEPTH); + break; + case 10: + default: + /* not needed */ + break; + } + + switch (radeon_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1: + WREG32(AVIVO_TMDSA_BIT_DEPTH_CONTROL, tmp); + break; + case ENCODER_OBJECT_ID_INTERNAL_LVTM1: + WREG32(AVIVO_LVTMA_BIT_DEPTH_CONTROL, tmp); + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: + WREG32(AVIVO_DVOA_BIT_DEPTH_CONTROL, tmp); + break; + case ENCODER_OBJECT_ID_INTERNAL_DDI: + WREG32(AVIVO_DDIA_BIT_DEPTH_CONTROL, tmp); + break; + default: + break; + } } void rs600_pm_misc(struct radeon_device *rdev) @@ -407,11 +465,9 @@ int rs600_asic_reset(struct radeon_device *rdev) WREG32(RADEON_CP_RB_RPTR_WR, 0); WREG32(RADEON_CP_RB_WPTR, 0); WREG32(RADEON_CP_RB_CNTL, tmp); -#ifdef notyet pci_save_state(rdev->pdev); /* disable bus mastering */ pci_clear_master(rdev->pdev); -#endif mdelay(1); /* reset GA+VAP */ WREG32(R_0000F0_RBBM_SOFT_RESET, S_0000F0_SOFT_RESET_VAP(1) | @@ -439,9 +495,7 @@ int rs600_asic_reset(struct radeon_device *rdev) status = RREG32(R_000E40_RBBM_STATUS); dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status); /* restore PCI & busmastering */ -#ifdef notyet pci_restore_state(rdev->pdev); -#endif /* Check if GPU is idle */ if (G_000E40_GA_BUSY(status) || G_000E40_VAP_BUSY(status)) { dev_err(rdev->dev, "failed to reset GPU\n"); @@ -502,7 +556,6 @@ static int rs600_gart_enable(struct radeon_device *rdev) r = radeon_gart_table_vram_pin(rdev); if (r) return r; - radeon_gart_restore(rdev); /* Enable bus master */ tmp = RREG32(RADEON_BUS_CNTL) & ~RS600_BUS_MASTER_DIS; WREG32(RADEON_BUS_CNTL, tmp); @@ -573,25 +626,26 @@ static void rs600_gart_fini(struct radeon_device *rdev) radeon_gart_table_vram_free(rdev); } -#define R600_PTE_VALID (1 << 0) -#define R600_PTE_SYSTEM (1 << 1) -#define R600_PTE_SNOOPED (1 << 2) -#define R600_PTE_READABLE (1 << 5) -#define R600_PTE_WRITEABLE (1 << 6) - -int rs600_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr) +uint64_t rs600_gart_get_page_entry(uint64_t addr, uint32_t flags) { - volatile uint64_t *ptr = rdev->gart.ptr; - - if (i < 0 || i > rdev->gart.num_gpu_pages) { - return -EINVAL; - } addr = addr & 0xFFFFFFFFFFFFF000ULL; - addr |= R600_PTE_VALID | R600_PTE_SYSTEM | R600_PTE_SNOOPED; - addr |= R600_PTE_READABLE | R600_PTE_WRITEABLE; - ptr += i; - *ptr = addr; - return 0; + addr |= R600_PTE_SYSTEM; + if (flags & RADEON_GART_PAGE_VALID) + addr |= R600_PTE_VALID; + if (flags & RADEON_GART_PAGE_READ) + addr |= R600_PTE_READABLE; + if (flags & RADEON_GART_PAGE_WRITE) + addr |= R600_PTE_WRITEABLE; + if (flags & RADEON_GART_PAGE_SNOOP) + addr |= R600_PTE_SNOOPED; + return addr; +} + +void rs600_gart_set_page(struct radeon_device *rdev, unsigned i, + uint64_t entry) +{ + void __iomem *ptr = (void *)rdev->gart.ptr; + writeq(entry, ptr + (i * 8)); } int rs600_irq_set(struct radeon_device *rdev) @@ -640,6 +694,10 @@ int rs600_irq_set(struct radeon_device *rdev) WREG32(R_007D18_DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2); if (ASIC_IS_DCE2(rdev)) WREG32(R_007408_HDMI0_AUDIO_PACKET_CONTROL, hdmi0); + + /* posting read */ + RREG32(R_000040_GEN_INT_CNTL); + return 0; } @@ -729,7 +787,7 @@ int rs600_irq_process(struct radeon_device *rdev) wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[0])) - radeon_crtc_handle_flip(rdev, 0); + radeon_crtc_handle_vblank(rdev, 0); } if (G_007EDC_LB_D2_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) { if (rdev->irq.crtc_vblank_int[1]) { @@ -738,7 +796,7 @@ int rs600_irq_process(struct radeon_device *rdev) wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[1])) - radeon_crtc_handle_flip(rdev, 1); + radeon_crtc_handle_vblank(rdev, 1); } if (G_007EDC_DC_HOT_PLUG_DETECT1_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) { queue_hotplug = true; @@ -755,9 +813,9 @@ int rs600_irq_process(struct radeon_device *rdev) status = rs600_irq_ack(rdev); } if (queue_hotplug) - task_add(systq, &rdev->hotplug_task); + schedule_delayed_work(&rdev->hotplug_work, 0); if (queue_hdmi) - task_add(systq, &rdev->audio_task); + schedule_work(&rdev->audio_work); if (rdev->msi_enabled) { switch (rdev->family) { case CHIP_RS600: @@ -830,6 +888,9 @@ void rs600_bandwidth_update(struct radeon_device *rdev) u32 d1mode_priority_a_cnt, d2mode_priority_a_cnt; /* FIXME: implement full support */ + if (!rdev->mode_info.mode_config_initialized) + return; + radeon_update_display_priority(rdev); if (rdev->mode_info.crtcs[0]->base.enabled) @@ -853,16 +914,26 @@ void rs600_bandwidth_update(struct radeon_device *rdev) uint32_t rs600_mc_rreg(struct radeon_device *rdev, uint32_t reg) { + unsigned long flags; + u32 r; + + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(R_000070_MC_IND_INDEX, S_000070_MC_IND_ADDR(reg) | S_000070_MC_IND_CITF_ARB0(1)); - return RREG32(R_000074_MC_IND_DATA); + r = RREG32(R_000074_MC_IND_DATA); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); + return r; } void rs600_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v) { + unsigned long flags; + + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(R_000070_MC_IND_INDEX, S_000070_MC_IND_ADDR(reg) | S_000070_MC_IND_CITF_ARB0(1) | S_000070_MC_IND_WR_EN(1)); WREG32(R_000074_MC_IND_DATA, v); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); } static void rs600_debugfs(struct radeon_device *rdev) @@ -950,7 +1021,7 @@ static int rs600_startup(struct radeon_device *rdev) return r; } - r = r600_audio_init(rdev); + r = radeon_audio_init(rdev); if (r) { dev_err(rdev->dev, "failed initializing audio\n"); return r; @@ -990,7 +1061,8 @@ int rs600_resume(struct radeon_device *rdev) int rs600_suspend(struct radeon_device *rdev) { - r600_audio_fini(rdev); + radeon_pm_suspend(rdev); + radeon_audio_fini(rdev); r100_cp_disable(rdev); radeon_wb_disable(rdev); rs600_irq_disable(rdev); @@ -1000,7 +1072,8 @@ int rs600_suspend(struct radeon_device *rdev) void rs600_fini(struct radeon_device *rdev) { - r600_audio_fini(rdev); + radeon_pm_fini(rdev); + radeon_audio_fini(rdev); r100_cp_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); @@ -1068,6 +1141,9 @@ int rs600_init(struct radeon_device *rdev) return r; rs600_set_safe_registers(rdev); + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->accel_working = true; r = rs600_startup(rdev); if (r) { diff --git a/sys/dev/pci/drm/radeon/rs600d.h b/sys/dev/pci/drm/radeon/rs600d.h index 20e76044f83..f1f89414dc6 100644 --- a/sys/dev/pci/drm/radeon/rs600d.h +++ b/sys/dev/pci/drm/radeon/rs600d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: rs600d.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/rs690.c b/sys/dev/pci/drm/radeon/rs690.c index 5ff501a2ac2..488341c2021 100644 --- a/sys/dev/pci/drm/radeon/rs690.c +++ b/sys/dev/pci/drm/radeon/rs690.c @@ -1,4 +1,3 @@ -/* $OpenBSD: rs690.c,v 1.8 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -29,6 +28,7 @@ #include #include "radeon.h" #include "radeon_asic.h" +#include "radeon_audio.h" #include "atom.h" #include "rs690d.h" @@ -149,6 +149,8 @@ void rs690_pm_info(struct radeon_device *rdev) static void rs690_mc_init(struct radeon_device *rdev) { u64 base; + uint32_t h_addr, l_addr; + unsigned long long k8_addr; rs400_gart_adjust_size(rdev); rdev->mc.vram_is_ddr = true; @@ -172,6 +174,26 @@ static void rs690_mc_init(struct radeon_device *rdev) rdev->mc.mc_vram_size = rdev->mc.real_vram_size; } + /* Use K8 direct mapping for fast fb access. */ + rdev->fastfb_working = false; + h_addr = G_00005F_K8_ADDR_EXT(RREG32_MC(R_00005F_MC_MISC_UMA_CNTL)); + l_addr = RREG32_MC(R_00001E_K8_FB_LOCATION); + k8_addr = ((unsigned long long)h_addr) << 32 | l_addr; +#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE) + if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL) +#endif + { + /* FastFB shall be used with UMA memory. Here it is simply disabled when sideport + * memory is present. + */ + if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) { + DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n", + (unsigned long long)rdev->mc.aper_base, k8_addr); + rdev->mc.aper_base = (resource_size_t)k8_addr; + rdev->fastfb_working = true; + } + } + rs690_pm_info(rdev); radeon_vram_location(rdev, &rdev->mc, base); rdev->mc.gtt_base_align = rdev->mc.gtt_size - 1; @@ -185,6 +207,9 @@ void rs690_line_buffer_adjust(struct radeon_device *rdev, { u32 tmp; + /* Guess line buffer size to be 8192 pixels */ + u32 lb_size = 8192; + /* * Line Buffer Setup * There is a single line buffer shared by both display controllers. @@ -221,6 +246,13 @@ void rs690_line_buffer_adjust(struct radeon_device *rdev, tmp |= V_006520_DC_LB_MEMORY_SPLIT_D1_1Q_D2_3Q; } WREG32(R_006520_DC_LB_MEMORY_SPLIT, tmp); + + /* Save number of lines the linebuffer leads before the scanout */ + if (mode1) + rdev->mode_info.crtcs[0]->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode1->crtc_hdisplay); + + if (mode2) + rdev->mode_info.crtcs[1]->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode2->crtc_hdisplay); } struct rs690_watermark { @@ -237,13 +269,16 @@ struct rs690_watermark { }; static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev, - struct radeon_crtc *crtc, - struct rs690_watermark *wm) + struct radeon_crtc *crtc, + struct rs690_watermark *wm, + bool low) { struct drm_display_mode *mode = &crtc->base.mode; fixed20_12 a, b, c; fixed20_12 pclk, request_fifo_depth, tolerable_latency, estimated_width; fixed20_12 consumption_time, line_time, chunk_time, read_delay_latency; + fixed20_12 sclk, core_bandwidth, max_bandwidth; + u32 selected_sclk; if (!crtc->base.enabled) { /* FIXME: wouldn't it better to set priority mark to maximum */ @@ -251,6 +286,21 @@ static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev, return; } + if (((rdev->family == CHIP_RS780) || (rdev->family == CHIP_RS880)) && + (rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) + selected_sclk = radeon_dpm_get_sclk(rdev, low); + else + selected_sclk = rdev->pm.current_sclk; + + /* sclk in Mhz */ + a.full = dfixed_const(100); + sclk.full = dfixed_const(selected_sclk); + sclk.full = dfixed_div(sclk, a); + + /* core_bandwidth = sclk(Mhz) * 16 */ + a.full = dfixed_const(16); + core_bandwidth.full = dfixed_div(rdev->pm.sclk, a); + if (crtc->vsc.full > dfixed_const(2)) wm->num_line_pair.full = dfixed_const(2); else @@ -311,38 +361,38 @@ static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev, wm->active_time.full = dfixed_div(wm->active_time, a); /* Maximun bandwidth is the minimun bandwidth of all component */ - rdev->pm.max_bandwidth = rdev->pm.core_bandwidth; + max_bandwidth = core_bandwidth; if (rdev->mc.igp_sideport_enabled) { - if (rdev->pm.max_bandwidth.full > rdev->pm.sideport_bandwidth.full && + if (max_bandwidth.full > rdev->pm.sideport_bandwidth.full && rdev->pm.sideport_bandwidth.full) - rdev->pm.max_bandwidth = rdev->pm.sideport_bandwidth; + max_bandwidth = rdev->pm.sideport_bandwidth; read_delay_latency.full = dfixed_const(370 * 800); a.full = dfixed_const(1000); b.full = dfixed_div(rdev->pm.igp_sideport_mclk, a); read_delay_latency.full = dfixed_div(read_delay_latency, b); read_delay_latency.full = dfixed_mul(read_delay_latency, a); } else { - if (rdev->pm.max_bandwidth.full > rdev->pm.k8_bandwidth.full && + if (max_bandwidth.full > rdev->pm.k8_bandwidth.full && rdev->pm.k8_bandwidth.full) - rdev->pm.max_bandwidth = rdev->pm.k8_bandwidth; - if (rdev->pm.max_bandwidth.full > rdev->pm.ht_bandwidth.full && + max_bandwidth = rdev->pm.k8_bandwidth; + if (max_bandwidth.full > rdev->pm.ht_bandwidth.full && rdev->pm.ht_bandwidth.full) - rdev->pm.max_bandwidth = rdev->pm.ht_bandwidth; + max_bandwidth = rdev->pm.ht_bandwidth; read_delay_latency.full = dfixed_const(5000); } /* sclk = system clocks(ns) = 1000 / max_bandwidth / 16 */ a.full = dfixed_const(16); - rdev->pm.sclk.full = dfixed_mul(rdev->pm.max_bandwidth, a); + sclk.full = dfixed_mul(max_bandwidth, a); a.full = dfixed_const(1000); - rdev->pm.sclk.full = dfixed_div(a, rdev->pm.sclk); + sclk.full = dfixed_div(a, sclk); /* Determine chunk time * ChunkTime = the time it takes the DCP to send one chunk of data * to the LB which consists of pipeline delay and inter chunk gap * sclk = system clock(ns) */ a.full = dfixed_const(256 * 13); - chunk_time.full = dfixed_mul(rdev->pm.sclk, a); + chunk_time.full = dfixed_mul(sclk, a); a.full = dfixed_const(10); chunk_time.full = dfixed_div(chunk_time, a); @@ -406,185 +456,220 @@ static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev, } } -void rs690_bandwidth_update(struct radeon_device *rdev) +static void rs690_compute_mode_priority(struct radeon_device *rdev, + struct rs690_watermark *wm0, + struct rs690_watermark *wm1, + struct drm_display_mode *mode0, + struct drm_display_mode *mode1, + u32 *d1mode_priority_a_cnt, + u32 *d2mode_priority_a_cnt) { - struct drm_display_mode *mode0 = NULL; - struct drm_display_mode *mode1 = NULL; - struct rs690_watermark wm0; - struct rs690_watermark wm1; - u32 tmp; - u32 d1mode_priority_a_cnt = S_006548_D1MODE_PRIORITY_A_OFF(1); - u32 d2mode_priority_a_cnt = S_006548_D1MODE_PRIORITY_A_OFF(1); fixed20_12 priority_mark02, priority_mark12, fill_rate; fixed20_12 a, b; - radeon_update_display_priority(rdev); - - if (rdev->mode_info.crtcs[0]->base.enabled) - mode0 = &rdev->mode_info.crtcs[0]->base.mode; - if (rdev->mode_info.crtcs[1]->base.enabled) - mode1 = &rdev->mode_info.crtcs[1]->base.mode; - /* - * Set display0/1 priority up in the memory controller for - * modes if the user specifies HIGH for displaypriority - * option. - */ - if ((rdev->disp_priority == 2) && - ((rdev->family == CHIP_RS690) || (rdev->family == CHIP_RS740))) { - tmp = RREG32_MC(R_000104_MC_INIT_MISC_LAT_TIMER); - tmp &= C_000104_MC_DISP0R_INIT_LAT; - tmp &= C_000104_MC_DISP1R_INIT_LAT; - if (mode0) - tmp |= S_000104_MC_DISP0R_INIT_LAT(1); - if (mode1) - tmp |= S_000104_MC_DISP1R_INIT_LAT(1); - WREG32_MC(R_000104_MC_INIT_MISC_LAT_TIMER, tmp); - } - rs690_line_buffer_adjust(rdev, mode0, mode1); - - if ((rdev->family == CHIP_RS690) || (rdev->family == CHIP_RS740)) - WREG32(R_006C9C_DCP_CONTROL, 0); - if ((rdev->family == CHIP_RS780) || (rdev->family == CHIP_RS880)) - WREG32(R_006C9C_DCP_CONTROL, 2); - - rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0); - rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1); - - tmp = (wm0.lb_request_fifo_depth - 1); - tmp |= (wm1.lb_request_fifo_depth - 1) << 16; - WREG32(R_006D58_LB_MAX_REQ_OUTSTANDING, tmp); + *d1mode_priority_a_cnt = S_006548_D1MODE_PRIORITY_A_OFF(1); + *d2mode_priority_a_cnt = S_006548_D1MODE_PRIORITY_A_OFF(1); if (mode0 && mode1) { - if (dfixed_trunc(wm0.dbpp) > 64) - a.full = dfixed_mul(wm0.dbpp, wm0.num_line_pair); + if (dfixed_trunc(wm0->dbpp) > 64) + a.full = dfixed_mul(wm0->dbpp, wm0->num_line_pair); else - a.full = wm0.num_line_pair.full; - if (dfixed_trunc(wm1.dbpp) > 64) - b.full = dfixed_mul(wm1.dbpp, wm1.num_line_pair); + a.full = wm0->num_line_pair.full; + if (dfixed_trunc(wm1->dbpp) > 64) + b.full = dfixed_mul(wm1->dbpp, wm1->num_line_pair); else - b.full = wm1.num_line_pair.full; + b.full = wm1->num_line_pair.full; a.full += b.full; - fill_rate.full = dfixed_div(wm0.sclk, a); - if (wm0.consumption_rate.full > fill_rate.full) { - b.full = wm0.consumption_rate.full - fill_rate.full; - b.full = dfixed_mul(b, wm0.active_time); - a.full = dfixed_mul(wm0.worst_case_latency, - wm0.consumption_rate); + fill_rate.full = dfixed_div(wm0->sclk, a); + if (wm0->consumption_rate.full > fill_rate.full) { + b.full = wm0->consumption_rate.full - fill_rate.full; + b.full = dfixed_mul(b, wm0->active_time); + a.full = dfixed_mul(wm0->worst_case_latency, + wm0->consumption_rate); a.full = a.full + b.full; b.full = dfixed_const(16 * 1000); priority_mark02.full = dfixed_div(a, b); } else { - a.full = dfixed_mul(wm0.worst_case_latency, - wm0.consumption_rate); + a.full = dfixed_mul(wm0->worst_case_latency, + wm0->consumption_rate); b.full = dfixed_const(16 * 1000); priority_mark02.full = dfixed_div(a, b); } - if (wm1.consumption_rate.full > fill_rate.full) { - b.full = wm1.consumption_rate.full - fill_rate.full; - b.full = dfixed_mul(b, wm1.active_time); - a.full = dfixed_mul(wm1.worst_case_latency, - wm1.consumption_rate); + if (wm1->consumption_rate.full > fill_rate.full) { + b.full = wm1->consumption_rate.full - fill_rate.full; + b.full = dfixed_mul(b, wm1->active_time); + a.full = dfixed_mul(wm1->worst_case_latency, + wm1->consumption_rate); a.full = a.full + b.full; b.full = dfixed_const(16 * 1000); priority_mark12.full = dfixed_div(a, b); } else { - a.full = dfixed_mul(wm1.worst_case_latency, - wm1.consumption_rate); + a.full = dfixed_mul(wm1->worst_case_latency, + wm1->consumption_rate); b.full = dfixed_const(16 * 1000); priority_mark12.full = dfixed_div(a, b); } - if (wm0.priority_mark.full > priority_mark02.full) - priority_mark02.full = wm0.priority_mark.full; - if (wm0.priority_mark_max.full > priority_mark02.full) - priority_mark02.full = wm0.priority_mark_max.full; - if (wm1.priority_mark.full > priority_mark12.full) - priority_mark12.full = wm1.priority_mark.full; - if (wm1.priority_mark_max.full > priority_mark12.full) - priority_mark12.full = wm1.priority_mark_max.full; - d1mode_priority_a_cnt = dfixed_trunc(priority_mark02); - d2mode_priority_a_cnt = dfixed_trunc(priority_mark12); + if (wm0->priority_mark.full > priority_mark02.full) + priority_mark02.full = wm0->priority_mark.full; + if (wm0->priority_mark_max.full > priority_mark02.full) + priority_mark02.full = wm0->priority_mark_max.full; + if (wm1->priority_mark.full > priority_mark12.full) + priority_mark12.full = wm1->priority_mark.full; + if (wm1->priority_mark_max.full > priority_mark12.full) + priority_mark12.full = wm1->priority_mark_max.full; + *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02); + *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12); if (rdev->disp_priority == 2) { - d1mode_priority_a_cnt |= S_006548_D1MODE_PRIORITY_A_ALWAYS_ON(1); - d2mode_priority_a_cnt |= S_006D48_D2MODE_PRIORITY_A_ALWAYS_ON(1); + *d1mode_priority_a_cnt |= S_006548_D1MODE_PRIORITY_A_ALWAYS_ON(1); + *d2mode_priority_a_cnt |= S_006D48_D2MODE_PRIORITY_A_ALWAYS_ON(1); } } else if (mode0) { - if (dfixed_trunc(wm0.dbpp) > 64) - a.full = dfixed_mul(wm0.dbpp, wm0.num_line_pair); + if (dfixed_trunc(wm0->dbpp) > 64) + a.full = dfixed_mul(wm0->dbpp, wm0->num_line_pair); else - a.full = wm0.num_line_pair.full; - fill_rate.full = dfixed_div(wm0.sclk, a); - if (wm0.consumption_rate.full > fill_rate.full) { - b.full = wm0.consumption_rate.full - fill_rate.full; - b.full = dfixed_mul(b, wm0.active_time); - a.full = dfixed_mul(wm0.worst_case_latency, - wm0.consumption_rate); + a.full = wm0->num_line_pair.full; + fill_rate.full = dfixed_div(wm0->sclk, a); + if (wm0->consumption_rate.full > fill_rate.full) { + b.full = wm0->consumption_rate.full - fill_rate.full; + b.full = dfixed_mul(b, wm0->active_time); + a.full = dfixed_mul(wm0->worst_case_latency, + wm0->consumption_rate); a.full = a.full + b.full; b.full = dfixed_const(16 * 1000); priority_mark02.full = dfixed_div(a, b); } else { - a.full = dfixed_mul(wm0.worst_case_latency, - wm0.consumption_rate); + a.full = dfixed_mul(wm0->worst_case_latency, + wm0->consumption_rate); b.full = dfixed_const(16 * 1000); priority_mark02.full = dfixed_div(a, b); } - if (wm0.priority_mark.full > priority_mark02.full) - priority_mark02.full = wm0.priority_mark.full; - if (wm0.priority_mark_max.full > priority_mark02.full) - priority_mark02.full = wm0.priority_mark_max.full; - d1mode_priority_a_cnt = dfixed_trunc(priority_mark02); + if (wm0->priority_mark.full > priority_mark02.full) + priority_mark02.full = wm0->priority_mark.full; + if (wm0->priority_mark_max.full > priority_mark02.full) + priority_mark02.full = wm0->priority_mark_max.full; + *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02); if (rdev->disp_priority == 2) - d1mode_priority_a_cnt |= S_006548_D1MODE_PRIORITY_A_ALWAYS_ON(1); + *d1mode_priority_a_cnt |= S_006548_D1MODE_PRIORITY_A_ALWAYS_ON(1); } else if (mode1) { - if (dfixed_trunc(wm1.dbpp) > 64) - a.full = dfixed_mul(wm1.dbpp, wm1.num_line_pair); + if (dfixed_trunc(wm1->dbpp) > 64) + a.full = dfixed_mul(wm1->dbpp, wm1->num_line_pair); else - a.full = wm1.num_line_pair.full; - fill_rate.full = dfixed_div(wm1.sclk, a); - if (wm1.consumption_rate.full > fill_rate.full) { - b.full = wm1.consumption_rate.full - fill_rate.full; - b.full = dfixed_mul(b, wm1.active_time); - a.full = dfixed_mul(wm1.worst_case_latency, - wm1.consumption_rate); + a.full = wm1->num_line_pair.full; + fill_rate.full = dfixed_div(wm1->sclk, a); + if (wm1->consumption_rate.full > fill_rate.full) { + b.full = wm1->consumption_rate.full - fill_rate.full; + b.full = dfixed_mul(b, wm1->active_time); + a.full = dfixed_mul(wm1->worst_case_latency, + wm1->consumption_rate); a.full = a.full + b.full; b.full = dfixed_const(16 * 1000); priority_mark12.full = dfixed_div(a, b); } else { - a.full = dfixed_mul(wm1.worst_case_latency, - wm1.consumption_rate); + a.full = dfixed_mul(wm1->worst_case_latency, + wm1->consumption_rate); b.full = dfixed_const(16 * 1000); priority_mark12.full = dfixed_div(a, b); } - if (wm1.priority_mark.full > priority_mark12.full) - priority_mark12.full = wm1.priority_mark.full; - if (wm1.priority_mark_max.full > priority_mark12.full) - priority_mark12.full = wm1.priority_mark_max.full; - d2mode_priority_a_cnt = dfixed_trunc(priority_mark12); + if (wm1->priority_mark.full > priority_mark12.full) + priority_mark12.full = wm1->priority_mark.full; + if (wm1->priority_mark_max.full > priority_mark12.full) + priority_mark12.full = wm1->priority_mark_max.full; + *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12); if (rdev->disp_priority == 2) - d2mode_priority_a_cnt |= S_006D48_D2MODE_PRIORITY_A_ALWAYS_ON(1); + *d2mode_priority_a_cnt |= S_006D48_D2MODE_PRIORITY_A_ALWAYS_ON(1); } +} + +void rs690_bandwidth_update(struct radeon_device *rdev) +{ + struct drm_display_mode *mode0 = NULL; + struct drm_display_mode *mode1 = NULL; + struct rs690_watermark wm0_high, wm0_low; + struct rs690_watermark wm1_high, wm1_low; + u32 tmp; + u32 d1mode_priority_a_cnt, d1mode_priority_b_cnt; + u32 d2mode_priority_a_cnt, d2mode_priority_b_cnt; + + if (!rdev->mode_info.mode_config_initialized) + return; + + radeon_update_display_priority(rdev); + + if (rdev->mode_info.crtcs[0]->base.enabled) + mode0 = &rdev->mode_info.crtcs[0]->base.mode; + if (rdev->mode_info.crtcs[1]->base.enabled) + mode1 = &rdev->mode_info.crtcs[1]->base.mode; + /* + * Set display0/1 priority up in the memory controller for + * modes if the user specifies HIGH for displaypriority + * option. + */ + if ((rdev->disp_priority == 2) && + ((rdev->family == CHIP_RS690) || (rdev->family == CHIP_RS740))) { + tmp = RREG32_MC(R_000104_MC_INIT_MISC_LAT_TIMER); + tmp &= C_000104_MC_DISP0R_INIT_LAT; + tmp &= C_000104_MC_DISP1R_INIT_LAT; + if (mode0) + tmp |= S_000104_MC_DISP0R_INIT_LAT(1); + if (mode1) + tmp |= S_000104_MC_DISP1R_INIT_LAT(1); + WREG32_MC(R_000104_MC_INIT_MISC_LAT_TIMER, tmp); + } + rs690_line_buffer_adjust(rdev, mode0, mode1); + + if ((rdev->family == CHIP_RS690) || (rdev->family == CHIP_RS740)) + WREG32(R_006C9C_DCP_CONTROL, 0); + if ((rdev->family == CHIP_RS780) || (rdev->family == CHIP_RS880)) + WREG32(R_006C9C_DCP_CONTROL, 2); + + rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0_high, false); + rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1_high, false); + + rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0_low, true); + rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1_low, true); + + tmp = (wm0_high.lb_request_fifo_depth - 1); + tmp |= (wm1_high.lb_request_fifo_depth - 1) << 16; + WREG32(R_006D58_LB_MAX_REQ_OUTSTANDING, tmp); + + rs690_compute_mode_priority(rdev, + &wm0_high, &wm1_high, + mode0, mode1, + &d1mode_priority_a_cnt, &d2mode_priority_a_cnt); + rs690_compute_mode_priority(rdev, + &wm0_low, &wm1_low, + mode0, mode1, + &d1mode_priority_b_cnt, &d2mode_priority_b_cnt); WREG32(R_006548_D1MODE_PRIORITY_A_CNT, d1mode_priority_a_cnt); - WREG32(R_00654C_D1MODE_PRIORITY_B_CNT, d1mode_priority_a_cnt); + WREG32(R_00654C_D1MODE_PRIORITY_B_CNT, d1mode_priority_b_cnt); WREG32(R_006D48_D2MODE_PRIORITY_A_CNT, d2mode_priority_a_cnt); - WREG32(R_006D4C_D2MODE_PRIORITY_B_CNT, d2mode_priority_a_cnt); + WREG32(R_006D4C_D2MODE_PRIORITY_B_CNT, d2mode_priority_b_cnt); } uint32_t rs690_mc_rreg(struct radeon_device *rdev, uint32_t reg) { + unsigned long flags; uint32_t r; + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(R_000078_MC_INDEX, S_000078_MC_IND_ADDR(reg)); r = RREG32(R_00007C_MC_DATA); WREG32(R_000078_MC_INDEX, ~C_000078_MC_IND_ADDR); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); return r; } void rs690_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v) { + unsigned long flags; + + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(R_000078_MC_INDEX, S_000078_MC_IND_ADDR(reg) | S_000078_MC_IND_WR_EN(1)); WREG32(R_00007C_MC_DATA, v); WREG32(R_000078_MC_INDEX, 0x7F); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); } static void rs690_mc_program(struct radeon_device *rdev) @@ -655,7 +740,7 @@ static int rs690_startup(struct radeon_device *rdev) return r; } - r = r600_audio_init(rdev); + r = radeon_audio_init(rdev); if (r) { dev_err(rdev->dev, "failed initializing audio\n"); return r; @@ -695,7 +780,8 @@ int rs690_resume(struct radeon_device *rdev) int rs690_suspend(struct radeon_device *rdev) { - r600_audio_fini(rdev); + radeon_pm_suspend(rdev); + radeon_audio_fini(rdev); r100_cp_disable(rdev); radeon_wb_disable(rdev); rs600_irq_disable(rdev); @@ -705,7 +791,8 @@ int rs690_suspend(struct radeon_device *rdev) void rs690_fini(struct radeon_device *rdev) { - r600_audio_fini(rdev); + radeon_pm_fini(rdev); + radeon_audio_fini(rdev); r100_cp_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); @@ -774,6 +861,9 @@ int rs690_init(struct radeon_device *rdev) return r; rs600_set_safe_registers(rdev); + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->accel_working = true; r = rs690_startup(rdev); if (r) { diff --git a/sys/dev/pci/drm/radeon/rs690d.h b/sys/dev/pci/drm/radeon/rs690d.h index 5b2b38efd4a..8af3ccf20cc 100644 --- a/sys/dev/pci/drm/radeon/rs690d.h +++ b/sys/dev/pci/drm/radeon/rs690d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: rs690d.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -30,6 +29,9 @@ #define __RS690D_H__ /* Registers */ +#define R_00001E_K8_FB_LOCATION 0x00001E +#define R_00005F_MC_MISC_UMA_CNTL 0x00005F +#define G_00005F_K8_ADDR_EXT(x) (((x) >> 0) & 0xFF) #define R_000078_MC_INDEX 0x000078 #define S_000078_MC_IND_ADDR(x) (((x) & 0x1FF) << 0) #define G_000078_MC_IND_ADDR(x) (((x) >> 0) & 0x1FF) diff --git a/sys/dev/pci/drm/radeon/rs780_dpm.c b/sys/dev/pci/drm/radeon/rs780_dpm.c new file mode 100644 index 00000000000..c2a6ddf5434 --- /dev/null +++ b/sys/dev/pci/drm/radeon/rs780_dpm.c @@ -0,0 +1,1075 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "rs780d.h" +#include "r600_dpm.h" +#include "rs780_dpm.h" +#include "atom.h" + +static struct igp_ps *rs780_get_ps(struct radeon_ps *rps) +{ + struct igp_ps *ps = rps->ps_priv; + + return ps; +} + +static struct igp_power_info *rs780_get_pi(struct radeon_device *rdev) +{ + struct igp_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +static void rs780_get_pm_mode_parameters(struct radeon_device *rdev) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + struct radeon_mode_info *minfo = &rdev->mode_info; + struct drm_crtc *crtc; + struct radeon_crtc *radeon_crtc; + int i; + + /* defaults */ + pi->crtc_id = 0; + pi->refresh_rate = 60; + + for (i = 0; i < rdev->num_crtc; i++) { + crtc = (struct drm_crtc *)minfo->crtcs[i]; + if (crtc && crtc->enabled) { + radeon_crtc = to_radeon_crtc(crtc); + pi->crtc_id = radeon_crtc->crtc_id; + if (crtc->mode.htotal && crtc->mode.vtotal) + pi->refresh_rate = drm_mode_vrefresh(&crtc->mode); + break; + } + } +} + +static void rs780_voltage_scaling_enable(struct radeon_device *rdev, bool enable); + +static int rs780_initialize_dpm_power_state(struct radeon_device *rdev, + struct radeon_ps *boot_ps) +{ + struct atom_clock_dividers dividers; + struct igp_ps *default_state = rs780_get_ps(boot_ps); + int i, ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + default_state->sclk_low, false, ÷rs); + if (ret) + return ret; + + r600_engine_clock_entry_set_reference_divider(rdev, 0, dividers.ref_div); + r600_engine_clock_entry_set_feedback_divider(rdev, 0, dividers.fb_div); + r600_engine_clock_entry_set_post_divider(rdev, 0, dividers.post_div); + + if (dividers.enable_post_div) + r600_engine_clock_entry_enable_post_divider(rdev, 0, true); + else + r600_engine_clock_entry_enable_post_divider(rdev, 0, false); + + r600_engine_clock_entry_set_step_time(rdev, 0, R600_SST_DFLT); + r600_engine_clock_entry_enable_pulse_skipping(rdev, 0, false); + + r600_engine_clock_entry_enable(rdev, 0, true); + for (i = 1; i < R600_PM_NUMBER_OF_SCLKS; i++) + r600_engine_clock_entry_enable(rdev, i, false); + + r600_enable_mclk_control(rdev, false); + r600_voltage_control_enable_pins(rdev, 0); + + return 0; +} + +static int rs780_initialize_dpm_parameters(struct radeon_device *rdev, + struct radeon_ps *boot_ps) +{ + int ret = 0; + int i; + + r600_set_bsp(rdev, R600_BSU_DFLT, R600_BSP_DFLT); + + r600_set_at(rdev, 0, 0, 0, 0); + + r600_set_git(rdev, R600_GICST_DFLT); + + for (i = 0; i < R600_PM_NUMBER_OF_TC; i++) + r600_set_tc(rdev, i, 0, 0); + + r600_select_td(rdev, R600_TD_DFLT); + r600_set_vrc(rdev, 0); + + r600_set_tpu(rdev, R600_TPU_DFLT); + r600_set_tpc(rdev, R600_TPC_DFLT); + + r600_set_sstu(rdev, R600_SSTU_DFLT); + r600_set_sst(rdev, R600_SST_DFLT); + + r600_set_fctu(rdev, R600_FCTU_DFLT); + r600_set_fct(rdev, R600_FCT_DFLT); + + r600_set_vddc3d_oorsu(rdev, R600_VDDC3DOORSU_DFLT); + r600_set_vddc3d_oorphc(rdev, R600_VDDC3DOORPHC_DFLT); + r600_set_vddc3d_oorsdc(rdev, R600_VDDC3DOORSDC_DFLT); + r600_set_ctxcgtt3d_rphc(rdev, R600_CTXCGTT3DRPHC_DFLT); + r600_set_ctxcgtt3d_rsdc(rdev, R600_CTXCGTT3DRSDC_DFLT); + + r600_vid_rt_set_vru(rdev, R600_VRU_DFLT); + r600_vid_rt_set_vrt(rdev, R600_VOLTAGERESPONSETIME_DFLT); + r600_vid_rt_set_ssu(rdev, R600_SPLLSTEPUNIT_DFLT); + + ret = rs780_initialize_dpm_power_state(rdev, boot_ps); + + r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW, 0); + r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM, 0); + r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_HIGH, 0); + + r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW, 0); + r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM, 0); + r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_HIGH, 0); + + r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW, 0); + r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM, 0); + r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_HIGH, 0); + + r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW, R600_DISPLAY_WATERMARK_HIGH); + r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM, R600_DISPLAY_WATERMARK_HIGH); + r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_HIGH, R600_DISPLAY_WATERMARK_HIGH); + + r600_power_level_enable(rdev, R600_POWER_LEVEL_CTXSW, false); + r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false); + r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false); + r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true); + + r600_power_level_set_enter_index(rdev, R600_POWER_LEVEL_LOW); + + r600_set_vrc(rdev, RS780_CGFTV_DFLT); + + return ret; +} + +static void rs780_start_dpm(struct radeon_device *rdev) +{ + r600_enable_sclk_control(rdev, false); + r600_enable_mclk_control(rdev, false); + + r600_dynamicpm_enable(rdev, true); + + radeon_wait_for_vblank(rdev, 0); + radeon_wait_for_vblank(rdev, 1); + + r600_enable_spll_bypass(rdev, true); + r600_wait_for_spll_change(rdev); + r600_enable_spll_bypass(rdev, false); + r600_wait_for_spll_change(rdev); + + r600_enable_spll_bypass(rdev, true); + r600_wait_for_spll_change(rdev); + r600_enable_spll_bypass(rdev, false); + r600_wait_for_spll_change(rdev); + + r600_enable_sclk_control(rdev, true); +} + + +static void rs780_preset_ranges_slow_clk_fbdiv_en(struct radeon_device *rdev) +{ + WREG32_P(FVTHROT_SLOW_CLK_FEEDBACK_DIV_REG1, RANGE_SLOW_CLK_FEEDBACK_DIV_EN, + ~RANGE_SLOW_CLK_FEEDBACK_DIV_EN); + + WREG32_P(FVTHROT_SLOW_CLK_FEEDBACK_DIV_REG1, + RANGE0_SLOW_CLK_FEEDBACK_DIV(RS780_SLOWCLKFEEDBACKDIV_DFLT), + ~RANGE0_SLOW_CLK_FEEDBACK_DIV_MASK); +} + +static void rs780_preset_starting_fbdiv(struct radeon_device *rdev) +{ + u32 fbdiv = (RREG32(CG_SPLL_FUNC_CNTL) & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT; + + WREG32_P(FVTHROT_FBDIV_REG1, STARTING_FEEDBACK_DIV(fbdiv), + ~STARTING_FEEDBACK_DIV_MASK); + + WREG32_P(FVTHROT_FBDIV_REG2, FORCED_FEEDBACK_DIV(fbdiv), + ~FORCED_FEEDBACK_DIV_MASK); + + WREG32_P(FVTHROT_FBDIV_REG1, FORCE_FEEDBACK_DIV, ~FORCE_FEEDBACK_DIV); +} + +static void rs780_voltage_scaling_init(struct radeon_device *rdev) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + struct drm_device *dev = rdev->ddev; + u32 fv_throt_pwm_fb_div_range[3]; + u32 fv_throt_pwm_range[4]; + + if (dev->pdev->device == 0x9614) { + fv_throt_pwm_fb_div_range[0] = RS780D_FVTHROTPWMFBDIVRANGEREG0_DFLT; + fv_throt_pwm_fb_div_range[1] = RS780D_FVTHROTPWMFBDIVRANGEREG1_DFLT; + fv_throt_pwm_fb_div_range[2] = RS780D_FVTHROTPWMFBDIVRANGEREG2_DFLT; + } else if ((dev->pdev->device == 0x9714) || + (dev->pdev->device == 0x9715)) { + fv_throt_pwm_fb_div_range[0] = RS880D_FVTHROTPWMFBDIVRANGEREG0_DFLT; + fv_throt_pwm_fb_div_range[1] = RS880D_FVTHROTPWMFBDIVRANGEREG1_DFLT; + fv_throt_pwm_fb_div_range[2] = RS880D_FVTHROTPWMFBDIVRANGEREG2_DFLT; + } else { + fv_throt_pwm_fb_div_range[0] = RS780_FVTHROTPWMFBDIVRANGEREG0_DFLT; + fv_throt_pwm_fb_div_range[1] = RS780_FVTHROTPWMFBDIVRANGEREG1_DFLT; + fv_throt_pwm_fb_div_range[2] = RS780_FVTHROTPWMFBDIVRANGEREG2_DFLT; + } + + if (pi->pwm_voltage_control) { + fv_throt_pwm_range[0] = pi->min_voltage; + fv_throt_pwm_range[1] = pi->min_voltage; + fv_throt_pwm_range[2] = pi->max_voltage; + fv_throt_pwm_range[3] = pi->max_voltage; + } else { + fv_throt_pwm_range[0] = pi->invert_pwm_required ? + RS780_FVTHROTPWMRANGE3_GPIO_DFLT : RS780_FVTHROTPWMRANGE0_GPIO_DFLT; + fv_throt_pwm_range[1] = pi->invert_pwm_required ? + RS780_FVTHROTPWMRANGE2_GPIO_DFLT : RS780_FVTHROTPWMRANGE1_GPIO_DFLT; + fv_throt_pwm_range[2] = pi->invert_pwm_required ? + RS780_FVTHROTPWMRANGE1_GPIO_DFLT : RS780_FVTHROTPWMRANGE2_GPIO_DFLT; + fv_throt_pwm_range[3] = pi->invert_pwm_required ? + RS780_FVTHROTPWMRANGE0_GPIO_DFLT : RS780_FVTHROTPWMRANGE3_GPIO_DFLT; + } + + WREG32_P(FVTHROT_PWM_CTRL_REG0, + STARTING_PWM_HIGHTIME(pi->max_voltage), + ~STARTING_PWM_HIGHTIME_MASK); + + WREG32_P(FVTHROT_PWM_CTRL_REG0, + NUMBER_OF_CYCLES_IN_PERIOD(pi->num_of_cycles_in_period), + ~NUMBER_OF_CYCLES_IN_PERIOD_MASK); + + WREG32_P(FVTHROT_PWM_CTRL_REG0, FORCE_STARTING_PWM_HIGHTIME, + ~FORCE_STARTING_PWM_HIGHTIME); + + if (pi->invert_pwm_required) + WREG32_P(FVTHROT_PWM_CTRL_REG0, INVERT_PWM_WAVEFORM, ~INVERT_PWM_WAVEFORM); + else + WREG32_P(FVTHROT_PWM_CTRL_REG0, 0, ~INVERT_PWM_WAVEFORM); + + rs780_voltage_scaling_enable(rdev, true); + + WREG32(FVTHROT_PWM_CTRL_REG1, + (MIN_PWM_HIGHTIME(pi->min_voltage) | + MAX_PWM_HIGHTIME(pi->max_voltage))); + + WREG32(FVTHROT_PWM_US_REG0, RS780_FVTHROTPWMUSREG0_DFLT); + WREG32(FVTHROT_PWM_US_REG1, RS780_FVTHROTPWMUSREG1_DFLT); + WREG32(FVTHROT_PWM_DS_REG0, RS780_FVTHROTPWMDSREG0_DFLT); + WREG32(FVTHROT_PWM_DS_REG1, RS780_FVTHROTPWMDSREG1_DFLT); + + WREG32_P(FVTHROT_PWM_FEEDBACK_DIV_REG1, + RANGE0_PWM_FEEDBACK_DIV(fv_throt_pwm_fb_div_range[0]), + ~RANGE0_PWM_FEEDBACK_DIV_MASK); + + WREG32(FVTHROT_PWM_FEEDBACK_DIV_REG2, + (RANGE1_PWM_FEEDBACK_DIV(fv_throt_pwm_fb_div_range[1]) | + RANGE2_PWM_FEEDBACK_DIV(fv_throt_pwm_fb_div_range[2]))); + + WREG32(FVTHROT_PWM_FEEDBACK_DIV_REG3, + (RANGE0_PWM(fv_throt_pwm_range[1]) | + RANGE1_PWM(fv_throt_pwm_range[2]))); + WREG32(FVTHROT_PWM_FEEDBACK_DIV_REG4, + (RANGE2_PWM(fv_throt_pwm_range[1]) | + RANGE3_PWM(fv_throt_pwm_range[2]))); +} + +static void rs780_clk_scaling_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(FVTHROT_CNTRL_REG, ENABLE_FV_THROT | ENABLE_FV_UPDATE, + ~(ENABLE_FV_THROT | ENABLE_FV_UPDATE)); + else + WREG32_P(FVTHROT_CNTRL_REG, 0, + ~(ENABLE_FV_THROT | ENABLE_FV_UPDATE)); +} + +static void rs780_voltage_scaling_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(FVTHROT_CNTRL_REG, ENABLE_FV_THROT_IO, ~ENABLE_FV_THROT_IO); + else + WREG32_P(FVTHROT_CNTRL_REG, 0, ~ENABLE_FV_THROT_IO); +} + +static void rs780_set_engine_clock_wfc(struct radeon_device *rdev) +{ + WREG32(FVTHROT_UTC0, RS780_FVTHROTUTC0_DFLT); + WREG32(FVTHROT_UTC1, RS780_FVTHROTUTC1_DFLT); + WREG32(FVTHROT_UTC2, RS780_FVTHROTUTC2_DFLT); + WREG32(FVTHROT_UTC3, RS780_FVTHROTUTC3_DFLT); + WREG32(FVTHROT_UTC4, RS780_FVTHROTUTC4_DFLT); + + WREG32(FVTHROT_DTC0, RS780_FVTHROTDTC0_DFLT); + WREG32(FVTHROT_DTC1, RS780_FVTHROTDTC1_DFLT); + WREG32(FVTHROT_DTC2, RS780_FVTHROTDTC2_DFLT); + WREG32(FVTHROT_DTC3, RS780_FVTHROTDTC3_DFLT); + WREG32(FVTHROT_DTC4, RS780_FVTHROTDTC4_DFLT); +} + +static void rs780_set_engine_clock_sc(struct radeon_device *rdev) +{ + WREG32_P(FVTHROT_FBDIV_REG2, + FB_DIV_TIMER_VAL(RS780_FBDIVTIMERVAL_DFLT), + ~FB_DIV_TIMER_VAL_MASK); + + WREG32_P(FVTHROT_CNTRL_REG, + REFRESH_RATE_DIVISOR(0) | MINIMUM_CIP(0xf), + ~(REFRESH_RATE_DIVISOR_MASK | MINIMUM_CIP_MASK)); +} + +static void rs780_set_engine_clock_tdc(struct radeon_device *rdev) +{ + WREG32_P(FVTHROT_CNTRL_REG, 0, ~(FORCE_TREND_SEL | TREND_SEL_MODE)); +} + +static void rs780_set_engine_clock_ssc(struct radeon_device *rdev) +{ + WREG32(FVTHROT_FB_US_REG0, RS780_FVTHROTFBUSREG0_DFLT); + WREG32(FVTHROT_FB_US_REG1, RS780_FVTHROTFBUSREG1_DFLT); + WREG32(FVTHROT_FB_DS_REG0, RS780_FVTHROTFBDSREG0_DFLT); + WREG32(FVTHROT_FB_DS_REG1, RS780_FVTHROTFBDSREG1_DFLT); + + WREG32_P(FVTHROT_FBDIV_REG1, MAX_FEEDBACK_STEP(1), ~MAX_FEEDBACK_STEP_MASK); +} + +static void rs780_program_at(struct radeon_device *rdev) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + + WREG32(FVTHROT_TARGET_REG, 30000000 / pi->refresh_rate); + WREG32(FVTHROT_CB1, 1000000 * 5 / pi->refresh_rate); + WREG32(FVTHROT_CB2, 1000000 * 10 / pi->refresh_rate); + WREG32(FVTHROT_CB3, 1000000 * 30 / pi->refresh_rate); + WREG32(FVTHROT_CB4, 1000000 * 50 / pi->refresh_rate); +} + +static void rs780_disable_vbios_powersaving(struct radeon_device *rdev) +{ + WREG32_P(CG_INTGFX_MISC, 0, ~0xFFF00000); +} + +static void rs780_force_voltage(struct radeon_device *rdev, u16 voltage) +{ + struct igp_ps *current_state = rs780_get_ps(rdev->pm.dpm.current_ps); + + if ((current_state->max_voltage == RS780_VDDC_LEVEL_HIGH) && + (current_state->min_voltage == RS780_VDDC_LEVEL_HIGH)) + return; + + WREG32_P(GFX_MACRO_BYPASS_CNTL, SPLL_BYPASS_CNTL, ~SPLL_BYPASS_CNTL); + + udelay(1); + + WREG32_P(FVTHROT_PWM_CTRL_REG0, + STARTING_PWM_HIGHTIME(voltage), + ~STARTING_PWM_HIGHTIME_MASK); + + WREG32_P(FVTHROT_PWM_CTRL_REG0, + FORCE_STARTING_PWM_HIGHTIME, ~FORCE_STARTING_PWM_HIGHTIME); + + WREG32_P(FVTHROT_PWM_FEEDBACK_DIV_REG1, 0, + ~RANGE_PWM_FEEDBACK_DIV_EN); + + udelay(1); + + WREG32_P(GFX_MACRO_BYPASS_CNTL, 0, ~SPLL_BYPASS_CNTL); +} + +static void rs780_force_fbdiv(struct radeon_device *rdev, u32 fb_div) +{ + struct igp_ps *current_state = rs780_get_ps(rdev->pm.dpm.current_ps); + + if (current_state->sclk_low == current_state->sclk_high) + return; + + WREG32_P(GFX_MACRO_BYPASS_CNTL, SPLL_BYPASS_CNTL, ~SPLL_BYPASS_CNTL); + + WREG32_P(FVTHROT_FBDIV_REG2, FORCED_FEEDBACK_DIV(fb_div), + ~FORCED_FEEDBACK_DIV_MASK); + WREG32_P(FVTHROT_FBDIV_REG1, STARTING_FEEDBACK_DIV(fb_div), + ~STARTING_FEEDBACK_DIV_MASK); + WREG32_P(FVTHROT_FBDIV_REG1, FORCE_FEEDBACK_DIV, ~FORCE_FEEDBACK_DIV); + + udelay(100); + + WREG32_P(GFX_MACRO_BYPASS_CNTL, 0, ~SPLL_BYPASS_CNTL); +} + +static int rs780_set_engine_clock_scaling(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct atom_clock_dividers min_dividers, max_dividers, current_max_dividers; + struct igp_ps *new_state = rs780_get_ps(new_ps); + struct igp_ps *old_state = rs780_get_ps(old_ps); + int ret; + + if ((new_state->sclk_high == old_state->sclk_high) && + (new_state->sclk_low == old_state->sclk_low)) + return 0; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + new_state->sclk_low, false, &min_dividers); + if (ret) + return ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + new_state->sclk_high, false, &max_dividers); + if (ret) + return ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + old_state->sclk_high, false, ¤t_max_dividers); + if (ret) + return ret; + + if ((min_dividers.ref_div != max_dividers.ref_div) || + (min_dividers.post_div != max_dividers.post_div) || + (max_dividers.ref_div != current_max_dividers.ref_div) || + (max_dividers.post_div != current_max_dividers.post_div)) + return -EINVAL; + + rs780_force_fbdiv(rdev, max_dividers.fb_div); + + if (max_dividers.fb_div > min_dividers.fb_div) { + WREG32_P(FVTHROT_FBDIV_REG0, + MIN_FEEDBACK_DIV(min_dividers.fb_div) | + MAX_FEEDBACK_DIV(max_dividers.fb_div), + ~(MIN_FEEDBACK_DIV_MASK | MAX_FEEDBACK_DIV_MASK)); + + WREG32_P(FVTHROT_FBDIV_REG1, 0, ~FORCE_FEEDBACK_DIV); + } + + return 0; +} + +static void rs780_set_engine_clock_spc(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct igp_ps *new_state = rs780_get_ps(new_ps); + struct igp_ps *old_state = rs780_get_ps(old_ps); + struct igp_power_info *pi = rs780_get_pi(rdev); + + if ((new_state->sclk_high == old_state->sclk_high) && + (new_state->sclk_low == old_state->sclk_low)) + return; + + if (pi->crtc_id == 0) + WREG32_P(CG_INTGFX_MISC, 0, ~FVTHROT_VBLANK_SEL); + else + WREG32_P(CG_INTGFX_MISC, FVTHROT_VBLANK_SEL, ~FVTHROT_VBLANK_SEL); + +} + +static void rs780_activate_engine_clk_scaling(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct igp_ps *new_state = rs780_get_ps(new_ps); + struct igp_ps *old_state = rs780_get_ps(old_ps); + + if ((new_state->sclk_high == old_state->sclk_high) && + (new_state->sclk_low == old_state->sclk_low)) + return; + + if (new_state->sclk_high == new_state->sclk_low) + return; + + rs780_clk_scaling_enable(rdev, true); +} + +static u32 rs780_get_voltage_for_vddc_level(struct radeon_device *rdev, + enum rs780_vddc_level vddc) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + + if (vddc == RS780_VDDC_LEVEL_HIGH) + return pi->max_voltage; + else if (vddc == RS780_VDDC_LEVEL_LOW) + return pi->min_voltage; + else + return pi->max_voltage; +} + +static void rs780_enable_voltage_scaling(struct radeon_device *rdev, + struct radeon_ps *new_ps) +{ + struct igp_ps *new_state = rs780_get_ps(new_ps); + struct igp_power_info *pi = rs780_get_pi(rdev); + enum rs780_vddc_level vddc_high, vddc_low; + + udelay(100); + + if ((new_state->max_voltage == RS780_VDDC_LEVEL_HIGH) && + (new_state->min_voltage == RS780_VDDC_LEVEL_HIGH)) + return; + + vddc_high = rs780_get_voltage_for_vddc_level(rdev, + new_state->max_voltage); + vddc_low = rs780_get_voltage_for_vddc_level(rdev, + new_state->min_voltage); + + WREG32_P(GFX_MACRO_BYPASS_CNTL, SPLL_BYPASS_CNTL, ~SPLL_BYPASS_CNTL); + + udelay(1); + if (vddc_high > vddc_low) { + WREG32_P(FVTHROT_PWM_FEEDBACK_DIV_REG1, + RANGE_PWM_FEEDBACK_DIV_EN, ~RANGE_PWM_FEEDBACK_DIV_EN); + + WREG32_P(FVTHROT_PWM_CTRL_REG0, 0, ~FORCE_STARTING_PWM_HIGHTIME); + } else if (vddc_high == vddc_low) { + if (pi->max_voltage != vddc_high) { + WREG32_P(FVTHROT_PWM_CTRL_REG0, + STARTING_PWM_HIGHTIME(vddc_high), + ~STARTING_PWM_HIGHTIME_MASK); + + WREG32_P(FVTHROT_PWM_CTRL_REG0, + FORCE_STARTING_PWM_HIGHTIME, + ~FORCE_STARTING_PWM_HIGHTIME); + } + } + + WREG32_P(GFX_MACRO_BYPASS_CNTL, 0, ~SPLL_BYPASS_CNTL); +} + +static void rs780_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct igp_ps *new_state = rs780_get_ps(new_ps); + struct igp_ps *current_state = rs780_get_ps(old_ps); + + if ((new_ps->vclk == old_ps->vclk) && + (new_ps->dclk == old_ps->dclk)) + return; + + if (new_state->sclk_high >= current_state->sclk_high) + return; + + radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk); +} + +static void rs780_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct igp_ps *new_state = rs780_get_ps(new_ps); + struct igp_ps *current_state = rs780_get_ps(old_ps); + + if ((new_ps->vclk == old_ps->vclk) && + (new_ps->dclk == old_ps->dclk)) + return; + + if (new_state->sclk_high < current_state->sclk_high) + return; + + radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk); +} + +int rs780_dpm_enable(struct radeon_device *rdev) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + int ret; + + rs780_get_pm_mode_parameters(rdev); + rs780_disable_vbios_powersaving(rdev); + + if (r600_dynamicpm_enabled(rdev)) + return -EINVAL; + ret = rs780_initialize_dpm_parameters(rdev, boot_ps); + if (ret) + return ret; + rs780_start_dpm(rdev); + + rs780_preset_ranges_slow_clk_fbdiv_en(rdev); + rs780_preset_starting_fbdiv(rdev); + if (pi->voltage_control) + rs780_voltage_scaling_init(rdev); + rs780_clk_scaling_enable(rdev, true); + rs780_set_engine_clock_sc(rdev); + rs780_set_engine_clock_wfc(rdev); + rs780_program_at(rdev); + rs780_set_engine_clock_tdc(rdev); + rs780_set_engine_clock_ssc(rdev); + + if (pi->gfx_clock_gating) + r600_gfx_clockgating_enable(rdev, true); + + return 0; +} + +void rs780_dpm_disable(struct radeon_device *rdev) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + + r600_dynamicpm_enable(rdev, false); + + rs780_clk_scaling_enable(rdev, false); + rs780_voltage_scaling_enable(rdev, false); + + if (pi->gfx_clock_gating) + r600_gfx_clockgating_enable(rdev, false); + + if (rdev->irq.installed && + (rdev->pm.int_thermal_type == THERMAL_TYPE_RV6XX)) { + rdev->irq.dpm_thermal = false; + radeon_irq_set(rdev); + } +} + +int rs780_dpm_set_power_state(struct radeon_device *rdev) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps; + struct radeon_ps *old_ps = rdev->pm.dpm.current_ps; + int ret; + + rs780_get_pm_mode_parameters(rdev); + + rs780_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps); + + if (pi->voltage_control) { + rs780_force_voltage(rdev, pi->max_voltage); + mdelay(5); + } + + ret = rs780_set_engine_clock_scaling(rdev, new_ps, old_ps); + if (ret) + return ret; + rs780_set_engine_clock_spc(rdev, new_ps, old_ps); + + rs780_activate_engine_clk_scaling(rdev, new_ps, old_ps); + + if (pi->voltage_control) + rs780_enable_voltage_scaling(rdev, new_ps); + + rs780_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps); + + return 0; +} + +void rs780_dpm_setup_asic(struct radeon_device *rdev) +{ + +} + +void rs780_dpm_display_configuration_changed(struct radeon_device *rdev) +{ + rs780_get_pm_mode_parameters(rdev); + rs780_program_at(rdev); +} + +union igp_info { + struct _ATOM_INTEGRATED_SYSTEM_INFO info; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2; +}; + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; +}; + +union pplib_clock_info { + struct _ATOM_PPLIB_R600_CLOCK_INFO r600; + struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; + struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; + struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; +}; + +union pplib_power_state { + struct _ATOM_PPLIB_STATE v1; + struct _ATOM_PPLIB_STATE_V2 v2; +}; + +static void rs780_parse_pplib_non_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info, + u8 table_rev) +{ + rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); + rps->class = le16_to_cpu(non_clock_info->usClassification); + rps->class2 = le16_to_cpu(non_clock_info->usClassification2); + + if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) { + rps->vclk = le32_to_cpu(non_clock_info->ulVCLK); + rps->dclk = le32_to_cpu(non_clock_info->ulDCLK); + } else { + rps->vclk = 0; + rps->dclk = 0; + } + + if (r600_is_uvd_state(rps->class, rps->class2)) { + if ((rps->vclk == 0) || (rps->dclk == 0)) { + rps->vclk = RS780_DEFAULT_VCLK_FREQ; + rps->dclk = RS780_DEFAULT_DCLK_FREQ; + } + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) + rdev->pm.dpm.boot_ps = rps; + if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + rdev->pm.dpm.uvd_ps = rps; +} + +static void rs780_parse_pplib_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + union pplib_clock_info *clock_info) +{ + struct igp_ps *ps = rs780_get_ps(rps); + u32 sclk; + + sclk = le16_to_cpu(clock_info->rs780.usLowEngineClockLow); + sclk |= clock_info->rs780.ucLowEngineClockHigh << 16; + ps->sclk_low = sclk; + sclk = le16_to_cpu(clock_info->rs780.usHighEngineClockLow); + sclk |= clock_info->rs780.ucHighEngineClockHigh << 16; + ps->sclk_high = sclk; + switch (le16_to_cpu(clock_info->rs780.usVDDC)) { + case ATOM_PPLIB_RS780_VOLTAGE_NONE: + default: + ps->min_voltage = RS780_VDDC_LEVEL_UNKNOWN; + ps->max_voltage = RS780_VDDC_LEVEL_UNKNOWN; + break; + case ATOM_PPLIB_RS780_VOLTAGE_LOW: + ps->min_voltage = RS780_VDDC_LEVEL_LOW; + ps->max_voltage = RS780_VDDC_LEVEL_LOW; + break; + case ATOM_PPLIB_RS780_VOLTAGE_HIGH: + ps->min_voltage = RS780_VDDC_LEVEL_HIGH; + ps->max_voltage = RS780_VDDC_LEVEL_HIGH; + break; + case ATOM_PPLIB_RS780_VOLTAGE_VARIABLE: + ps->min_voltage = RS780_VDDC_LEVEL_LOW; + ps->max_voltage = RS780_VDDC_LEVEL_HIGH; + break; + } + ps->flags = le32_to_cpu(clock_info->rs780.ulFlags); + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { + ps->sclk_low = rdev->clock.default_sclk; + ps->sclk_high = rdev->clock.default_sclk; + ps->min_voltage = RS780_VDDC_LEVEL_HIGH; + ps->max_voltage = RS780_VDDC_LEVEL_HIGH; + } +} + +static int rs780_parse_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; + union pplib_power_state *power_state; + int i; + union pplib_clock_info *clock_info; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + struct igp_ps *ps; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) * + power_info->pplib.ucNumStates, GFP_KERNEL); + if (!rdev->pm.dpm.ps) + return -ENOMEM; + + for (i = 0; i < power_info->pplib.ucNumStates; i++) { + power_state = (union pplib_power_state *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usStateArrayOffset) + + i * power_info->pplib.ucStateEntrySize); + non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) + + (power_state->v1.ucNonClockStateIndex * + power_info->pplib.ucNonClockSize)); + if (power_info->pplib.ucStateEntrySize - 1) { + clock_info = (union pplib_clock_info *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) + + (power_state->v1.ucClockStateIndices[0] * + power_info->pplib.ucClockInfoSize)); + ps = kzalloc(sizeof(struct igp_ps), GFP_KERNEL); + if (ps == NULL) { + kfree(rdev->pm.dpm.ps); + return -ENOMEM; + } + rdev->pm.dpm.ps[i].ps_priv = ps; + rs780_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i], + non_clock_info, + power_info->pplib.ucNonClockSize); + rs780_parse_pplib_clock_info(rdev, + &rdev->pm.dpm.ps[i], + clock_info); + } + } + rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates; + return 0; +} + +int rs780_dpm_init(struct radeon_device *rdev) +{ + struct igp_power_info *pi; + int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo); + union igp_info *info; + u16 data_offset; + u8 frev, crev; + int ret; + + pi = kzalloc(sizeof(struct igp_power_info), GFP_KERNEL); + if (pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = pi; + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = rs780_parse_power_table(rdev); + if (ret) + return ret; + + pi->voltage_control = false; + pi->gfx_clock_gating = true; + + if (atom_parse_data_header(rdev->mode_info.atom_context, index, NULL, + &frev, &crev, &data_offset)) { + info = (union igp_info *)(rdev->mode_info.atom_context->bios + data_offset); + + /* Get various system informations from bios */ + switch (crev) { + case 1: + pi->num_of_cycles_in_period = + info->info.ucNumberOfCyclesInPeriod; + pi->num_of_cycles_in_period |= + info->info.ucNumberOfCyclesInPeriodHi << 8; + pi->invert_pwm_required = + (pi->num_of_cycles_in_period & 0x8000) ? true : false; + pi->boot_voltage = info->info.ucStartingPWM_HighTime; + pi->max_voltage = info->info.ucMaxNBVoltage; + pi->max_voltage |= info->info.ucMaxNBVoltageHigh << 8; + pi->min_voltage = info->info.ucMinNBVoltage; + pi->min_voltage |= info->info.ucMinNBVoltageHigh << 8; + pi->inter_voltage_low = + le16_to_cpu(info->info.usInterNBVoltageLow); + pi->inter_voltage_high = + le16_to_cpu(info->info.usInterNBVoltageHigh); + pi->voltage_control = true; + pi->bootup_uma_clk = info->info.usK8MemoryClock * 100; + break; + case 2: + pi->num_of_cycles_in_period = + le16_to_cpu(info->info_2.usNumberOfCyclesInPeriod); + pi->invert_pwm_required = + (pi->num_of_cycles_in_period & 0x8000) ? true : false; + pi->boot_voltage = + le16_to_cpu(info->info_2.usBootUpNBVoltage); + pi->max_voltage = + le16_to_cpu(info->info_2.usMaxNBVoltage); + pi->min_voltage = + le16_to_cpu(info->info_2.usMinNBVoltage); + pi->system_config = + le32_to_cpu(info->info_2.ulSystemConfig); + pi->pwm_voltage_control = + (pi->system_config & 0x4) ? true : false; + pi->voltage_control = true; + pi->bootup_uma_clk = le32_to_cpu(info->info_2.ulBootUpUMAClock); + break; + default: + DRM_ERROR("No integrated system info for your GPU\n"); + return -EINVAL; + } + if (pi->min_voltage > pi->max_voltage) + pi->voltage_control = false; + if (pi->pwm_voltage_control) { + if ((pi->num_of_cycles_in_period == 0) || + (pi->max_voltage == 0) || + (pi->min_voltage == 0)) + pi->voltage_control = false; + } else { + if ((pi->num_of_cycles_in_period == 0) || + (pi->max_voltage == 0)) + pi->voltage_control = false; + } + + return 0; + } + radeon_dpm_fini(rdev); + return -EINVAL; +} + +void rs780_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct igp_ps *ps = rs780_get_ps(rps); + + r600_dpm_print_class_info(rps->class, rps->class2); + r600_dpm_print_cap_info(rps->caps); + printk("\tuvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + printk("\t\tpower level 0 sclk: %u vddc_index: %d\n", + ps->sclk_low, ps->min_voltage); + printk("\t\tpower level 1 sclk: %u vddc_index: %d\n", + ps->sclk_high, ps->max_voltage); + r600_dpm_print_ps_status(rdev, rps); +} + +void rs780_dpm_fini(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); +} + +u32 rs780_dpm_get_sclk(struct radeon_device *rdev, bool low) +{ + struct igp_ps *requested_state = rs780_get_ps(rdev->pm.dpm.requested_ps); + + if (low) + return requested_state->sclk_low; + else + return requested_state->sclk_high; +} + +u32 rs780_dpm_get_mclk(struct radeon_device *rdev, bool low) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + + return pi->bootup_uma_clk; +} + +void rs780_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct radeon_ps *rps = rdev->pm.dpm.current_ps; + struct igp_ps *ps = rs780_get_ps(rps); + u32 current_fb_div = RREG32(FVTHROT_STATUS_REG0) & CURRENT_FEEDBACK_DIV_MASK; + u32 func_cntl = RREG32(CG_SPLL_FUNC_CNTL); + u32 ref_div = ((func_cntl & SPLL_REF_DIV_MASK) >> SPLL_REF_DIV_SHIFT) + 1; + u32 post_div = ((func_cntl & SPLL_SW_HILEN_MASK) >> SPLL_SW_HILEN_SHIFT) + 1 + + ((func_cntl & SPLL_SW_LOLEN_MASK) >> SPLL_SW_LOLEN_SHIFT) + 1; + u32 sclk = (rdev->clock.spll.reference_freq * current_fb_div) / + (post_div * ref_div); + + seq_printf(m, "uvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + + /* guess based on the current sclk */ + if (sclk < (ps->sclk_low + 500)) + seq_printf(m, "power level 0 sclk: %u vddc_index: %d\n", + ps->sclk_low, ps->min_voltage); + else + seq_printf(m, "power level 1 sclk: %u vddc_index: %d\n", + ps->sclk_high, ps->max_voltage); +} + +/* get the current sclk in 10 khz units */ +u32 rs780_dpm_get_current_sclk(struct radeon_device *rdev) +{ + u32 current_fb_div = RREG32(FVTHROT_STATUS_REG0) & CURRENT_FEEDBACK_DIV_MASK; + u32 func_cntl = RREG32(CG_SPLL_FUNC_CNTL); + u32 ref_div = ((func_cntl & SPLL_REF_DIV_MASK) >> SPLL_REF_DIV_SHIFT) + 1; + u32 post_div = ((func_cntl & SPLL_SW_HILEN_MASK) >> SPLL_SW_HILEN_SHIFT) + 1 + + ((func_cntl & SPLL_SW_LOLEN_MASK) >> SPLL_SW_LOLEN_SHIFT) + 1; + u32 sclk = (rdev->clock.spll.reference_freq * current_fb_div) / + (post_div * ref_div); + + return sclk; +} + +/* get the current mclk in 10 khz units */ +u32 rs780_dpm_get_current_mclk(struct radeon_device *rdev) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + + return pi->bootup_uma_clk; +} + +int rs780_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level) +{ + struct igp_power_info *pi = rs780_get_pi(rdev); + struct radeon_ps *rps = rdev->pm.dpm.current_ps; + struct igp_ps *ps = rs780_get_ps(rps); + struct atom_clock_dividers dividers; + int ret; + + rs780_clk_scaling_enable(rdev, false); + rs780_voltage_scaling_enable(rdev, false); + + if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + if (pi->voltage_control) + rs780_force_voltage(rdev, pi->max_voltage); + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + ps->sclk_high, false, ÷rs); + if (ret) + return ret; + + rs780_force_fbdiv(rdev, dividers.fb_div); + } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + ps->sclk_low, false, ÷rs); + if (ret) + return ret; + + rs780_force_fbdiv(rdev, dividers.fb_div); + + if (pi->voltage_control) + rs780_force_voltage(rdev, pi->min_voltage); + } else { + if (pi->voltage_control) + rs780_force_voltage(rdev, pi->max_voltage); + + if (ps->sclk_high != ps->sclk_low) { + WREG32_P(FVTHROT_FBDIV_REG1, 0, ~FORCE_FEEDBACK_DIV); + rs780_clk_scaling_enable(rdev, true); + } + + if (pi->voltage_control) { + rs780_voltage_scaling_enable(rdev, true); + rs780_enable_voltage_scaling(rdev, rps); + } + } + + rdev->pm.dpm.forced_level = level; + + return 0; +} diff --git a/sys/dev/pci/drm/radeon/rs780_dpm.h b/sys/dev/pci/drm/radeon/rs780_dpm.h new file mode 100644 index 00000000000..47a40b14fa4 --- /dev/null +++ b/sys/dev/pci/drm/radeon/rs780_dpm.h @@ -0,0 +1,109 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __RS780_DPM_H__ +#define __RS780_DPM_H__ + +enum rs780_vddc_level { + RS780_VDDC_LEVEL_UNKNOWN = 0, + RS780_VDDC_LEVEL_LOW = 1, + RS780_VDDC_LEVEL_HIGH = 2, +}; + +struct igp_power_info { + /* flags */ + bool invert_pwm_required; + bool pwm_voltage_control; + bool voltage_control; + bool gfx_clock_gating; + /* stored values */ + u32 system_config; + u32 bootup_uma_clk; + u16 max_voltage; + u16 min_voltage; + u16 boot_voltage; + u16 inter_voltage_low; + u16 inter_voltage_high; + u16 num_of_cycles_in_period; + /* variable */ + int crtc_id; + int refresh_rate; +}; + +struct igp_ps { + enum rs780_vddc_level min_voltage; + enum rs780_vddc_level max_voltage; + u32 sclk_low; + u32 sclk_high; + u32 flags; +}; + +#define RS780_CGFTV_DFLT 0x0303000f +#define RS780_FBDIVTIMERVAL_DFLT 0x2710 + +#define RS780_FVTHROTUTC0_DFLT 0x04010040 +#define RS780_FVTHROTUTC1_DFLT 0x04010040 +#define RS780_FVTHROTUTC2_DFLT 0x04010040 +#define RS780_FVTHROTUTC3_DFLT 0x04010040 +#define RS780_FVTHROTUTC4_DFLT 0x04010040 + +#define RS780_FVTHROTDTC0_DFLT 0x04010040 +#define RS780_FVTHROTDTC1_DFLT 0x04010040 +#define RS780_FVTHROTDTC2_DFLT 0x04010040 +#define RS780_FVTHROTDTC3_DFLT 0x04010040 +#define RS780_FVTHROTDTC4_DFLT 0x04010040 + +#define RS780_FVTHROTFBUSREG0_DFLT 0x00001001 +#define RS780_FVTHROTFBUSREG1_DFLT 0x00002002 +#define RS780_FVTHROTFBDSREG0_DFLT 0x00004001 +#define RS780_FVTHROTFBDSREG1_DFLT 0x00020010 + +#define RS780_FVTHROTPWMUSREG0_DFLT 0x00002001 +#define RS780_FVTHROTPWMUSREG1_DFLT 0x00004003 +#define RS780_FVTHROTPWMDSREG0_DFLT 0x00002001 +#define RS780_FVTHROTPWMDSREG1_DFLT 0x00004003 + +#define RS780_FVTHROTPWMFBDIVRANGEREG0_DFLT 0x37 +#define RS780_FVTHROTPWMFBDIVRANGEREG1_DFLT 0x4b +#define RS780_FVTHROTPWMFBDIVRANGEREG2_DFLT 0x8b + +#define RS780D_FVTHROTPWMFBDIVRANGEREG0_DFLT 0x8b +#define RS780D_FVTHROTPWMFBDIVRANGEREG1_DFLT 0x8c +#define RS780D_FVTHROTPWMFBDIVRANGEREG2_DFLT 0xb5 + +#define RS880D_FVTHROTPWMFBDIVRANGEREG0_DFLT 0x8d +#define RS880D_FVTHROTPWMFBDIVRANGEREG1_DFLT 0x8e +#define RS880D_FVTHROTPWMFBDIVRANGEREG2_DFLT 0xBa + +#define RS780_FVTHROTPWMRANGE0_GPIO_DFLT 0x1a +#define RS780_FVTHROTPWMRANGE1_GPIO_DFLT 0x1a +#define RS780_FVTHROTPWMRANGE2_GPIO_DFLT 0x0 +#define RS780_FVTHROTPWMRANGE3_GPIO_DFLT 0x0 + +#define RS780_SLOWCLKFEEDBACKDIV_DFLT 110 + +#define RS780_CGCLKGATING_DFLT 0x0000E204 + +#define RS780_DEFAULT_VCLK_FREQ 53300 /* 10 khz */ +#define RS780_DEFAULT_DCLK_FREQ 40000 /* 10 khz */ + +#endif diff --git a/sys/dev/pci/drm/radeon/rs780d.h b/sys/dev/pci/drm/radeon/rs780d.h new file mode 100644 index 00000000000..cfbe9a43d97 --- /dev/null +++ b/sys/dev/pci/drm/radeon/rs780d.h @@ -0,0 +1,171 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __RS780D_H__ +#define __RS780D_H__ + +#define CG_SPLL_FUNC_CNTL 0x600 +# define SPLL_RESET (1 << 0) +# define SPLL_SLEEP (1 << 1) +# define SPLL_REF_DIV(x) ((x) << 2) +# define SPLL_REF_DIV_MASK (7 << 2) +# define SPLL_REF_DIV_SHIFT 2 +# define SPLL_FB_DIV(x) ((x) << 5) +# define SPLL_FB_DIV_MASK (0xff << 2) +# define SPLL_FB_DIV_SHIFT 2 +# define SPLL_PULSEEN (1 << 13) +# define SPLL_PULSENUM(x) ((x) << 14) +# define SPLL_PULSENUM_MASK (3 << 14) +# define SPLL_SW_HILEN(x) ((x) << 16) +# define SPLL_SW_HILEN_MASK (0xf << 16) +# define SPLL_SW_HILEN_SHIFT 16 +# define SPLL_SW_LOLEN(x) ((x) << 20) +# define SPLL_SW_LOLEN_MASK (0xf << 20) +# define SPLL_SW_LOLEN_SHIFT 20 +# define SPLL_DIVEN (1 << 24) +# define SPLL_BYPASS_EN (1 << 25) +# define SPLL_CHG_STATUS (1 << 29) +# define SPLL_CTLREQ (1 << 30) +# define SPLL_CTLACK (1 << 31) + +/* RS780/RS880 PM */ +#define FVTHROT_CNTRL_REG 0x3000 +#define DONT_WAIT_FOR_FBDIV_WRAP (1 << 0) +#define MINIMUM_CIP(x) ((x) << 1) +#define MINIMUM_CIP_SHIFT 1 +#define MINIMUM_CIP_MASK 0x1fffffe +#define REFRESH_RATE_DIVISOR(x) ((x) << 25) +#define REFRESH_RATE_DIVISOR_SHIFT 25 +#define REFRESH_RATE_DIVISOR_MASK (0x3 << 25) +#define ENABLE_FV_THROT (1 << 27) +#define ENABLE_FV_UPDATE (1 << 28) +#define TREND_SEL_MODE (1 << 29) +#define FORCE_TREND_SEL (1 << 30) +#define ENABLE_FV_THROT_IO (1 << 31) +#define FVTHROT_TARGET_REG 0x3004 +#define TARGET_IDLE_COUNT(x) ((x) << 0) +#define TARGET_IDLE_COUNT_MASK 0xffffff +#define TARGET_IDLE_COUNT_SHIFT 0 +#define FVTHROT_CB1 0x3008 +#define FVTHROT_CB2 0x300c +#define FVTHROT_CB3 0x3010 +#define FVTHROT_CB4 0x3014 +#define FVTHROT_UTC0 0x3018 +#define FVTHROT_UTC1 0x301c +#define FVTHROT_UTC2 0x3020 +#define FVTHROT_UTC3 0x3024 +#define FVTHROT_UTC4 0x3028 +#define FVTHROT_DTC0 0x302c +#define FVTHROT_DTC1 0x3030 +#define FVTHROT_DTC2 0x3034 +#define FVTHROT_DTC3 0x3038 +#define FVTHROT_DTC4 0x303c +#define FVTHROT_FBDIV_REG0 0x3040 +#define MIN_FEEDBACK_DIV(x) ((x) << 0) +#define MIN_FEEDBACK_DIV_MASK 0xfff +#define MIN_FEEDBACK_DIV_SHIFT 0 +#define MAX_FEEDBACK_DIV(x) ((x) << 12) +#define MAX_FEEDBACK_DIV_MASK (0xfff << 12) +#define MAX_FEEDBACK_DIV_SHIFT 12 +#define FVTHROT_FBDIV_REG1 0x3044 +#define MAX_FEEDBACK_STEP(x) ((x) << 0) +#define MAX_FEEDBACK_STEP_MASK 0xfff +#define MAX_FEEDBACK_STEP_SHIFT 0 +#define STARTING_FEEDBACK_DIV(x) ((x) << 12) +#define STARTING_FEEDBACK_DIV_MASK (0xfff << 12) +#define STARTING_FEEDBACK_DIV_SHIFT 12 +#define FORCE_FEEDBACK_DIV (1 << 24) +#define FVTHROT_FBDIV_REG2 0x3048 +#define FORCED_FEEDBACK_DIV(x) ((x) << 0) +#define FORCED_FEEDBACK_DIV_MASK 0xfff +#define FORCED_FEEDBACK_DIV_SHIFT 0 +#define FB_DIV_TIMER_VAL(x) ((x) << 12) +#define FB_DIV_TIMER_VAL_MASK (0xffff << 12) +#define FB_DIV_TIMER_VAL_SHIFT 12 +#define FVTHROT_FB_US_REG0 0x304c +#define FVTHROT_FB_US_REG1 0x3050 +#define FVTHROT_FB_DS_REG0 0x3054 +#define FVTHROT_FB_DS_REG1 0x3058 +#define FVTHROT_PWM_CTRL_REG0 0x305c +#define STARTING_PWM_HIGHTIME(x) ((x) << 0) +#define STARTING_PWM_HIGHTIME_MASK 0xfff +#define STARTING_PWM_HIGHTIME_SHIFT 0 +#define NUMBER_OF_CYCLES_IN_PERIOD(x) ((x) << 12) +#define NUMBER_OF_CYCLES_IN_PERIOD_MASK (0xfff << 12) +#define NUMBER_OF_CYCLES_IN_PERIOD_SHIFT 12 +#define FORCE_STARTING_PWM_HIGHTIME (1 << 24) +#define INVERT_PWM_WAVEFORM (1 << 25) +#define FVTHROT_PWM_CTRL_REG1 0x3060 +#define MIN_PWM_HIGHTIME(x) ((x) << 0) +#define MIN_PWM_HIGHTIME_MASK 0xfff +#define MIN_PWM_HIGHTIME_SHIFT 0 +#define MAX_PWM_HIGHTIME(x) ((x) << 12) +#define MAX_PWM_HIGHTIME_MASK (0xfff << 12) +#define MAX_PWM_HIGHTIME_SHIFT 12 +#define FVTHROT_PWM_US_REG0 0x3064 +#define FVTHROT_PWM_US_REG1 0x3068 +#define FVTHROT_PWM_DS_REG0 0x306c +#define FVTHROT_PWM_DS_REG1 0x3070 +#define FVTHROT_STATUS_REG0 0x3074 +#define CURRENT_FEEDBACK_DIV_MASK 0xfff +#define CURRENT_FEEDBACK_DIV_SHIFT 0 +#define FVTHROT_STATUS_REG1 0x3078 +#define FVTHROT_STATUS_REG2 0x307c +#define CG_INTGFX_MISC 0x3080 +#define FVTHROT_VBLANK_SEL (1 << 9) +#define FVTHROT_PWM_FEEDBACK_DIV_REG1 0x308c +#define RANGE0_PWM_FEEDBACK_DIV(x) ((x) << 0) +#define RANGE0_PWM_FEEDBACK_DIV_MASK 0xfff +#define RANGE0_PWM_FEEDBACK_DIV_SHIFT 0 +#define RANGE_PWM_FEEDBACK_DIV_EN (1 << 12) +#define FVTHROT_PWM_FEEDBACK_DIV_REG2 0x3090 +#define RANGE1_PWM_FEEDBACK_DIV(x) ((x) << 0) +#define RANGE1_PWM_FEEDBACK_DIV_MASK 0xfff +#define RANGE1_PWM_FEEDBACK_DIV_SHIFT 0 +#define RANGE2_PWM_FEEDBACK_DIV(x) ((x) << 12) +#define RANGE2_PWM_FEEDBACK_DIV_MASK (0xfff << 12) +#define RANGE2_PWM_FEEDBACK_DIV_SHIFT 12 +#define FVTHROT_PWM_FEEDBACK_DIV_REG3 0x3094 +#define RANGE0_PWM(x) ((x) << 0) +#define RANGE0_PWM_MASK 0xfff +#define RANGE0_PWM_SHIFT 0 +#define RANGE1_PWM(x) ((x) << 12) +#define RANGE1_PWM_MASK (0xfff << 12) +#define RANGE1_PWM_SHIFT 12 +#define FVTHROT_PWM_FEEDBACK_DIV_REG4 0x3098 +#define RANGE2_PWM(x) ((x) << 0) +#define RANGE2_PWM_MASK 0xfff +#define RANGE2_PWM_SHIFT 0 +#define RANGE3_PWM(x) ((x) << 12) +#define RANGE3_PWM_MASK (0xfff << 12) +#define RANGE3_PWM_SHIFT 12 +#define FVTHROT_SLOW_CLK_FEEDBACK_DIV_REG1 0x30ac +#define RANGE0_SLOW_CLK_FEEDBACK_DIV(x) ((x) << 0) +#define RANGE0_SLOW_CLK_FEEDBACK_DIV_MASK 0xfff +#define RANGE0_SLOW_CLK_FEEDBACK_DIV_SHIFT 0 +#define RANGE_SLOW_CLK_FEEDBACK_DIV_EN (1 << 12) + +#define GFX_MACRO_BYPASS_CNTL 0x30c0 +#define SPLL_BYPASS_CNTL (1 << 0) +#define UPLL_BYPASS_CNTL (1 << 1) + +#endif diff --git a/sys/dev/pci/drm/radeon/rv200d.h b/sys/dev/pci/drm/radeon/rv200d.h index 384502a5beb..c5b398330c2 100644 --- a/sys/dev/pci/drm/radeon/rv200d.h +++ b/sys/dev/pci/drm/radeon/rv200d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: rv200d.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/rv250d.h b/sys/dev/pci/drm/radeon/rv250d.h index f20d17a3cb0..e5a70b06fe1 100644 --- a/sys/dev/pci/drm/radeon/rv250d.h +++ b/sys/dev/pci/drm/radeon/rv250d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: rv250d.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/rv350d.h b/sys/dev/pci/drm/radeon/rv350d.h index 09dcb1b7d70..c75c5ed9e65 100644 --- a/sys/dev/pci/drm/radeon/rv350d.h +++ b/sys/dev/pci/drm/radeon/rv350d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: rv350d.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. diff --git a/sys/dev/pci/drm/radeon/rv515.c b/sys/dev/pci/drm/radeon/rv515.c index 225b7245577..995e01221db 100644 --- a/sys/dev/pci/drm/radeon/rv515.c +++ b/sys/dev/pci/drm/radeon/rv515.c @@ -1,4 +1,3 @@ -/* $OpenBSD: rv515.c,v 1.8 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -123,7 +122,7 @@ void rv515_ring_start(struct radeon_device *rdev, struct radeon_ring *ring) radeon_ring_write(ring, GEOMETRY_ROUND_NEAREST | COLOR_ROUND_NEAREST); radeon_ring_write(ring, PACKET0(0x20C8, 0)); radeon_ring_write(ring, 0); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); } int rv515_mc_wait_for_idle(struct radeon_device *rdev) @@ -208,19 +207,27 @@ static void rv515_mc_init(struct radeon_device *rdev) uint32_t rv515_mc_rreg(struct radeon_device *rdev, uint32_t reg) { + unsigned long flags; uint32_t r; + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(MC_IND_INDEX, 0x7f0000 | (reg & 0xffff)); r = RREG32(MC_IND_DATA); WREG32(MC_IND_INDEX, 0); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); + return r; } void rv515_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v) { + unsigned long flags; + + spin_lock_irqsave(&rdev->mc_idx_lock, flags); WREG32(MC_IND_INDEX, 0xff0000 | ((reg) & 0xffff)); WREG32(MC_IND_DATA, (v)); WREG32(MC_IND_INDEX, 0); + spin_unlock_irqrestore(&rdev->mc_idx_lock, flags); } #if defined(CONFIG_DEBUG_FS) @@ -397,8 +404,9 @@ void rv515_mc_resume(struct radeon_device *rdev, struct rv515_mc_save *save) for (i = 0; i < rdev->num_crtc; i++) { if (save->crtc_enabled[i]) { tmp = RREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + crtc_offsets[i]); - if ((tmp & 0x3) != 0) { - tmp &= ~0x3; + if ((tmp & 0x7) != 3) { + tmp &= ~0x7; + tmp |= 0x3; WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + crtc_offsets[i], tmp); } tmp = RREG32(AVIVO_D1GRPH_UPDATE + crtc_offsets[i]); @@ -587,6 +595,7 @@ int rv515_resume(struct radeon_device *rdev) int rv515_suspend(struct radeon_device *rdev) { + radeon_pm_suspend(rdev); r100_cp_disable(rdev); radeon_wb_disable(rdev); rs600_irq_disable(rdev); @@ -603,6 +612,7 @@ void rv515_set_safe_registers(struct radeon_device *rdev) void rv515_fini(struct radeon_device *rdev) { + radeon_pm_fini(rdev); r100_cp_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); @@ -676,6 +686,9 @@ int rv515_init(struct radeon_device *rdev) return r; rv515_set_safe_registers(rdev); + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->accel_working = true; r = rv515_startup(rdev); if (r) { @@ -936,13 +949,16 @@ struct rv515_watermark { }; static void rv515_crtc_bandwidth_compute(struct radeon_device *rdev, - struct radeon_crtc *crtc, - struct rv515_watermark *wm) + struct radeon_crtc *crtc, + struct rv515_watermark *wm, + bool low) { struct drm_display_mode *mode = &crtc->base.mode; fixed20_12 a, b, c; fixed20_12 pclk, request_fifo_depth, tolerable_latency, estimated_width; fixed20_12 consumption_time, line_time, chunk_time, read_delay_latency; + fixed20_12 sclk; + u32 selected_sclk; if (!crtc->base.enabled) { /* FIXME: wouldn't it better to set priority mark to maximum */ @@ -950,6 +966,18 @@ static void rv515_crtc_bandwidth_compute(struct radeon_device *rdev, return; } + /* rv6xx, rv7xx */ + if ((rdev->family >= CHIP_RV610) && + (rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) + selected_sclk = radeon_dpm_get_sclk(rdev, low); + else + selected_sclk = rdev->pm.current_sclk; + + /* sclk in Mhz */ + a.full = dfixed_const(100); + sclk.full = dfixed_const(selected_sclk); + sclk.full = dfixed_div(sclk, a); + if (crtc->vsc.full > dfixed_const(2)) wm->num_line_pair.full = dfixed_const(2); else @@ -1015,7 +1043,7 @@ static void rv515_crtc_bandwidth_compute(struct radeon_device *rdev, * sclk = system clock(Mhz) */ a.full = dfixed_const(600 * 1000); - chunk_time.full = dfixed_div(a, rdev->pm.sclk); + chunk_time.full = dfixed_div(a, sclk); read_delay_latency.full = dfixed_const(1000); /* Determine the worst case latency @@ -1076,144 +1104,169 @@ static void rv515_crtc_bandwidth_compute(struct radeon_device *rdev, } } -void rv515_bandwidth_avivo_update(struct radeon_device *rdev) +static void rv515_compute_mode_priority(struct radeon_device *rdev, + struct rv515_watermark *wm0, + struct rv515_watermark *wm1, + struct drm_display_mode *mode0, + struct drm_display_mode *mode1, + u32 *d1mode_priority_a_cnt, + u32 *d2mode_priority_a_cnt) { - struct drm_display_mode *mode0 = NULL; - struct drm_display_mode *mode1 = NULL; - struct rv515_watermark wm0; - struct rv515_watermark wm1; - u32 tmp; - u32 d1mode_priority_a_cnt = MODE_PRIORITY_OFF; - u32 d2mode_priority_a_cnt = MODE_PRIORITY_OFF; fixed20_12 priority_mark02, priority_mark12, fill_rate; fixed20_12 a, b; - if (rdev->mode_info.crtcs[0]->base.enabled) - mode0 = &rdev->mode_info.crtcs[0]->base.mode; - if (rdev->mode_info.crtcs[1]->base.enabled) - mode1 = &rdev->mode_info.crtcs[1]->base.mode; - rs690_line_buffer_adjust(rdev, mode0, mode1); - - rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0); - rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1); - - tmp = wm0.lb_request_fifo_depth; - tmp |= wm1.lb_request_fifo_depth << 16; - WREG32(LB_MAX_REQ_OUTSTANDING, tmp); + *d1mode_priority_a_cnt = MODE_PRIORITY_OFF; + *d2mode_priority_a_cnt = MODE_PRIORITY_OFF; if (mode0 && mode1) { - if (dfixed_trunc(wm0.dbpp) > 64) - a.full = dfixed_div(wm0.dbpp, wm0.num_line_pair); + if (dfixed_trunc(wm0->dbpp) > 64) + a.full = dfixed_div(wm0->dbpp, wm0->num_line_pair); else - a.full = wm0.num_line_pair.full; - if (dfixed_trunc(wm1.dbpp) > 64) - b.full = dfixed_div(wm1.dbpp, wm1.num_line_pair); + a.full = wm0->num_line_pair.full; + if (dfixed_trunc(wm1->dbpp) > 64) + b.full = dfixed_div(wm1->dbpp, wm1->num_line_pair); else - b.full = wm1.num_line_pair.full; + b.full = wm1->num_line_pair.full; a.full += b.full; - fill_rate.full = dfixed_div(wm0.sclk, a); - if (wm0.consumption_rate.full > fill_rate.full) { - b.full = wm0.consumption_rate.full - fill_rate.full; - b.full = dfixed_mul(b, wm0.active_time); + fill_rate.full = dfixed_div(wm0->sclk, a); + if (wm0->consumption_rate.full > fill_rate.full) { + b.full = wm0->consumption_rate.full - fill_rate.full; + b.full = dfixed_mul(b, wm0->active_time); a.full = dfixed_const(16); b.full = dfixed_div(b, a); - a.full = dfixed_mul(wm0.worst_case_latency, - wm0.consumption_rate); + a.full = dfixed_mul(wm0->worst_case_latency, + wm0->consumption_rate); priority_mark02.full = a.full + b.full; } else { - a.full = dfixed_mul(wm0.worst_case_latency, - wm0.consumption_rate); + a.full = dfixed_mul(wm0->worst_case_latency, + wm0->consumption_rate); b.full = dfixed_const(16 * 1000); priority_mark02.full = dfixed_div(a, b); } - if (wm1.consumption_rate.full > fill_rate.full) { - b.full = wm1.consumption_rate.full - fill_rate.full; - b.full = dfixed_mul(b, wm1.active_time); + if (wm1->consumption_rate.full > fill_rate.full) { + b.full = wm1->consumption_rate.full - fill_rate.full; + b.full = dfixed_mul(b, wm1->active_time); a.full = dfixed_const(16); b.full = dfixed_div(b, a); - a.full = dfixed_mul(wm1.worst_case_latency, - wm1.consumption_rate); + a.full = dfixed_mul(wm1->worst_case_latency, + wm1->consumption_rate); priority_mark12.full = a.full + b.full; } else { - a.full = dfixed_mul(wm1.worst_case_latency, - wm1.consumption_rate); + a.full = dfixed_mul(wm1->worst_case_latency, + wm1->consumption_rate); b.full = dfixed_const(16 * 1000); priority_mark12.full = dfixed_div(a, b); } - if (wm0.priority_mark.full > priority_mark02.full) - priority_mark02.full = wm0.priority_mark.full; - if (wm0.priority_mark_max.full > priority_mark02.full) - priority_mark02.full = wm0.priority_mark_max.full; - if (wm1.priority_mark.full > priority_mark12.full) - priority_mark12.full = wm1.priority_mark.full; - if (wm1.priority_mark_max.full > priority_mark12.full) - priority_mark12.full = wm1.priority_mark_max.full; - d1mode_priority_a_cnt = dfixed_trunc(priority_mark02); - d2mode_priority_a_cnt = dfixed_trunc(priority_mark12); + if (wm0->priority_mark.full > priority_mark02.full) + priority_mark02.full = wm0->priority_mark.full; + if (wm0->priority_mark_max.full > priority_mark02.full) + priority_mark02.full = wm0->priority_mark_max.full; + if (wm1->priority_mark.full > priority_mark12.full) + priority_mark12.full = wm1->priority_mark.full; + if (wm1->priority_mark_max.full > priority_mark12.full) + priority_mark12.full = wm1->priority_mark_max.full; + *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02); + *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12); if (rdev->disp_priority == 2) { - d1mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON; - d2mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON; + *d1mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON; + *d2mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON; } } else if (mode0) { - if (dfixed_trunc(wm0.dbpp) > 64) - a.full = dfixed_div(wm0.dbpp, wm0.num_line_pair); + if (dfixed_trunc(wm0->dbpp) > 64) + a.full = dfixed_div(wm0->dbpp, wm0->num_line_pair); else - a.full = wm0.num_line_pair.full; - fill_rate.full = dfixed_div(wm0.sclk, a); - if (wm0.consumption_rate.full > fill_rate.full) { - b.full = wm0.consumption_rate.full - fill_rate.full; - b.full = dfixed_mul(b, wm0.active_time); + a.full = wm0->num_line_pair.full; + fill_rate.full = dfixed_div(wm0->sclk, a); + if (wm0->consumption_rate.full > fill_rate.full) { + b.full = wm0->consumption_rate.full - fill_rate.full; + b.full = dfixed_mul(b, wm0->active_time); a.full = dfixed_const(16); b.full = dfixed_div(b, a); - a.full = dfixed_mul(wm0.worst_case_latency, - wm0.consumption_rate); + a.full = dfixed_mul(wm0->worst_case_latency, + wm0->consumption_rate); priority_mark02.full = a.full + b.full; } else { - a.full = dfixed_mul(wm0.worst_case_latency, - wm0.consumption_rate); + a.full = dfixed_mul(wm0->worst_case_latency, + wm0->consumption_rate); b.full = dfixed_const(16); priority_mark02.full = dfixed_div(a, b); } - if (wm0.priority_mark.full > priority_mark02.full) - priority_mark02.full = wm0.priority_mark.full; - if (wm0.priority_mark_max.full > priority_mark02.full) - priority_mark02.full = wm0.priority_mark_max.full; - d1mode_priority_a_cnt = dfixed_trunc(priority_mark02); + if (wm0->priority_mark.full > priority_mark02.full) + priority_mark02.full = wm0->priority_mark.full; + if (wm0->priority_mark_max.full > priority_mark02.full) + priority_mark02.full = wm0->priority_mark_max.full; + *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02); if (rdev->disp_priority == 2) - d1mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON; + *d1mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON; } else if (mode1) { - if (dfixed_trunc(wm1.dbpp) > 64) - a.full = dfixed_div(wm1.dbpp, wm1.num_line_pair); + if (dfixed_trunc(wm1->dbpp) > 64) + a.full = dfixed_div(wm1->dbpp, wm1->num_line_pair); else - a.full = wm1.num_line_pair.full; - fill_rate.full = dfixed_div(wm1.sclk, a); - if (wm1.consumption_rate.full > fill_rate.full) { - b.full = wm1.consumption_rate.full - fill_rate.full; - b.full = dfixed_mul(b, wm1.active_time); + a.full = wm1->num_line_pair.full; + fill_rate.full = dfixed_div(wm1->sclk, a); + if (wm1->consumption_rate.full > fill_rate.full) { + b.full = wm1->consumption_rate.full - fill_rate.full; + b.full = dfixed_mul(b, wm1->active_time); a.full = dfixed_const(16); b.full = dfixed_div(b, a); - a.full = dfixed_mul(wm1.worst_case_latency, - wm1.consumption_rate); + a.full = dfixed_mul(wm1->worst_case_latency, + wm1->consumption_rate); priority_mark12.full = a.full + b.full; } else { - a.full = dfixed_mul(wm1.worst_case_latency, - wm1.consumption_rate); + a.full = dfixed_mul(wm1->worst_case_latency, + wm1->consumption_rate); b.full = dfixed_const(16 * 1000); priority_mark12.full = dfixed_div(a, b); } - if (wm1.priority_mark.full > priority_mark12.full) - priority_mark12.full = wm1.priority_mark.full; - if (wm1.priority_mark_max.full > priority_mark12.full) - priority_mark12.full = wm1.priority_mark_max.full; - d2mode_priority_a_cnt = dfixed_trunc(priority_mark12); + if (wm1->priority_mark.full > priority_mark12.full) + priority_mark12.full = wm1->priority_mark.full; + if (wm1->priority_mark_max.full > priority_mark12.full) + priority_mark12.full = wm1->priority_mark_max.full; + *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12); if (rdev->disp_priority == 2) - d2mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON; + *d2mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON; } +} + +void rv515_bandwidth_avivo_update(struct radeon_device *rdev) +{ + struct drm_display_mode *mode0 = NULL; + struct drm_display_mode *mode1 = NULL; + struct rv515_watermark wm0_high, wm0_low; + struct rv515_watermark wm1_high, wm1_low; + u32 tmp; + u32 d1mode_priority_a_cnt, d1mode_priority_b_cnt; + u32 d2mode_priority_a_cnt, d2mode_priority_b_cnt; + + if (rdev->mode_info.crtcs[0]->base.enabled) + mode0 = &rdev->mode_info.crtcs[0]->base.mode; + if (rdev->mode_info.crtcs[1]->base.enabled) + mode1 = &rdev->mode_info.crtcs[1]->base.mode; + rs690_line_buffer_adjust(rdev, mode0, mode1); + + rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0_high, false); + rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1_high, false); + + rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0_low, false); + rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1_low, false); + + tmp = wm0_high.lb_request_fifo_depth; + tmp |= wm1_high.lb_request_fifo_depth << 16; + WREG32(LB_MAX_REQ_OUTSTANDING, tmp); + + rv515_compute_mode_priority(rdev, + &wm0_high, &wm1_high, + mode0, mode1, + &d1mode_priority_a_cnt, &d2mode_priority_a_cnt); + rv515_compute_mode_priority(rdev, + &wm0_low, &wm1_low, + mode0, mode1, + &d1mode_priority_b_cnt, &d2mode_priority_b_cnt); WREG32(D1MODE_PRIORITY_A_CNT, d1mode_priority_a_cnt); - WREG32(D1MODE_PRIORITY_B_CNT, d1mode_priority_a_cnt); + WREG32(D1MODE_PRIORITY_B_CNT, d1mode_priority_b_cnt); WREG32(D2MODE_PRIORITY_A_CNT, d2mode_priority_a_cnt); - WREG32(D2MODE_PRIORITY_B_CNT, d2mode_priority_a_cnt); + WREG32(D2MODE_PRIORITY_B_CNT, d2mode_priority_b_cnt); } void rv515_bandwidth_update(struct radeon_device *rdev) @@ -1222,6 +1275,9 @@ void rv515_bandwidth_update(struct radeon_device *rdev) struct drm_display_mode *mode0 = NULL; struct drm_display_mode *mode1 = NULL; + if (!rdev->mode_info.mode_config_initialized) + return; + radeon_update_display_priority(rdev); if (rdev->mode_info.crtcs[0]->base.enabled) diff --git a/sys/dev/pci/drm/radeon/rv515d.h b/sys/dev/pci/drm/radeon/rv515d.h index 707c6f33d90..6927a200daf 100644 --- a/sys/dev/pci/drm/radeon/rv515d.h +++ b/sys/dev/pci/drm/radeon/rv515d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: rv515d.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -206,17 +205,6 @@ REG_SET(PACKET3_IT_OPCODE, (op)) | \ REG_SET(PACKET3_COUNT, (n))) -#define PACKET_TYPE0 0 -#define PACKET_TYPE1 1 -#define PACKET_TYPE2 2 -#define PACKET_TYPE3 3 - -#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) -#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) -#define CP_PACKET0_GET_REG(h) (((h) & 0x1FFF) << 2) -#define CP_PACKET0_GET_ONE_REG_WR(h) (((h) >> 15) & 1) -#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) - /* Registers */ #define R_0000F0_RBBM_SOFT_RESET 0x0000F0 #define S_0000F0_SOFT_RESET_CP(x) (((x) & 0x1) << 0) diff --git a/sys/dev/pci/drm/radeon/rv6xx_dpm.c b/sys/dev/pci/drm/radeon/rv6xx_dpm.c new file mode 100644 index 00000000000..6278967ee77 --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv6xx_dpm.c @@ -0,0 +1,2158 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "rv6xxd.h" +#include "r600_dpm.h" +#include "rv6xx_dpm.h" +#include "atom.h" + +static u32 rv6xx_scale_count_given_unit(struct radeon_device *rdev, + u32 unscaled_count, u32 unit); + +static struct rv6xx_ps *rv6xx_get_ps(struct radeon_ps *rps) +{ + struct rv6xx_ps *ps = rps->ps_priv; + + return ps; +} + +static struct rv6xx_power_info *rv6xx_get_pi(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +static void rv6xx_force_pcie_gen1(struct radeon_device *rdev) +{ + u32 tmp; + int i; + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + tmp &= LC_GEN2_EN; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + tmp |= LC_INITIATE_LINK_SPEED_CHANGE; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (!(RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE)) + break; + udelay(1); + } + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + tmp &= ~LC_INITIATE_LINK_SPEED_CHANGE; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); +} + +static void rv6xx_enable_pcie_gen2_support(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) && + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) { + tmp |= LC_GEN2_EN; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + } +} + +static void rv6xx_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev, + bool enable) +{ + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & ~LC_HW_VOLTAGE_IF_CONTROL_MASK; + if (enable) + tmp |= LC_HW_VOLTAGE_IF_CONTROL(1); + else + tmp |= LC_HW_VOLTAGE_IF_CONTROL(0); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); +} + +static void rv6xx_enable_l0s(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L0S_INACTIVITY_MASK; + tmp |= LC_L0S_INACTIVITY(3); + WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp); +} + +static void rv6xx_enable_l1(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL); + tmp &= ~LC_L1_INACTIVITY_MASK; + tmp |= LC_L1_INACTIVITY(4); + tmp &= ~LC_PMI_TO_L1_DIS; + tmp &= ~LC_ASPM_TO_L1_DIS; + WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp); +} + +static void rv6xx_enable_pll_sleep_in_l1(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L1_INACTIVITY_MASK; + tmp |= LC_L1_INACTIVITY(8); + WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp); + + /* NOTE, this is a PCIE indirect reg, not PCIE PORT */ + tmp = RREG32_PCIE(PCIE_P_CNTL); + tmp |= P_PLL_PWRDN_IN_L1L23; + tmp &= ~P_PLL_BUF_PDNB; + tmp &= ~P_PLL_PDNB; + tmp |= P_ALLOW_PRX_FRONTEND_SHUTOFF; + WREG32_PCIE(PCIE_P_CNTL, tmp); +} + +static int rv6xx_convert_clock_to_stepping(struct radeon_device *rdev, + u32 clock, struct rv6xx_sclk_stepping *step) +{ + int ret; + struct atom_clock_dividers dividers; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + clock, false, ÷rs); + if (ret) + return ret; + + if (dividers.enable_post_div) + step->post_divider = 2 + (dividers.post_div & 0xF) + (dividers.post_div >> 4); + else + step->post_divider = 1; + + step->vco_frequency = clock * step->post_divider; + + return 0; +} + +static void rv6xx_output_stepping(struct radeon_device *rdev, + u32 step_index, struct rv6xx_sclk_stepping *step) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + u32 ref_clk = rdev->clock.spll.reference_freq; + u32 fb_divider; + u32 spll_step_count = rv6xx_scale_count_given_unit(rdev, + R600_SPLLSTEPTIME_DFLT * + pi->spll_ref_div, + R600_SPLLSTEPUNIT_DFLT); + + r600_engine_clock_entry_enable(rdev, step_index, true); + r600_engine_clock_entry_enable_pulse_skipping(rdev, step_index, false); + + if (step->post_divider == 1) + r600_engine_clock_entry_enable_post_divider(rdev, step_index, false); + else { + u32 lo_len = (step->post_divider - 2) / 2; + u32 hi_len = step->post_divider - 2 - lo_len; + + r600_engine_clock_entry_enable_post_divider(rdev, step_index, true); + r600_engine_clock_entry_set_post_divider(rdev, step_index, (hi_len << 4) | lo_len); + } + + fb_divider = ((step->vco_frequency * pi->spll_ref_div) / ref_clk) >> + pi->fb_div_scale; + + r600_engine_clock_entry_set_reference_divider(rdev, step_index, + pi->spll_ref_div - 1); + r600_engine_clock_entry_set_feedback_divider(rdev, step_index, fb_divider); + r600_engine_clock_entry_set_step_time(rdev, step_index, spll_step_count); + +} + +static struct rv6xx_sclk_stepping rv6xx_next_vco_step(struct radeon_device *rdev, + struct rv6xx_sclk_stepping *cur, + bool increasing_vco, u32 step_size) +{ + struct rv6xx_sclk_stepping next; + + next.post_divider = cur->post_divider; + + if (increasing_vco) + next.vco_frequency = (cur->vco_frequency * (100 + step_size)) / 100; + else + next.vco_frequency = (cur->vco_frequency * 100 + 99 + step_size) / (100 + step_size); + + return next; +} + +static bool rv6xx_can_step_post_div(struct radeon_device *rdev, + struct rv6xx_sclk_stepping *cur, + struct rv6xx_sclk_stepping *target) +{ + return (cur->post_divider > target->post_divider) && + ((cur->vco_frequency * target->post_divider) <= + (target->vco_frequency * (cur->post_divider - 1))); +} + +static struct rv6xx_sclk_stepping rv6xx_next_post_div_step(struct radeon_device *rdev, + struct rv6xx_sclk_stepping *cur, + struct rv6xx_sclk_stepping *target) +{ + struct rv6xx_sclk_stepping next = *cur; + + while (rv6xx_can_step_post_div(rdev, &next, target)) + next.post_divider--; + + return next; +} + +static bool rv6xx_reached_stepping_target(struct radeon_device *rdev, + struct rv6xx_sclk_stepping *cur, + struct rv6xx_sclk_stepping *target, + bool increasing_vco) +{ + return (increasing_vco && (cur->vco_frequency >= target->vco_frequency)) || + (!increasing_vco && (cur->vco_frequency <= target->vco_frequency)); +} + +static void rv6xx_generate_steps(struct radeon_device *rdev, + u32 low, u32 high, + u32 start_index, u8 *end_index) +{ + struct rv6xx_sclk_stepping cur; + struct rv6xx_sclk_stepping target; + bool increasing_vco; + u32 step_index = start_index; + + rv6xx_convert_clock_to_stepping(rdev, low, &cur); + rv6xx_convert_clock_to_stepping(rdev, high, &target); + + rv6xx_output_stepping(rdev, step_index++, &cur); + + increasing_vco = (target.vco_frequency >= cur.vco_frequency); + + if (target.post_divider > cur.post_divider) + cur.post_divider = target.post_divider; + + while (1) { + struct rv6xx_sclk_stepping next; + + if (rv6xx_can_step_post_div(rdev, &cur, &target)) + next = rv6xx_next_post_div_step(rdev, &cur, &target); + else + next = rv6xx_next_vco_step(rdev, &cur, increasing_vco, R600_VCOSTEPPCT_DFLT); + + if (rv6xx_reached_stepping_target(rdev, &next, &target, increasing_vco)) { + struct rv6xx_sclk_stepping tiny = + rv6xx_next_vco_step(rdev, &target, !increasing_vco, R600_ENDINGVCOSTEPPCT_DFLT); + tiny.post_divider = next.post_divider; + + if (!rv6xx_reached_stepping_target(rdev, &tiny, &cur, !increasing_vco)) + rv6xx_output_stepping(rdev, step_index++, &tiny); + + if ((next.post_divider != target.post_divider) && + (next.vco_frequency != target.vco_frequency)) { + struct rv6xx_sclk_stepping final_vco; + + final_vco.vco_frequency = target.vco_frequency; + final_vco.post_divider = next.post_divider; + + rv6xx_output_stepping(rdev, step_index++, &final_vco); + } + + rv6xx_output_stepping(rdev, step_index++, &target); + break; + } else + rv6xx_output_stepping(rdev, step_index++, &next); + + cur = next; + } + + *end_index = (u8)step_index - 1; + +} + +static void rv6xx_generate_single_step(struct radeon_device *rdev, + u32 clock, u32 index) +{ + struct rv6xx_sclk_stepping step; + + rv6xx_convert_clock_to_stepping(rdev, clock, &step); + rv6xx_output_stepping(rdev, index, &step); +} + +static void rv6xx_invalidate_intermediate_steps_range(struct radeon_device *rdev, + u32 start_index, u32 end_index) +{ + u32 step_index; + + for (step_index = start_index + 1; step_index < end_index; step_index++) + r600_engine_clock_entry_enable(rdev, step_index, false); +} + +static void rv6xx_set_engine_spread_spectrum_clk_s(struct radeon_device *rdev, + u32 index, u32 clk_s) +{ + WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4), + CLKS(clk_s), ~CLKS_MASK); +} + +static void rv6xx_set_engine_spread_spectrum_clk_v(struct radeon_device *rdev, + u32 index, u32 clk_v) +{ + WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4), + CLKV(clk_v), ~CLKV_MASK); +} + +static void rv6xx_enable_engine_spread_spectrum(struct radeon_device *rdev, + u32 index, bool enable) +{ + if (enable) + WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4), + SSEN, ~SSEN); + else + WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4), + 0, ~SSEN); +} + +static void rv6xx_set_memory_spread_spectrum_clk_s(struct radeon_device *rdev, + u32 clk_s) +{ + WREG32_P(CG_MPLL_SPREAD_SPECTRUM, CLKS(clk_s), ~CLKS_MASK); +} + +static void rv6xx_set_memory_spread_spectrum_clk_v(struct radeon_device *rdev, + u32 clk_v) +{ + WREG32_P(CG_MPLL_SPREAD_SPECTRUM, CLKV(clk_v), ~CLKV_MASK); +} + +static void rv6xx_enable_memory_spread_spectrum(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(CG_MPLL_SPREAD_SPECTRUM, SSEN, ~SSEN); + else + WREG32_P(CG_MPLL_SPREAD_SPECTRUM, 0, ~SSEN); +} + +static void rv6xx_enable_dynamic_spread_spectrum(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN); + else + WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN); +} + +static void rv6xx_memory_clock_entry_enable_post_divider(struct radeon_device *rdev, + u32 index, bool enable) +{ + if (enable) + WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), + LEVEL0_MPLL_DIV_EN, ~LEVEL0_MPLL_DIV_EN); + else + WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), 0, ~LEVEL0_MPLL_DIV_EN); +} + +static void rv6xx_memory_clock_entry_set_post_divider(struct radeon_device *rdev, + u32 index, u32 divider) +{ + WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), + LEVEL0_MPLL_POST_DIV(divider), ~LEVEL0_MPLL_POST_DIV_MASK); +} + +static void rv6xx_memory_clock_entry_set_feedback_divider(struct radeon_device *rdev, + u32 index, u32 divider) +{ + WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), LEVEL0_MPLL_FB_DIV(divider), + ~LEVEL0_MPLL_FB_DIV_MASK); +} + +static void rv6xx_memory_clock_entry_set_reference_divider(struct radeon_device *rdev, + u32 index, u32 divider) +{ + WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), + LEVEL0_MPLL_REF_DIV(divider), ~LEVEL0_MPLL_REF_DIV_MASK); +} + +static void rv6xx_vid_response_set_brt(struct radeon_device *rdev, u32 rt) +{ + WREG32_P(VID_RT, BRT(rt), ~BRT_MASK); +} + +static void rv6xx_enable_engine_feedback_and_reference_sync(struct radeon_device *rdev) +{ + WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC); +} + +static u32 rv6xx_clocks_per_unit(u32 unit) +{ + u32 tmp = 1 << (2 * unit); + + return tmp; +} + +static u32 rv6xx_scale_count_given_unit(struct radeon_device *rdev, + u32 unscaled_count, u32 unit) +{ + u32 count_per_unit = rv6xx_clocks_per_unit(unit); + + return (unscaled_count + count_per_unit - 1) / count_per_unit; +} + +static u32 rv6xx_compute_count_for_delay(struct radeon_device *rdev, + u32 delay_us, u32 unit) +{ + u32 ref_clk = rdev->clock.spll.reference_freq; + + return rv6xx_scale_count_given_unit(rdev, delay_us * (ref_clk / 100), unit); +} + +static void rv6xx_calculate_engine_speed_stepping_parameters(struct radeon_device *rdev, + struct rv6xx_ps *state) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + pi->hw.sclks[R600_POWER_LEVEL_LOW] = + state->low.sclk; + pi->hw.sclks[R600_POWER_LEVEL_MEDIUM] = + state->medium.sclk; + pi->hw.sclks[R600_POWER_LEVEL_HIGH] = + state->high.sclk; + + pi->hw.low_sclk_index = R600_POWER_LEVEL_LOW; + pi->hw.medium_sclk_index = R600_POWER_LEVEL_MEDIUM; + pi->hw.high_sclk_index = R600_POWER_LEVEL_HIGH; +} + +static void rv6xx_calculate_memory_clock_stepping_parameters(struct radeon_device *rdev, + struct rv6xx_ps *state) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + pi->hw.mclks[R600_POWER_LEVEL_CTXSW] = + state->high.mclk; + pi->hw.mclks[R600_POWER_LEVEL_HIGH] = + state->high.mclk; + pi->hw.mclks[R600_POWER_LEVEL_MEDIUM] = + state->medium.mclk; + pi->hw.mclks[R600_POWER_LEVEL_LOW] = + state->low.mclk; + + pi->hw.high_mclk_index = R600_POWER_LEVEL_HIGH; + + if (state->high.mclk == state->medium.mclk) + pi->hw.medium_mclk_index = + pi->hw.high_mclk_index; + else + pi->hw.medium_mclk_index = R600_POWER_LEVEL_MEDIUM; + + + if (state->medium.mclk == state->low.mclk) + pi->hw.low_mclk_index = + pi->hw.medium_mclk_index; + else + pi->hw.low_mclk_index = R600_POWER_LEVEL_LOW; +} + +static void rv6xx_calculate_voltage_stepping_parameters(struct radeon_device *rdev, + struct rv6xx_ps *state) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + pi->hw.vddc[R600_POWER_LEVEL_CTXSW] = state->high.vddc; + pi->hw.vddc[R600_POWER_LEVEL_HIGH] = state->high.vddc; + pi->hw.vddc[R600_POWER_LEVEL_MEDIUM] = state->medium.vddc; + pi->hw.vddc[R600_POWER_LEVEL_LOW] = state->low.vddc; + + pi->hw.backbias[R600_POWER_LEVEL_CTXSW] = + (state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false; + pi->hw.backbias[R600_POWER_LEVEL_HIGH] = + (state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false; + pi->hw.backbias[R600_POWER_LEVEL_MEDIUM] = + (state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false; + pi->hw.backbias[R600_POWER_LEVEL_LOW] = + (state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false; + + pi->hw.pcie_gen2[R600_POWER_LEVEL_HIGH] = + (state->high.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false; + pi->hw.pcie_gen2[R600_POWER_LEVEL_MEDIUM] = + (state->medium.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false; + pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW] = + (state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false; + + pi->hw.high_vddc_index = R600_POWER_LEVEL_HIGH; + + if ((state->high.vddc == state->medium.vddc) && + ((state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) == + (state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE))) + pi->hw.medium_vddc_index = + pi->hw.high_vddc_index; + else + pi->hw.medium_vddc_index = R600_POWER_LEVEL_MEDIUM; + + if ((state->medium.vddc == state->low.vddc) && + ((state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) == + (state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE))) + pi->hw.low_vddc_index = + pi->hw.medium_vddc_index; + else + pi->hw.medium_vddc_index = R600_POWER_LEVEL_LOW; +} + +static inline u32 rv6xx_calculate_vco_frequency(u32 ref_clock, + struct atom_clock_dividers *dividers, + u32 fb_divider_scale) +{ + return ref_clock * ((dividers->fb_div & ~1) << fb_divider_scale) / + (dividers->ref_div + 1); +} + +static inline u32 rv6xx_calculate_spread_spectrum_clk_v(u32 vco_freq, u32 ref_freq, + u32 ss_rate, u32 ss_percent, + u32 fb_divider_scale) +{ + u32 fb_divider = vco_freq / ref_freq; + + return (ss_percent * ss_rate * 4 * (fb_divider * fb_divider) / + (5375 * ((vco_freq * 10) / (4096 >> fb_divider_scale)))); +} + +static inline u32 rv6xx_calculate_spread_spectrum_clk_s(u32 ss_rate, u32 ref_freq) +{ + return (((ref_freq * 10) / (ss_rate * 2)) - 1) / 4; +} + +static void rv6xx_program_engine_spread_spectrum(struct radeon_device *rdev, + u32 clock, enum r600_power_level level) +{ + u32 ref_clk = rdev->clock.spll.reference_freq; + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + struct atom_clock_dividers dividers; + struct radeon_atom_ss ss; + u32 vco_freq, clk_v, clk_s; + + rv6xx_enable_engine_spread_spectrum(rdev, level, false); + + if (clock && pi->sclk_ss) { + if (radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, clock, false, ÷rs) == 0) { + vco_freq = rv6xx_calculate_vco_frequency(ref_clk, ÷rs, + pi->fb_div_scale); + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_ENGINE_SS, vco_freq)) { + clk_v = rv6xx_calculate_spread_spectrum_clk_v(vco_freq, + (ref_clk / (dividers.ref_div + 1)), + ss.rate, + ss.percentage, + pi->fb_div_scale); + + clk_s = rv6xx_calculate_spread_spectrum_clk_s(ss.rate, + (ref_clk / (dividers.ref_div + 1))); + + rv6xx_set_engine_spread_spectrum_clk_v(rdev, level, clk_v); + rv6xx_set_engine_spread_spectrum_clk_s(rdev, level, clk_s); + rv6xx_enable_engine_spread_spectrum(rdev, level, true); + } + } + } +} + +static void rv6xx_program_sclk_spread_spectrum_parameters_except_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + rv6xx_program_engine_spread_spectrum(rdev, + pi->hw.sclks[R600_POWER_LEVEL_HIGH], + R600_POWER_LEVEL_HIGH); + + rv6xx_program_engine_spread_spectrum(rdev, + pi->hw.sclks[R600_POWER_LEVEL_MEDIUM], + R600_POWER_LEVEL_MEDIUM); + +} + +static int rv6xx_program_mclk_stepping_entry(struct radeon_device *rdev, + u32 entry, u32 clock) +{ + struct atom_clock_dividers dividers; + + if (radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, clock, false, ÷rs)) + return -EINVAL; + + + rv6xx_memory_clock_entry_set_reference_divider(rdev, entry, dividers.ref_div); + rv6xx_memory_clock_entry_set_feedback_divider(rdev, entry, dividers.fb_div); + rv6xx_memory_clock_entry_set_post_divider(rdev, entry, dividers.post_div); + + if (dividers.enable_post_div) + rv6xx_memory_clock_entry_enable_post_divider(rdev, entry, true); + else + rv6xx_memory_clock_entry_enable_post_divider(rdev, entry, false); + + return 0; +} + +static void rv6xx_program_mclk_stepping_parameters_except_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + int i; + + for (i = 1; i < R600_PM_NUMBER_OF_MCLKS; i++) { + if (pi->hw.mclks[i]) + rv6xx_program_mclk_stepping_entry(rdev, i, + pi->hw.mclks[i]); + } +} + +static void rv6xx_find_memory_clock_with_highest_vco(struct radeon_device *rdev, + u32 requested_memory_clock, + u32 ref_clk, + struct atom_clock_dividers *dividers, + u32 *vco_freq) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + struct atom_clock_dividers req_dividers; + u32 vco_freq_temp; + + if (radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, + requested_memory_clock, false, &req_dividers) == 0) { + vco_freq_temp = rv6xx_calculate_vco_frequency(ref_clk, &req_dividers, + pi->fb_div_scale); + + if (vco_freq_temp > *vco_freq) { + *dividers = req_dividers; + *vco_freq = vco_freq_temp; + } + } +} + +static void rv6xx_program_mclk_spread_spectrum_parameters(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + u32 ref_clk = rdev->clock.mpll.reference_freq; + struct atom_clock_dividers dividers; + struct radeon_atom_ss ss; + u32 vco_freq = 0, clk_v, clk_s; + + rv6xx_enable_memory_spread_spectrum(rdev, false); + + if (pi->mclk_ss) { + rv6xx_find_memory_clock_with_highest_vco(rdev, + pi->hw.mclks[pi->hw.high_mclk_index], + ref_clk, + ÷rs, + &vco_freq); + + rv6xx_find_memory_clock_with_highest_vco(rdev, + pi->hw.mclks[pi->hw.medium_mclk_index], + ref_clk, + ÷rs, + &vco_freq); + + rv6xx_find_memory_clock_with_highest_vco(rdev, + pi->hw.mclks[pi->hw.low_mclk_index], + ref_clk, + ÷rs, + &vco_freq); + + if (vco_freq) { + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_MEMORY_SS, vco_freq)) { + clk_v = rv6xx_calculate_spread_spectrum_clk_v(vco_freq, + (ref_clk / (dividers.ref_div + 1)), + ss.rate, + ss.percentage, + pi->fb_div_scale); + + clk_s = rv6xx_calculate_spread_spectrum_clk_s(ss.rate, + (ref_clk / (dividers.ref_div + 1))); + + rv6xx_set_memory_spread_spectrum_clk_v(rdev, clk_v); + rv6xx_set_memory_spread_spectrum_clk_s(rdev, clk_s); + rv6xx_enable_memory_spread_spectrum(rdev, true); + } + } + } +} + +static int rv6xx_program_voltage_stepping_entry(struct radeon_device *rdev, + u32 entry, u16 voltage) +{ + u32 mask, set_pins; + int ret; + + ret = radeon_atom_get_voltage_gpio_settings(rdev, voltage, + SET_VOLTAGE_TYPE_ASIC_VDDC, + &set_pins, &mask); + if (ret) + return ret; + + r600_voltage_control_program_voltages(rdev, entry, set_pins); + + return 0; +} + +static void rv6xx_program_voltage_stepping_parameters_except_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + int i; + + for (i = 1; i < R600_PM_NUMBER_OF_VOLTAGE_LEVELS; i++) + rv6xx_program_voltage_stepping_entry(rdev, i, + pi->hw.vddc[i]); + +} + +static void rv6xx_program_backbias_stepping_parameters_except_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + if (pi->hw.backbias[1]) + WREG32_P(VID_UPPER_GPIO_CNTL, MEDIUM_BACKBIAS_VALUE, ~MEDIUM_BACKBIAS_VALUE); + else + WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~MEDIUM_BACKBIAS_VALUE); + + if (pi->hw.backbias[2]) + WREG32_P(VID_UPPER_GPIO_CNTL, HIGH_BACKBIAS_VALUE, ~HIGH_BACKBIAS_VALUE); + else + WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~HIGH_BACKBIAS_VALUE); +} + +static void rv6xx_program_sclk_spread_spectrum_parameters_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + rv6xx_program_engine_spread_spectrum(rdev, + pi->hw.sclks[R600_POWER_LEVEL_LOW], + R600_POWER_LEVEL_LOW); +} + +static void rv6xx_program_mclk_stepping_parameters_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + if (pi->hw.mclks[0]) + rv6xx_program_mclk_stepping_entry(rdev, 0, + pi->hw.mclks[0]); +} + +static void rv6xx_program_voltage_stepping_parameters_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + rv6xx_program_voltage_stepping_entry(rdev, 0, + pi->hw.vddc[0]); + +} + +static void rv6xx_program_backbias_stepping_parameters_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + if (pi->hw.backbias[0]) + WREG32_P(VID_UPPER_GPIO_CNTL, LOW_BACKBIAS_VALUE, ~LOW_BACKBIAS_VALUE); + else + WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~LOW_BACKBIAS_VALUE); +} + +static u32 calculate_memory_refresh_rate(struct radeon_device *rdev, + u32 engine_clock) +{ + u32 dram_rows, dram_refresh_rate; + u32 tmp; + + tmp = (RREG32(RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT; + dram_rows = 1 << (tmp + 10); + dram_refresh_rate = 1 << ((RREG32(MC_SEQ_RESERVE_M) & 0x3) + 3); + + return ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64; +} + +static void rv6xx_program_memory_timing_parameters(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + u32 sqm_ratio; + u32 arb_refresh_rate; + u32 high_clock; + + if (pi->hw.sclks[R600_POWER_LEVEL_HIGH] < + (pi->hw.sclks[R600_POWER_LEVEL_LOW] * 0xFF / 0x40)) + high_clock = pi->hw.sclks[R600_POWER_LEVEL_HIGH]; + else + high_clock = + pi->hw.sclks[R600_POWER_LEVEL_LOW] * 0xFF / 0x40; + + radeon_atom_set_engine_dram_timings(rdev, high_clock, 0); + + sqm_ratio = (STATE0(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_LOW]) | + STATE1(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_MEDIUM]) | + STATE2(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_HIGH]) | + STATE3(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_HIGH])); + WREG32(SQM_RATIO, sqm_ratio); + + arb_refresh_rate = + (POWERMODE0(calculate_memory_refresh_rate(rdev, + pi->hw.sclks[R600_POWER_LEVEL_LOW])) | + POWERMODE1(calculate_memory_refresh_rate(rdev, + pi->hw.sclks[R600_POWER_LEVEL_MEDIUM])) | + POWERMODE2(calculate_memory_refresh_rate(rdev, + pi->hw.sclks[R600_POWER_LEVEL_HIGH])) | + POWERMODE3(calculate_memory_refresh_rate(rdev, + pi->hw.sclks[R600_POWER_LEVEL_HIGH]))); + WREG32(ARB_RFSH_RATE, arb_refresh_rate); +} + +static void rv6xx_program_mpll_timing_parameters(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + r600_set_mpll_lock_time(rdev, R600_MPLLLOCKTIME_DFLT * + pi->mpll_ref_div); + r600_set_mpll_reset_time(rdev, R600_MPLLRESETTIME_DFLT); +} + +static void rv6xx_program_bsp(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + u32 ref_clk = rdev->clock.spll.reference_freq; + + r600_calculate_u_and_p(R600_ASI_DFLT, + ref_clk, 16, + &pi->bsp, + &pi->bsu); + + r600_set_bsp(rdev, pi->bsu, pi->bsp); +} + +static void rv6xx_program_at(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + r600_set_at(rdev, + (pi->hw.rp[0] * pi->bsp) / 200, + (pi->hw.rp[1] * pi->bsp) / 200, + (pi->hw.lp[2] * pi->bsp) / 200, + (pi->hw.lp[1] * pi->bsp) / 200); +} + +static void rv6xx_program_git(struct radeon_device *rdev) +{ + r600_set_git(rdev, R600_GICST_DFLT); +} + +static void rv6xx_program_tp(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < R600_PM_NUMBER_OF_TC; i++) + r600_set_tc(rdev, i, r600_utc[i], r600_dtc[i]); + + r600_select_td(rdev, R600_TD_DFLT); +} + +static void rv6xx_program_vc(struct radeon_device *rdev) +{ + r600_set_vrc(rdev, R600_VRC_DFLT); +} + +static void rv6xx_clear_vc(struct radeon_device *rdev) +{ + r600_set_vrc(rdev, 0); +} + +static void rv6xx_program_tpp(struct radeon_device *rdev) +{ + r600_set_tpu(rdev, R600_TPU_DFLT); + r600_set_tpc(rdev, R600_TPC_DFLT); +} + +static void rv6xx_program_sstp(struct radeon_device *rdev) +{ + r600_set_sstu(rdev, R600_SSTU_DFLT); + r600_set_sst(rdev, R600_SST_DFLT); +} + +static void rv6xx_program_fcp(struct radeon_device *rdev) +{ + r600_set_fctu(rdev, R600_FCTU_DFLT); + r600_set_fct(rdev, R600_FCT_DFLT); +} + +static void rv6xx_program_vddc3d_parameters(struct radeon_device *rdev) +{ + r600_set_vddc3d_oorsu(rdev, R600_VDDC3DOORSU_DFLT); + r600_set_vddc3d_oorphc(rdev, R600_VDDC3DOORPHC_DFLT); + r600_set_vddc3d_oorsdc(rdev, R600_VDDC3DOORSDC_DFLT); + r600_set_ctxcgtt3d_rphc(rdev, R600_CTXCGTT3DRPHC_DFLT); + r600_set_ctxcgtt3d_rsdc(rdev, R600_CTXCGTT3DRSDC_DFLT); +} + +static void rv6xx_program_voltage_timing_parameters(struct radeon_device *rdev) +{ + u32 rt; + + r600_vid_rt_set_vru(rdev, R600_VRU_DFLT); + + r600_vid_rt_set_vrt(rdev, + rv6xx_compute_count_for_delay(rdev, + rdev->pm.dpm.voltage_response_time, + R600_VRU_DFLT)); + + rt = rv6xx_compute_count_for_delay(rdev, + rdev->pm.dpm.backbias_response_time, + R600_VRU_DFLT); + + rv6xx_vid_response_set_brt(rdev, (rt + 0x1F) >> 5); +} + +static void rv6xx_program_engine_speed_parameters(struct radeon_device *rdev) +{ + r600_vid_rt_set_ssu(rdev, R600_SPLLSTEPUNIT_DFLT); + rv6xx_enable_engine_feedback_and_reference_sync(rdev); +} + +static u64 rv6xx_get_master_voltage_mask(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + u64 master_mask = 0; + int i; + + for (i = 0; i < R600_PM_NUMBER_OF_VOLTAGE_LEVELS; i++) { + u32 tmp_mask, tmp_set_pins; + int ret; + + ret = radeon_atom_get_voltage_gpio_settings(rdev, + pi->hw.vddc[i], + SET_VOLTAGE_TYPE_ASIC_VDDC, + &tmp_set_pins, &tmp_mask); + + if (ret == 0) + master_mask |= tmp_mask; + } + + return master_mask; +} + +static void rv6xx_program_voltage_gpio_pins(struct radeon_device *rdev) +{ + r600_voltage_control_enable_pins(rdev, + rv6xx_get_master_voltage_mask(rdev)); +} + +static void rv6xx_enable_static_voltage_control(struct radeon_device *rdev, + struct radeon_ps *new_ps, + bool enable) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + + if (enable) + radeon_atom_set_voltage(rdev, + new_state->low.vddc, + SET_VOLTAGE_TYPE_ASIC_VDDC); + else + r600_voltage_control_deactivate_static_control(rdev, + rv6xx_get_master_voltage_mask(rdev)); +} + +static void rv6xx_enable_display_gap(struct radeon_device *rdev, bool enable) +{ + if (enable) { + u32 tmp = (DISP1_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM) | + DISP2_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM) | + DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) | + DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) | + VBI_TIMER_COUNT(0x3FFF) | + VBI_TIMER_UNIT(7)); + WREG32(CG_DISPLAY_GAP_CNTL, tmp); + + WREG32_P(MCLK_PWRMGT_CNTL, USE_DISPLAY_GAP, ~USE_DISPLAY_GAP); + } else + WREG32_P(MCLK_PWRMGT_CNTL, 0, ~USE_DISPLAY_GAP); +} + +static void rv6xx_program_power_level_enter_state(struct radeon_device *rdev) +{ + r600_power_level_set_enter_index(rdev, R600_POWER_LEVEL_MEDIUM); +} + +static void rv6xx_calculate_t(u32 l_f, u32 h_f, int h, + int d_l, int d_r, u8 *l, u8 *r) +{ + int a_n, a_d, h_r, l_r; + + h_r = d_l; + l_r = 100 - d_r; + + a_n = (int)h_f * d_l + (int)l_f * (h - d_r); + a_d = (int)l_f * l_r + (int)h_f * h_r; + + if (a_d != 0) { + *l = d_l - h_r * a_n / a_d; + *r = d_r + l_r * a_n / a_d; + } +} + +static void rv6xx_calculate_ap(struct radeon_device *rdev, + struct rv6xx_ps *state) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + pi->hw.lp[0] = 0; + pi->hw.rp[R600_PM_NUMBER_OF_ACTIVITY_LEVELS - 1] + = 100; + + rv6xx_calculate_t(state->low.sclk, + state->medium.sclk, + R600_AH_DFLT, + R600_LMP_DFLT, + R600_RLP_DFLT, + &pi->hw.lp[1], + &pi->hw.rp[0]); + + rv6xx_calculate_t(state->medium.sclk, + state->high.sclk, + R600_AH_DFLT, + R600_LHP_DFLT, + R600_RMP_DFLT, + &pi->hw.lp[2], + &pi->hw.rp[1]); + +} + +static void rv6xx_calculate_stepping_parameters(struct radeon_device *rdev, + struct radeon_ps *new_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + + rv6xx_calculate_engine_speed_stepping_parameters(rdev, new_state); + rv6xx_calculate_memory_clock_stepping_parameters(rdev, new_state); + rv6xx_calculate_voltage_stepping_parameters(rdev, new_state); + rv6xx_calculate_ap(rdev, new_state); +} + +static void rv6xx_program_stepping_parameters_except_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + rv6xx_program_mclk_stepping_parameters_except_lowest_entry(rdev); + if (pi->voltage_control) + rv6xx_program_voltage_stepping_parameters_except_lowest_entry(rdev); + rv6xx_program_backbias_stepping_parameters_except_lowest_entry(rdev); + rv6xx_program_sclk_spread_spectrum_parameters_except_lowest_entry(rdev); + rv6xx_program_mclk_spread_spectrum_parameters(rdev); + rv6xx_program_memory_timing_parameters(rdev); +} + +static void rv6xx_program_stepping_parameters_lowest_entry(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + rv6xx_program_mclk_stepping_parameters_lowest_entry(rdev); + if (pi->voltage_control) + rv6xx_program_voltage_stepping_parameters_lowest_entry(rdev); + rv6xx_program_backbias_stepping_parameters_lowest_entry(rdev); + rv6xx_program_sclk_spread_spectrum_parameters_lowest_entry(rdev); +} + +static void rv6xx_program_power_level_low(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW, + pi->hw.low_vddc_index); + r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW, + pi->hw.low_mclk_index); + r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW, + pi->hw.low_sclk_index); + r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW, + R600_DISPLAY_WATERMARK_LOW); + r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_LOW, + pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]); +} + +static void rv6xx_program_power_level_low_to_lowest_state(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW, 0); + r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW, 0); + r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW, 0); + + r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW, + R600_DISPLAY_WATERMARK_LOW); + + r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_LOW, + pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]); + +} + +static void rv6xx_program_power_level_medium(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM, + pi->hw.medium_vddc_index); + r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM, + pi->hw.medium_mclk_index); + r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM, + pi->hw.medium_sclk_index); + r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM, + R600_DISPLAY_WATERMARK_LOW); + r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_MEDIUM, + pi->hw.pcie_gen2[R600_POWER_LEVEL_MEDIUM]); +} + +static void rv6xx_program_power_level_medium_for_transition(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + rv6xx_program_mclk_stepping_entry(rdev, + R600_POWER_LEVEL_CTXSW, + pi->hw.mclks[pi->hw.low_mclk_index]); + + r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM, 1); + + r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM, + R600_POWER_LEVEL_CTXSW); + r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM, + pi->hw.medium_sclk_index); + + r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM, + R600_DISPLAY_WATERMARK_LOW); + + rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_MEDIUM, false); + + r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_MEDIUM, + pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]); +} + +static void rv6xx_program_power_level_high(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_HIGH, + pi->hw.high_vddc_index); + r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_HIGH, + pi->hw.high_mclk_index); + r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_HIGH, + pi->hw.high_sclk_index); + + r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_HIGH, + R600_DISPLAY_WATERMARK_HIGH); + + r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_HIGH, + pi->hw.pcie_gen2[R600_POWER_LEVEL_HIGH]); +} + +static void rv6xx_enable_backbias(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL, + ~(BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL)); + else + WREG32_P(GENERAL_PWRMGT, 0, + ~(BACKBIAS_VALUE | BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL)); +} + +static void rv6xx_program_display_gap(struct radeon_device *rdev) +{ + u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL); + + tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK); + if (rdev->pm.dpm.new_active_crtcs & 1) { + tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK); + tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE); + } else if (rdev->pm.dpm.new_active_crtcs & 2) { + tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE); + tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK); + } else { + tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE); + tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE); + } + WREG32(CG_DISPLAY_GAP_CNTL, tmp); +} + +static void rv6xx_set_sw_voltage_to_safe(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps); + u16 safe_voltage; + + safe_voltage = (new_state->low.vddc >= old_state->low.vddc) ? + new_state->low.vddc : old_state->low.vddc; + + rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW, + safe_voltage); + + WREG32_P(GENERAL_PWRMGT, SW_GPIO_INDEX(R600_POWER_LEVEL_CTXSW), + ~SW_GPIO_INDEX_MASK); +} + +static void rv6xx_set_sw_voltage_to_low(struct radeon_device *rdev, + struct radeon_ps *old_ps) +{ + struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps); + + rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW, + old_state->low.vddc); + + WREG32_P(GENERAL_PWRMGT, SW_GPIO_INDEX(R600_POWER_LEVEL_CTXSW), + ~SW_GPIO_INDEX_MASK); +} + +static void rv6xx_set_safe_backbias(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps); + + if ((new_state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) && + (old_state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE)) + WREG32_P(GENERAL_PWRMGT, BACKBIAS_VALUE, ~BACKBIAS_VALUE); + else + WREG32_P(GENERAL_PWRMGT, 0, ~BACKBIAS_VALUE); +} + +static void rv6xx_set_safe_pcie_gen2(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps); + + if ((new_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) != + (old_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)) + rv6xx_force_pcie_gen1(rdev); +} + +static void rv6xx_enable_dynamic_voltage_control(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN); + else + WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN); +} + +static void rv6xx_enable_dynamic_backbias_control(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, BACKBIAS_DPM_CNTL, ~BACKBIAS_DPM_CNTL); + else + WREG32_P(GENERAL_PWRMGT, 0, ~BACKBIAS_DPM_CNTL); +} + +static int rv6xx_step_sw_voltage(struct radeon_device *rdev, + u16 initial_voltage, + u16 target_voltage) +{ + u16 current_voltage; + u16 true_target_voltage; + u16 voltage_step; + int signed_voltage_step; + + if ((radeon_atom_get_voltage_step(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, + &voltage_step)) || + (radeon_atom_round_to_true_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, + initial_voltage, ¤t_voltage)) || + (radeon_atom_round_to_true_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, + target_voltage, &true_target_voltage))) + return -EINVAL; + + if (true_target_voltage < current_voltage) + signed_voltage_step = -(int)voltage_step; + else + signed_voltage_step = voltage_step; + + while (current_voltage != true_target_voltage) { + current_voltage += signed_voltage_step; + rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW, + current_voltage); + drm_msleep((rdev->pm.dpm.voltage_response_time + 999) / 1000); + } + + return 0; +} + +static int rv6xx_step_voltage_if_increasing(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps); + + if (new_state->low.vddc > old_state->low.vddc) + return rv6xx_step_sw_voltage(rdev, + old_state->low.vddc, + new_state->low.vddc); + + return 0; +} + +static int rv6xx_step_voltage_if_decreasing(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps); + + if (new_state->low.vddc < old_state->low.vddc) + return rv6xx_step_sw_voltage(rdev, + old_state->low.vddc, + new_state->low.vddc); + else + return 0; +} + +static void rv6xx_enable_high(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + if ((pi->restricted_levels < 1) || + (pi->restricted_levels == 3)) + r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, true); +} + +static void rv6xx_enable_medium(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + if (pi->restricted_levels < 2) + r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true); +} + +static void rv6xx_set_dpm_event_sources(struct radeon_device *rdev, u32 sources) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + bool want_thermal_protection; + enum radeon_dpm_event_src dpm_event_src; + + switch (sources) { + case 0: + default: + want_thermal_protection = false; + break; + case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL; + break; + + case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL; + break; + + case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) | + (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL; + break; + } + + if (want_thermal_protection) { + WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK); + if (pi->thermal_protection) + WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS); + } else { + WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS); + } +} + +static void rv6xx_enable_auto_throttle_source(struct radeon_device *rdev, + enum radeon_dpm_auto_throttle_src source, + bool enable) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + if (enable) { + if (!(pi->active_auto_throttle_sources & (1 << source))) { + pi->active_auto_throttle_sources |= 1 << source; + rv6xx_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources); + } + } else { + if (pi->active_auto_throttle_sources & (1 << source)) { + pi->active_auto_throttle_sources &= ~(1 << source); + rv6xx_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources); + } + } +} + + +static void rv6xx_enable_thermal_protection(struct radeon_device *rdev, + bool enable) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + if (pi->active_auto_throttle_sources) + r600_enable_thermal_protection(rdev, enable); +} + +static void rv6xx_generate_transition_stepping(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps); + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + rv6xx_generate_steps(rdev, + old_state->low.sclk, + new_state->low.sclk, + 0, &pi->hw.medium_sclk_index); +} + +static void rv6xx_generate_low_step(struct radeon_device *rdev, + struct radeon_ps *new_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + pi->hw.low_sclk_index = 0; + rv6xx_generate_single_step(rdev, + new_state->low.sclk, + 0); +} + +static void rv6xx_invalidate_intermediate_steps(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + rv6xx_invalidate_intermediate_steps_range(rdev, 0, + pi->hw.medium_sclk_index); +} + +static void rv6xx_generate_stepping_table(struct radeon_device *rdev, + struct radeon_ps *new_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + pi->hw.low_sclk_index = 0; + + rv6xx_generate_steps(rdev, + new_state->low.sclk, + new_state->medium.sclk, + 0, + &pi->hw.medium_sclk_index); + rv6xx_generate_steps(rdev, + new_state->medium.sclk, + new_state->high.sclk, + pi->hw.medium_sclk_index, + &pi->hw.high_sclk_index); +} + +static void rv6xx_enable_spread_spectrum(struct radeon_device *rdev, + bool enable) +{ + if (enable) + rv6xx_enable_dynamic_spread_spectrum(rdev, true); + else { + rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_LOW, false); + rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_MEDIUM, false); + rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_HIGH, false); + rv6xx_enable_dynamic_spread_spectrum(rdev, false); + rv6xx_enable_memory_spread_spectrum(rdev, false); + } +} + +static void rv6xx_reset_lvtm_data_sync(struct radeon_device *rdev) +{ + if (ASIC_IS_DCE3(rdev)) + WREG32_P(DCE3_LVTMA_DATA_SYNCHRONIZATION, LVTMA_PFREQCHG, ~LVTMA_PFREQCHG); + else + WREG32_P(LVTMA_DATA_SYNCHRONIZATION, LVTMA_PFREQCHG, ~LVTMA_PFREQCHG); +} + +static void rv6xx_enable_dynamic_pcie_gen2(struct radeon_device *rdev, + struct radeon_ps *new_ps, + bool enable) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + + if (enable) { + rv6xx_enable_bif_dynamic_pcie_gen2(rdev, true); + rv6xx_enable_pcie_gen2_support(rdev); + r600_enable_dynamic_pcie_gen2(rdev, true); + } else { + if (!(new_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)) + rv6xx_force_pcie_gen1(rdev); + rv6xx_enable_bif_dynamic_pcie_gen2(rdev, false); + r600_enable_dynamic_pcie_gen2(rdev, false); + } +} + +static void rv6xx_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_ps *current_state = rv6xx_get_ps(old_ps); + + if ((new_ps->vclk == old_ps->vclk) && + (new_ps->dclk == old_ps->dclk)) + return; + + if (new_state->high.sclk >= current_state->high.sclk) + return; + + radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk); +} + +static void rv6xx_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps); + struct rv6xx_ps *current_state = rv6xx_get_ps(old_ps); + + if ((new_ps->vclk == old_ps->vclk) && + (new_ps->dclk == old_ps->dclk)) + return; + + if (new_state->high.sclk < current_state->high.sclk) + return; + + radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk); +} + +int rv6xx_dpm_enable(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + + if (r600_dynamicpm_enabled(rdev)) + return -EINVAL; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS) + rv6xx_enable_backbias(rdev, true); + + if (pi->dynamic_ss) + rv6xx_enable_spread_spectrum(rdev, true); + + rv6xx_program_mpll_timing_parameters(rdev); + rv6xx_program_bsp(rdev); + rv6xx_program_git(rdev); + rv6xx_program_tp(rdev); + rv6xx_program_tpp(rdev); + rv6xx_program_sstp(rdev); + rv6xx_program_fcp(rdev); + rv6xx_program_vddc3d_parameters(rdev); + rv6xx_program_voltage_timing_parameters(rdev); + rv6xx_program_engine_speed_parameters(rdev); + + rv6xx_enable_display_gap(rdev, true); + if (pi->display_gap == false) + rv6xx_enable_display_gap(rdev, false); + + rv6xx_program_power_level_enter_state(rdev); + + rv6xx_calculate_stepping_parameters(rdev, boot_ps); + + if (pi->voltage_control) + rv6xx_program_voltage_gpio_pins(rdev); + + rv6xx_generate_stepping_table(rdev, boot_ps); + + rv6xx_program_stepping_parameters_except_lowest_entry(rdev); + rv6xx_program_stepping_parameters_lowest_entry(rdev); + + rv6xx_program_power_level_low(rdev); + rv6xx_program_power_level_medium(rdev); + rv6xx_program_power_level_high(rdev); + rv6xx_program_vc(rdev); + rv6xx_program_at(rdev); + + r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true); + r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true); + r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, true); + + rv6xx_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + + r600_start_dpm(rdev); + + if (pi->voltage_control) + rv6xx_enable_static_voltage_control(rdev, boot_ps, false); + + if (pi->dynamic_pcie_gen2) + rv6xx_enable_dynamic_pcie_gen2(rdev, boot_ps, true); + + if (pi->gfx_clock_gating) + r600_gfx_clockgating_enable(rdev, true); + + return 0; +} + +void rv6xx_dpm_disable(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + + if (!r600_dynamicpm_enabled(rdev)) + return; + + r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true); + r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true); + rv6xx_enable_display_gap(rdev, false); + rv6xx_clear_vc(rdev); + r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF); + + if (pi->thermal_protection) + r600_enable_thermal_protection(rdev, false); + + r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW); + r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false); + r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false); + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS) + rv6xx_enable_backbias(rdev, false); + + rv6xx_enable_spread_spectrum(rdev, false); + + if (pi->voltage_control) + rv6xx_enable_static_voltage_control(rdev, boot_ps, true); + + if (pi->dynamic_pcie_gen2) + rv6xx_enable_dynamic_pcie_gen2(rdev, boot_ps, false); + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + rdev->irq.dpm_thermal = false; + radeon_irq_set(rdev); + } + + if (pi->gfx_clock_gating) + r600_gfx_clockgating_enable(rdev, false); + + r600_stop_dpm(rdev); +} + +int rv6xx_dpm_set_power_state(struct radeon_device *rdev) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps; + struct radeon_ps *old_ps = rdev->pm.dpm.current_ps; + int ret; + + pi->restricted_levels = 0; + + rv6xx_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps); + + rv6xx_clear_vc(rdev); + r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true); + r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF); + + if (pi->thermal_protection) + r600_enable_thermal_protection(rdev, false); + + r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW); + r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false); + r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false); + + rv6xx_generate_transition_stepping(rdev, new_ps, old_ps); + rv6xx_program_power_level_medium_for_transition(rdev); + + if (pi->voltage_control) { + rv6xx_set_sw_voltage_to_safe(rdev, new_ps, old_ps); + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) + rv6xx_set_sw_voltage_to_low(rdev, old_ps); + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS) + rv6xx_set_safe_backbias(rdev, new_ps, old_ps); + + if (pi->dynamic_pcie_gen2) + rv6xx_set_safe_pcie_gen2(rdev, new_ps, old_ps); + + if (pi->voltage_control) + rv6xx_enable_dynamic_voltage_control(rdev, false); + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS) + rv6xx_enable_dynamic_backbias_control(rdev, false); + + if (pi->voltage_control) { + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) + rv6xx_step_voltage_if_increasing(rdev, new_ps, old_ps); + drm_msleep((rdev->pm.dpm.voltage_response_time + 999) / 1000); + } + + r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true); + r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, false); + r600_wait_for_power_level_unequal(rdev, R600_POWER_LEVEL_LOW); + + rv6xx_generate_low_step(rdev, new_ps); + rv6xx_invalidate_intermediate_steps(rdev); + rv6xx_calculate_stepping_parameters(rdev, new_ps); + rv6xx_program_stepping_parameters_lowest_entry(rdev); + rv6xx_program_power_level_low_to_lowest_state(rdev); + + r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true); + r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW); + r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false); + + if (pi->voltage_control) { + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) { + ret = rv6xx_step_voltage_if_decreasing(rdev, new_ps, old_ps); + if (ret) + return ret; + } + rv6xx_enable_dynamic_voltage_control(rdev, true); + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS) + rv6xx_enable_dynamic_backbias_control(rdev, true); + + if (pi->dynamic_pcie_gen2) + rv6xx_enable_dynamic_pcie_gen2(rdev, new_ps, true); + + rv6xx_reset_lvtm_data_sync(rdev); + + rv6xx_generate_stepping_table(rdev, new_ps); + rv6xx_program_stepping_parameters_except_lowest_entry(rdev); + rv6xx_program_power_level_low(rdev); + rv6xx_program_power_level_medium(rdev); + rv6xx_program_power_level_high(rdev); + rv6xx_enable_medium(rdev); + rv6xx_enable_high(rdev); + + if (pi->thermal_protection) + rv6xx_enable_thermal_protection(rdev, true); + rv6xx_program_vc(rdev); + rv6xx_program_at(rdev); + + rv6xx_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps); + + return 0; +} + +void rv6xx_setup_asic(struct radeon_device *rdev) +{ + r600_enable_acpi_pm(rdev); + + if (radeon_aspm != 0) { + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s) + rv6xx_enable_l0s(rdev); + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1) + rv6xx_enable_l1(rdev); + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1) + rv6xx_enable_pll_sleep_in_l1(rdev); + } +} + +void rv6xx_dpm_display_configuration_changed(struct radeon_device *rdev) +{ + rv6xx_program_display_gap(rdev); +} + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; +}; + +union pplib_clock_info { + struct _ATOM_PPLIB_R600_CLOCK_INFO r600; + struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; + struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; + struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; +}; + +union pplib_power_state { + struct _ATOM_PPLIB_STATE v1; + struct _ATOM_PPLIB_STATE_V2 v2; +}; + +static void rv6xx_parse_pplib_non_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info) +{ + rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); + rps->class = le16_to_cpu(non_clock_info->usClassification); + rps->class2 = le16_to_cpu(non_clock_info->usClassification2); + + if (r600_is_uvd_state(rps->class, rps->class2)) { + rps->vclk = RV6XX_DEFAULT_VCLK_FREQ; + rps->dclk = RV6XX_DEFAULT_DCLK_FREQ; + } else { + rps->vclk = 0; + rps->dclk = 0; + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) + rdev->pm.dpm.boot_ps = rps; + if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + rdev->pm.dpm.uvd_ps = rps; +} + +static void rv6xx_parse_pplib_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, int index, + union pplib_clock_info *clock_info) +{ + struct rv6xx_ps *ps = rv6xx_get_ps(rps); + u32 sclk, mclk; + u16 vddc; + struct rv6xx_pl *pl; + + switch (index) { + case 0: + pl = &ps->low; + break; + case 1: + pl = &ps->medium; + break; + case 2: + default: + pl = &ps->high; + break; + } + + sclk = le16_to_cpu(clock_info->r600.usEngineClockLow); + sclk |= clock_info->r600.ucEngineClockHigh << 16; + mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow); + mclk |= clock_info->r600.ucMemoryClockHigh << 16; + + pl->mclk = mclk; + pl->sclk = sclk; + pl->vddc = le16_to_cpu(clock_info->r600.usVDDC); + pl->flags = le32_to_cpu(clock_info->r600.ulFlags); + + /* patch up vddc if necessary */ + if (pl->vddc == 0xff01) { + if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0) + pl->vddc = vddc; + } + + /* fix up pcie gen2 */ + if (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) { + if ((rdev->family == CHIP_RV610) || (rdev->family == CHIP_RV630)) { + if (pl->vddc < 1100) + pl->flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2; + } + } + + /* patch up boot state */ + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { + u16 vddc, vddci, mvdd; + radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd); + pl->mclk = rdev->clock.default_mclk; + pl->sclk = rdev->clock.default_sclk; + pl->vddc = vddc; + } +} + +static int rv6xx_parse_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; + union pplib_power_state *power_state; + int i, j; + union pplib_clock_info *clock_info; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + struct rv6xx_ps *ps; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) * + power_info->pplib.ucNumStates, GFP_KERNEL); + if (!rdev->pm.dpm.ps) + return -ENOMEM; + + for (i = 0; i < power_info->pplib.ucNumStates; i++) { + power_state = (union pplib_power_state *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usStateArrayOffset) + + i * power_info->pplib.ucStateEntrySize); + non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) + + (power_state->v1.ucNonClockStateIndex * + power_info->pplib.ucNonClockSize)); + if (power_info->pplib.ucStateEntrySize - 1) { + u8 *idx; + ps = kzalloc(sizeof(struct rv6xx_ps), GFP_KERNEL); + if (ps == NULL) { + kfree(rdev->pm.dpm.ps); + return -ENOMEM; + } + rdev->pm.dpm.ps[i].ps_priv = ps; + rv6xx_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i], + non_clock_info); + idx = (u8 *)&power_state->v1.ucClockStateIndices[0]; + for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) { + clock_info = (union pplib_clock_info *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) + + (idx[j] * power_info->pplib.ucClockInfoSize)); + rv6xx_parse_pplib_clock_info(rdev, + &rdev->pm.dpm.ps[i], j, + clock_info); + } + } + } + rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates; + return 0; +} + +int rv6xx_dpm_init(struct radeon_device *rdev) +{ + struct radeon_atom_ss ss; + struct atom_clock_dividers dividers; + struct rv6xx_power_info *pi; + int ret; + + pi = kzalloc(sizeof(struct rv6xx_power_info), GFP_KERNEL); + if (pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = pi; + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = rv6xx_parse_power_table(rdev); + if (ret) + return ret; + + if (rdev->pm.dpm.voltage_response_time == 0) + rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT; + if (rdev->pm.dpm.backbias_response_time == 0) + rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + 0, false, ÷rs); + if (ret) + pi->spll_ref_div = dividers.ref_div + 1; + else + pi->spll_ref_div = R600_REFERENCEDIVIDER_DFLT; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, + 0, false, ÷rs); + if (ret) + pi->mpll_ref_div = dividers.ref_div + 1; + else + pi->mpll_ref_div = R600_REFERENCEDIVIDER_DFLT; + + if (rdev->family >= CHIP_RV670) + pi->fb_div_scale = 1; + else + pi->fb_div_scale = 0; + + pi->voltage_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0); + + pi->gfx_clock_gating = true; + + pi->sclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_ENGINE_SS, 0); + pi->mclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_MEMORY_SS, 0); + + /* Disable sclk ss, causes hangs on a lot of systems */ + pi->sclk_ss = false; + + if (pi->sclk_ss || pi->mclk_ss) + pi->dynamic_ss = true; + else + pi->dynamic_ss = false; + + pi->dynamic_pcie_gen2 = true; + + if (pi->gfx_clock_gating && + (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)) + pi->thermal_protection = true; + else + pi->thermal_protection = false; + + pi->display_gap = true; + + return 0; +} + +void rv6xx_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct rv6xx_ps *ps = rv6xx_get_ps(rps); + struct rv6xx_pl *pl; + + r600_dpm_print_class_info(rps->class, rps->class2); + r600_dpm_print_cap_info(rps->caps); + printk("\tuvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + pl = &ps->low; + printk("\t\tpower level 0 sclk: %u mclk: %u vddc: %u\n", + pl->sclk, pl->mclk, pl->vddc); + pl = &ps->medium; + printk("\t\tpower level 1 sclk: %u mclk: %u vddc: %u\n", + pl->sclk, pl->mclk, pl->vddc); + pl = &ps->high; + printk("\t\tpower level 2 sclk: %u mclk: %u vddc: %u\n", + pl->sclk, pl->mclk, pl->vddc); + r600_dpm_print_ps_status(rdev, rps); +} + +void rv6xx_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct radeon_ps *rps = rdev->pm.dpm.current_ps; + struct rv6xx_ps *ps = rv6xx_get_ps(rps); + struct rv6xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >> + CURRENT_PROFILE_INDEX_SHIFT; + + if (current_index > 2) { + seq_printf(m, "invalid dpm profile %d\n", current_index); + } else { + if (current_index == 0) + pl = &ps->low; + else if (current_index == 1) + pl = &ps->medium; + else /* current_index == 2 */ + pl = &ps->high; + seq_printf(m, "uvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + seq_printf(m, "power level %d sclk: %u mclk: %u vddc: %u\n", + current_index, pl->sclk, pl->mclk, pl->vddc); + } +} + +/* get the current sclk in 10 khz units */ +u32 rv6xx_dpm_get_current_sclk(struct radeon_device *rdev) +{ + struct radeon_ps *rps = rdev->pm.dpm.current_ps; + struct rv6xx_ps *ps = rv6xx_get_ps(rps); + struct rv6xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >> + CURRENT_PROFILE_INDEX_SHIFT; + + if (current_index > 2) { + return 0; + } else { + if (current_index == 0) + pl = &ps->low; + else if (current_index == 1) + pl = &ps->medium; + else /* current_index == 2 */ + pl = &ps->high; + return pl->sclk; + } +} + +/* get the current mclk in 10 khz units */ +u32 rv6xx_dpm_get_current_mclk(struct radeon_device *rdev) +{ + struct radeon_ps *rps = rdev->pm.dpm.current_ps; + struct rv6xx_ps *ps = rv6xx_get_ps(rps); + struct rv6xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >> + CURRENT_PROFILE_INDEX_SHIFT; + + if (current_index > 2) { + return 0; + } else { + if (current_index == 0) + pl = &ps->low; + else if (current_index == 1) + pl = &ps->medium; + else /* current_index == 2 */ + pl = &ps->high; + return pl->mclk; + } +} + +void rv6xx_dpm_fini(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); +} + +u32 rv6xx_dpm_get_sclk(struct radeon_device *rdev, bool low) +{ + struct rv6xx_ps *requested_state = rv6xx_get_ps(rdev->pm.dpm.requested_ps); + + if (low) + return requested_state->low.sclk; + else + return requested_state->high.sclk; +} + +u32 rv6xx_dpm_get_mclk(struct radeon_device *rdev, bool low) +{ + struct rv6xx_ps *requested_state = rv6xx_get_ps(rdev->pm.dpm.requested_ps); + + if (low) + return requested_state->low.mclk; + else + return requested_state->high.mclk; +} + +int rv6xx_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level) +{ + struct rv6xx_power_info *pi = rv6xx_get_pi(rdev); + + if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + pi->restricted_levels = 3; + } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { + pi->restricted_levels = 2; + } else { + pi->restricted_levels = 0; + } + + rv6xx_clear_vc(rdev); + r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true); + r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF); + r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW); + r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false); + r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false); + rv6xx_enable_medium(rdev); + rv6xx_enable_high(rdev); + if (pi->restricted_levels == 3) + r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, false); + rv6xx_program_vc(rdev); + rv6xx_program_at(rdev); + + rdev->pm.dpm.forced_level = level; + + return 0; +} diff --git a/sys/dev/pci/drm/radeon/rv6xx_dpm.h b/sys/dev/pci/drm/radeon/rv6xx_dpm.h new file mode 100644 index 00000000000..8035d53ebea --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv6xx_dpm.h @@ -0,0 +1,95 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#ifndef __RV6XX_DPM_H__ +#define __RV6XX_DPM_H__ + +#include "r600_dpm.h" + +/* Represents a single SCLK step. */ +struct rv6xx_sclk_stepping +{ + u32 vco_frequency; + u32 post_divider; +}; + +struct rv6xx_pm_hw_state { + u32 sclks[R600_PM_NUMBER_OF_ACTIVITY_LEVELS]; + u32 mclks[R600_PM_NUMBER_OF_MCLKS]; + u16 vddc[R600_PM_NUMBER_OF_VOLTAGE_LEVELS]; + bool backbias[R600_PM_NUMBER_OF_VOLTAGE_LEVELS]; + bool pcie_gen2[R600_PM_NUMBER_OF_ACTIVITY_LEVELS]; + u8 high_sclk_index; + u8 medium_sclk_index; + u8 low_sclk_index; + u8 high_mclk_index; + u8 medium_mclk_index; + u8 low_mclk_index; + u8 high_vddc_index; + u8 medium_vddc_index; + u8 low_vddc_index; + u8 rp[R600_PM_NUMBER_OF_ACTIVITY_LEVELS]; + u8 lp[R600_PM_NUMBER_OF_ACTIVITY_LEVELS]; +}; + +struct rv6xx_power_info { + /* flags */ + bool voltage_control; + bool sclk_ss; + bool mclk_ss; + bool dynamic_ss; + bool dynamic_pcie_gen2; + bool thermal_protection; + bool display_gap; + bool gfx_clock_gating; + /* clk values */ + u32 fb_div_scale; + u32 spll_ref_div; + u32 mpll_ref_div; + u32 bsu; + u32 bsp; + /* */ + u32 active_auto_throttle_sources; + /* current power state */ + u32 restricted_levels; + struct rv6xx_pm_hw_state hw; +}; + +struct rv6xx_pl { + u32 sclk; + u32 mclk; + u16 vddc; + u32 flags; +}; + +struct rv6xx_ps { + struct rv6xx_pl high; + struct rv6xx_pl medium; + struct rv6xx_pl low; +}; + +#define RV6XX_DEFAULT_VCLK_FREQ 40000 /* 10 khz */ +#define RV6XX_DEFAULT_DCLK_FREQ 30000 /* 10 khz */ + +#endif diff --git a/sys/dev/pci/drm/radeon/rv6xxd.h b/sys/dev/pci/drm/radeon/rv6xxd.h new file mode 100644 index 00000000000..34e86f90b43 --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv6xxd.h @@ -0,0 +1,246 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef RV6XXD_H +#define RV6XXD_H + +/* RV6xx power management */ +#define SPLL_CNTL_MODE 0x60c +# define SPLL_DIV_SYNC (1 << 5) + +#define GENERAL_PWRMGT 0x618 +# define GLOBAL_PWRMGT_EN (1 << 0) +# define STATIC_PM_EN (1 << 1) +# define MOBILE_SU (1 << 2) +# define THERMAL_PROTECTION_DIS (1 << 3) +# define THERMAL_PROTECTION_TYPE (1 << 4) +# define ENABLE_GEN2PCIE (1 << 5) +# define SW_GPIO_INDEX(x) ((x) << 6) +# define SW_GPIO_INDEX_MASK (3 << 6) +# define LOW_VOLT_D2_ACPI (1 << 8) +# define LOW_VOLT_D3_ACPI (1 << 9) +# define VOLT_PWRMGT_EN (1 << 10) +# define BACKBIAS_PAD_EN (1 << 16) +# define BACKBIAS_VALUE (1 << 17) +# define BACKBIAS_DPM_CNTL (1 << 18) +# define DYN_SPREAD_SPECTRUM_EN (1 << 21) + +#define MCLK_PWRMGT_CNTL 0x624 +# define MPLL_PWRMGT_OFF (1 << 0) +# define YCLK_TURNOFF (1 << 1) +# define MPLL_TURNOFF (1 << 2) +# define SU_MCLK_USE_BCLK (1 << 3) +# define DLL_READY (1 << 4) +# define MC_BUSY (1 << 5) +# define MC_INT_CNTL (1 << 7) +# define MRDCKA_SLEEP (1 << 8) +# define MRDCKB_SLEEP (1 << 9) +# define MRDCKC_SLEEP (1 << 10) +# define MRDCKD_SLEEP (1 << 11) +# define MRDCKE_SLEEP (1 << 12) +# define MRDCKF_SLEEP (1 << 13) +# define MRDCKG_SLEEP (1 << 14) +# define MRDCKH_SLEEP (1 << 15) +# define MRDCKA_RESET (1 << 16) +# define MRDCKB_RESET (1 << 17) +# define MRDCKC_RESET (1 << 18) +# define MRDCKD_RESET (1 << 19) +# define MRDCKE_RESET (1 << 20) +# define MRDCKF_RESET (1 << 21) +# define MRDCKG_RESET (1 << 22) +# define MRDCKH_RESET (1 << 23) +# define DLL_READY_READ (1 << 24) +# define USE_DISPLAY_GAP (1 << 25) +# define USE_DISPLAY_URGENT_NORMAL (1 << 26) +# define USE_DISPLAY_GAP_CTXSW (1 << 27) +# define MPLL_TURNOFF_D2 (1 << 28) +# define USE_DISPLAY_URGENT_CTXSW (1 << 29) + +#define MPLL_FREQ_LEVEL_0 0x6e8 +# define LEVEL0_MPLL_POST_DIV(x) ((x) << 0) +# define LEVEL0_MPLL_POST_DIV_MASK (0xff << 0) +# define LEVEL0_MPLL_FB_DIV(x) ((x) << 8) +# define LEVEL0_MPLL_FB_DIV_MASK (0xfff << 8) +# define LEVEL0_MPLL_REF_DIV(x) ((x) << 20) +# define LEVEL0_MPLL_REF_DIV_MASK (0x3f << 20) +# define LEVEL0_MPLL_DIV_EN (1 << 28) +# define LEVEL0_DLL_BYPASS (1 << 29) +# define LEVEL0_DLL_RESET (1 << 30) + +#define VID_RT 0x6f8 +# define VID_CRT(x) ((x) << 0) +# define VID_CRT_MASK (0x1fff << 0) +# define VID_CRTU(x) ((x) << 13) +# define VID_CRTU_MASK (7 << 13) +# define SSTU(x) ((x) << 16) +# define SSTU_MASK (7 << 16) +# define VID_SWT(x) ((x) << 19) +# define VID_SWT_MASK (0x1f << 19) +# define BRT(x) ((x) << 24) +# define BRT_MASK (0xff << 24) + +#define TARGET_AND_CURRENT_PROFILE_INDEX 0x70c +# define TARGET_PROFILE_INDEX_MASK (3 << 0) +# define TARGET_PROFILE_INDEX_SHIFT 0 +# define CURRENT_PROFILE_INDEX_MASK (3 << 2) +# define CURRENT_PROFILE_INDEX_SHIFT 2 +# define DYN_PWR_ENTER_INDEX(x) ((x) << 4) +# define DYN_PWR_ENTER_INDEX_MASK (3 << 4) +# define DYN_PWR_ENTER_INDEX_SHIFT 4 +# define CURR_MCLK_INDEX_MASK (3 << 6) +# define CURR_MCLK_INDEX_SHIFT 6 +# define CURR_SCLK_INDEX_MASK (0x1f << 8) +# define CURR_SCLK_INDEX_SHIFT 8 +# define CURR_VID_INDEX_MASK (3 << 13) +# define CURR_VID_INDEX_SHIFT 13 + +#define VID_UPPER_GPIO_CNTL 0x740 +# define CTXSW_UPPER_GPIO_VALUES(x) ((x) << 0) +# define CTXSW_UPPER_GPIO_VALUES_MASK (7 << 0) +# define HIGH_UPPER_GPIO_VALUES(x) ((x) << 3) +# define HIGH_UPPER_GPIO_VALUES_MASK (7 << 3) +# define MEDIUM_UPPER_GPIO_VALUES(x) ((x) << 6) +# define MEDIUM_UPPER_GPIO_VALUES_MASK (7 << 6) +# define LOW_UPPER_GPIO_VALUES(x) ((x) << 9) +# define LOW_UPPER_GPIO_VALUES_MASK (7 << 9) +# define CTXSW_BACKBIAS_VALUE (1 << 12) +# define HIGH_BACKBIAS_VALUE (1 << 13) +# define MEDIUM_BACKBIAS_VALUE (1 << 14) +# define LOW_BACKBIAS_VALUE (1 << 15) + +#define CG_DISPLAY_GAP_CNTL 0x7dc +# define DISP1_GAP(x) ((x) << 0) +# define DISP1_GAP_MASK (3 << 0) +# define DISP2_GAP(x) ((x) << 2) +# define DISP2_GAP_MASK (3 << 2) +# define VBI_TIMER_COUNT(x) ((x) << 4) +# define VBI_TIMER_COUNT_MASK (0x3fff << 4) +# define VBI_TIMER_UNIT(x) ((x) << 20) +# define VBI_TIMER_UNIT_MASK (7 << 20) +# define DISP1_GAP_MCHG(x) ((x) << 24) +# define DISP1_GAP_MCHG_MASK (3 << 24) +# define DISP2_GAP_MCHG(x) ((x) << 26) +# define DISP2_GAP_MCHG_MASK (3 << 26) + +#define CG_THERMAL_CTRL 0x7f0 +# define DPM_EVENT_SRC(x) ((x) << 0) +# define DPM_EVENT_SRC_MASK (7 << 0) +# define THERM_INC_CLK (1 << 3) +# define TOFFSET(x) ((x) << 4) +# define TOFFSET_MASK (0xff << 4) +# define DIG_THERM_DPM(x) ((x) << 12) +# define DIG_THERM_DPM_MASK (0xff << 12) +# define CTF_SEL(x) ((x) << 20) +# define CTF_SEL_MASK (7 << 20) +# define CTF_PAD_POLARITY (1 << 23) +# define CTF_PAD_EN (1 << 24) + +#define CG_SPLL_SPREAD_SPECTRUM_LOW 0x820 +# define SSEN (1 << 0) +# define CLKS(x) ((x) << 3) +# define CLKS_MASK (0xff << 3) +# define CLKS_SHIFT 3 +# define CLKV(x) ((x) << 11) +# define CLKV_MASK (0x7ff << 11) +# define CLKV_SHIFT 11 +#define CG_MPLL_SPREAD_SPECTRUM 0x830 + +#define CITF_CNTL 0x200c +# define BLACKOUT_RD (1 << 0) +# define BLACKOUT_WR (1 << 1) + +#define RAMCFG 0x2408 +#define NOOFBANK_SHIFT 0 +#define NOOFBANK_MASK 0x00000001 +#define NOOFRANK_SHIFT 1 +#define NOOFRANK_MASK 0x00000002 +#define NOOFROWS_SHIFT 2 +#define NOOFROWS_MASK 0x0000001C +#define NOOFCOLS_SHIFT 5 +#define NOOFCOLS_MASK 0x00000060 +#define CHANSIZE_SHIFT 7 +#define CHANSIZE_MASK 0x00000080 +#define BURSTLENGTH_SHIFT 8 +#define BURSTLENGTH_MASK 0x00000100 +#define CHANSIZE_OVERRIDE (1 << 10) + +#define SQM_RATIO 0x2424 +# define STATE0(x) ((x) << 0) +# define STATE0_MASK (0xff << 0) +# define STATE1(x) ((x) << 8) +# define STATE1_MASK (0xff << 8) +# define STATE2(x) ((x) << 16) +# define STATE2_MASK (0xff << 16) +# define STATE3(x) ((x) << 24) +# define STATE3_MASK (0xff << 24) + +#define ARB_RFSH_CNTL 0x2460 +# define ENABLE (1 << 0) +#define ARB_RFSH_RATE 0x2464 +# define POWERMODE0(x) ((x) << 0) +# define POWERMODE0_MASK (0xff << 0) +# define POWERMODE1(x) ((x) << 8) +# define POWERMODE1_MASK (0xff << 8) +# define POWERMODE2(x) ((x) << 16) +# define POWERMODE2_MASK (0xff << 16) +# define POWERMODE3(x) ((x) << 24) +# define POWERMODE3_MASK (0xff << 24) + +#define MC_SEQ_DRAM 0x2608 +# define CKE_DYN (1 << 12) + +#define MC_SEQ_CMD 0x26c4 + +#define MC_SEQ_RESERVE_S 0x2890 +#define MC_SEQ_RESERVE_M 0x2894 + +#define LVTMA_DATA_SYNCHRONIZATION 0x7adc +# define LVTMA_PFREQCHG (1 << 8) +#define DCE3_LVTMA_DATA_SYNCHRONIZATION 0x7f98 + +/* PCIE indirect regs */ +#define PCIE_P_CNTL 0x40 +# define P_PLL_PWRDN_IN_L1L23 (1 << 3) +# define P_PLL_BUF_PDNB (1 << 4) +# define P_PLL_PDNB (1 << 9) +# define P_ALLOW_PRX_FRONTEND_SHUTOFF (1 << 12) +/* PCIE PORT indirect regs */ +#define PCIE_LC_CNTL 0xa0 +# define LC_L0S_INACTIVITY(x) ((x) << 8) +# define LC_L0S_INACTIVITY_MASK (0xf << 8) +# define LC_L0S_INACTIVITY_SHIFT 8 +# define LC_L1_INACTIVITY(x) ((x) << 12) +# define LC_L1_INACTIVITY_MASK (0xf << 12) +# define LC_L1_INACTIVITY_SHIFT 12 +# define LC_PMI_TO_L1_DIS (1 << 16) +# define LC_ASPM_TO_L1_DIS (1 << 24) +#define PCIE_LC_SPEED_CNTL 0xa4 +# define LC_GEN2_EN (1 << 0) +# define LC_INITIATE_LINK_SPEED_CHANGE (1 << 7) +# define LC_CURRENT_DATA_RATE (1 << 11) +# define LC_HW_VOLTAGE_IF_CONTROL(x) ((x) << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_MASK (3 << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_SHIFT 12 +# define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 23) +# define LC_OTHER_SIDE_SUPPORTS_GEN2 (1 << 24) + +#endif diff --git a/sys/dev/pci/drm/radeon/rv730_dpm.c b/sys/dev/pci/drm/radeon/rv730_dpm.c new file mode 100644 index 00000000000..6c07a0aa300 --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv730_dpm.c @@ -0,0 +1,508 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "rv730d.h" +#include "r600_dpm.h" +#include "rv770_dpm.h" +#include "atom.h" + +#define MC_CG_ARB_FREQ_F0 0x0a +#define MC_CG_ARB_FREQ_F1 0x0b +#define MC_CG_ARB_FREQ_F2 0x0c +#define MC_CG_ARB_FREQ_F3 0x0d + +struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps); +struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev); + +int rv730_populate_sclk_value(struct radeon_device *rdev, + u32 engine_clock, + RV770_SMC_SCLK_VALUE *sclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct atom_clock_dividers dividers; + u32 spll_func_cntl = pi->clk_regs.rv730.cg_spll_func_cntl; + u32 spll_func_cntl_2 = pi->clk_regs.rv730.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = pi->clk_regs.rv730.cg_spll_func_cntl_3; + u32 cg_spll_spread_spectrum = pi->clk_regs.rv730.cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2 = pi->clk_regs.rv730.cg_spll_spread_spectrum_2; + u64 tmp; + u32 reference_clock = rdev->clock.spll.reference_freq; + u32 reference_divider, post_divider; + u32 fbdiv; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + engine_clock, false, ÷rs); + if (ret) + return ret; + + reference_divider = 1 + dividers.ref_div; + + if (dividers.enable_post_div) + post_divider = ((dividers.post_div >> 4) & 0xf) + + (dividers.post_div & 0xf) + 2; + else + post_divider = 1; + + tmp = (u64) engine_clock * reference_divider * post_divider * 16384; + do_div(tmp, reference_clock); + fbdiv = (u32) tmp; + + /* set up registers */ + if (dividers.enable_post_div) + spll_func_cntl |= SPLL_DIVEN; + else + spll_func_cntl &= ~SPLL_DIVEN; + spll_func_cntl &= ~(SPLL_HILEN_MASK | SPLL_LOLEN_MASK | SPLL_REF_DIV_MASK); + spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div); + spll_func_cntl |= SPLL_HILEN((dividers.post_div >> 4) & 0xf); + spll_func_cntl |= SPLL_LOLEN(dividers.post_div & 0xf); + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(2); + + spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK; + spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv); + spll_func_cntl_3 |= SPLL_DITHEN; + + if (pi->sclk_ss) { + struct radeon_atom_ss ss; + u32 vco_freq = engine_clock * post_divider; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_ENGINE_SS, vco_freq)) { + u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate); + u32 clk_v = ss.percentage * fbdiv / (clk_s * 10000); + + cg_spll_spread_spectrum &= ~CLK_S_MASK; + cg_spll_spread_spectrum |= CLK_S(clk_s); + cg_spll_spread_spectrum |= SSEN; + + cg_spll_spread_spectrum_2 &= ~CLK_V_MASK; + cg_spll_spread_spectrum_2 |= CLK_V(clk_v); + } + } + + sclk->sclk_value = cpu_to_be32(engine_clock); + sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl); + sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2); + sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3); + sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(cg_spll_spread_spectrum); + sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(cg_spll_spread_spectrum_2); + + return 0; +} + +int rv730_populate_mclk_value(struct radeon_device *rdev, + u32 engine_clock, u32 memory_clock, + LPRV7XX_SMC_MCLK_VALUE mclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 mclk_pwrmgt_cntl = pi->clk_regs.rv730.mclk_pwrmgt_cntl; + u32 dll_cntl = pi->clk_regs.rv730.dll_cntl; + u32 mpll_func_cntl = pi->clk_regs.rv730.mpll_func_cntl; + u32 mpll_func_cntl_2 = pi->clk_regs.rv730.mpll_func_cntl2; + u32 mpll_func_cntl_3 = pi->clk_regs.rv730.mpll_func_cntl3; + u32 mpll_ss = pi->clk_regs.rv730.mpll_ss; + u32 mpll_ss2 = pi->clk_regs.rv730.mpll_ss2; + struct atom_clock_dividers dividers; + u32 post_divider, reference_divider; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, + memory_clock, false, ÷rs); + if (ret) + return ret; + + reference_divider = dividers.ref_div + 1; + + if (dividers.enable_post_div) + post_divider = ((dividers.post_div >> 4) & 0xf) + + (dividers.post_div & 0xf) + 2; + else + post_divider = 1; + + /* setup the registers */ + if (dividers.enable_post_div) + mpll_func_cntl |= MPLL_DIVEN; + else + mpll_func_cntl &= ~MPLL_DIVEN; + + mpll_func_cntl &= ~(MPLL_REF_DIV_MASK | MPLL_HILEN_MASK | MPLL_LOLEN_MASK); + mpll_func_cntl |= MPLL_REF_DIV(dividers.ref_div); + mpll_func_cntl |= MPLL_HILEN((dividers.post_div >> 4) & 0xf); + mpll_func_cntl |= MPLL_LOLEN(dividers.post_div & 0xf); + + mpll_func_cntl_3 &= ~MPLL_FB_DIV_MASK; + mpll_func_cntl_3 |= MPLL_FB_DIV(dividers.fb_div); + if (dividers.enable_dithen) + mpll_func_cntl_3 |= MPLL_DITHEN; + else + mpll_func_cntl_3 &= ~MPLL_DITHEN; + + if (pi->mclk_ss) { + struct radeon_atom_ss ss; + u32 vco_freq = memory_clock * post_divider; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_MEMORY_SS, vco_freq)) { + u32 reference_clock = rdev->clock.mpll.reference_freq; + u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate); + u32 clk_v = ss.percentage * dividers.fb_div / (clk_s * 10000); + + mpll_ss &= ~CLK_S_MASK; + mpll_ss |= CLK_S(clk_s); + mpll_ss |= SSEN; + + mpll_ss2 &= ~CLK_V_MASK; + mpll_ss |= CLK_V(clk_v); + } + } + + + mclk->mclk730.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + mclk->mclk730.vDLL_CNTL = cpu_to_be32(dll_cntl); + mclk->mclk730.mclk_value = cpu_to_be32(memory_clock); + mclk->mclk730.vMPLL_FUNC_CNTL = cpu_to_be32(mpll_func_cntl); + mclk->mclk730.vMPLL_FUNC_CNTL2 = cpu_to_be32(mpll_func_cntl_2); + mclk->mclk730.vMPLL_FUNC_CNTL3 = cpu_to_be32(mpll_func_cntl_3); + mclk->mclk730.vMPLL_SS = cpu_to_be32(mpll_ss); + mclk->mclk730.vMPLL_SS2 = cpu_to_be32(mpll_ss2); + + return 0; +} + +void rv730_read_clock_registers(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + pi->clk_regs.rv730.cg_spll_func_cntl = + RREG32(CG_SPLL_FUNC_CNTL); + pi->clk_regs.rv730.cg_spll_func_cntl_2 = + RREG32(CG_SPLL_FUNC_CNTL_2); + pi->clk_regs.rv730.cg_spll_func_cntl_3 = + RREG32(CG_SPLL_FUNC_CNTL_3); + pi->clk_regs.rv730.cg_spll_spread_spectrum = + RREG32(CG_SPLL_SPREAD_SPECTRUM); + pi->clk_regs.rv730.cg_spll_spread_spectrum_2 = + RREG32(CG_SPLL_SPREAD_SPECTRUM_2); + + pi->clk_regs.rv730.mclk_pwrmgt_cntl = + RREG32(TCI_MCLK_PWRMGT_CNTL); + pi->clk_regs.rv730.dll_cntl = + RREG32(TCI_DLL_CNTL); + pi->clk_regs.rv730.mpll_func_cntl = + RREG32(CG_MPLL_FUNC_CNTL); + pi->clk_regs.rv730.mpll_func_cntl2 = + RREG32(CG_MPLL_FUNC_CNTL_2); + pi->clk_regs.rv730.mpll_func_cntl3 = + RREG32(CG_MPLL_FUNC_CNTL_3); + pi->clk_regs.rv730.mpll_ss = + RREG32(CG_TCI_MPLL_SPREAD_SPECTRUM); + pi->clk_regs.rv730.mpll_ss2 = + RREG32(CG_TCI_MPLL_SPREAD_SPECTRUM_2); +} + +int rv730_populate_smc_acpi_state(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 mpll_func_cntl = 0; + u32 mpll_func_cntl_2 = 0 ; + u32 mpll_func_cntl_3 = 0; + u32 mclk_pwrmgt_cntl; + u32 dll_cntl; + u32 spll_func_cntl; + u32 spll_func_cntl_2; + u32 spll_func_cntl_3; + + table->ACPIState = table->initialState; + table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC; + + if (pi->acpi_vddc) { + rv770_populate_vddc_value(rdev, pi->acpi_vddc, + &table->ACPIState.levels[0].vddc); + table->ACPIState.levels[0].gen2PCIE = pi->pcie_gen2 ? + pi->acpi_pcie_gen2 : 0; + table->ACPIState.levels[0].gen2XSP = + pi->acpi_pcie_gen2; + } else { + rv770_populate_vddc_value(rdev, pi->min_vddc_in_table, + &table->ACPIState.levels[0].vddc); + table->ACPIState.levels[0].gen2PCIE = 0; + } + + mpll_func_cntl = pi->clk_regs.rv730.mpll_func_cntl; + mpll_func_cntl_2 = pi->clk_regs.rv730.mpll_func_cntl2; + mpll_func_cntl_3 = pi->clk_regs.rv730.mpll_func_cntl3; + + mpll_func_cntl |= MPLL_RESET | MPLL_BYPASS_EN; + mpll_func_cntl &= ~MPLL_SLEEP; + + mpll_func_cntl_2 &= ~MCLK_MUX_SEL_MASK; + mpll_func_cntl_2 |= MCLK_MUX_SEL(1); + + mclk_pwrmgt_cntl = (MRDCKA_RESET | + MRDCKB_RESET | + MRDCKC_RESET | + MRDCKD_RESET | + MRDCKE_RESET | + MRDCKF_RESET | + MRDCKG_RESET | + MRDCKH_RESET | + MRDCKA_SLEEP | + MRDCKB_SLEEP | + MRDCKC_SLEEP | + MRDCKD_SLEEP | + MRDCKE_SLEEP | + MRDCKF_SLEEP | + MRDCKG_SLEEP | + MRDCKH_SLEEP); + + dll_cntl = 0xff000000; + + spll_func_cntl = pi->clk_regs.rv730.cg_spll_func_cntl; + spll_func_cntl_2 = pi->clk_regs.rv730.cg_spll_func_cntl_2; + spll_func_cntl_3 = pi->clk_regs.rv730.cg_spll_func_cntl_3; + + spll_func_cntl |= SPLL_RESET | SPLL_BYPASS_EN; + spll_func_cntl &= ~SPLL_SLEEP; + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(4); + + table->ACPIState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL = cpu_to_be32(mpll_func_cntl); + table->ACPIState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL2 = cpu_to_be32(mpll_func_cntl_2); + table->ACPIState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL3 = cpu_to_be32(mpll_func_cntl_3); + table->ACPIState.levels[0].mclk.mclk730.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + table->ACPIState.levels[0].mclk.mclk730.vDLL_CNTL = cpu_to_be32(dll_cntl); + + table->ACPIState.levels[0].mclk.mclk730.mclk_value = 0; + + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3); + + table->ACPIState.levels[0].sclk.sclk_value = 0; + + rv770_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd); + + table->ACPIState.levels[1] = table->ACPIState.levels[0]; + table->ACPIState.levels[2] = table->ACPIState.levels[0]; + + return 0; +} + +int rv730_populate_smc_initial_state(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_ps *initial_state = rv770_get_ps(radeon_state); + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 a_t; + + table->initialState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL = + cpu_to_be32(pi->clk_regs.rv730.mpll_func_cntl); + table->initialState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL2 = + cpu_to_be32(pi->clk_regs.rv730.mpll_func_cntl2); + table->initialState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL3 = + cpu_to_be32(pi->clk_regs.rv730.mpll_func_cntl3); + table->initialState.levels[0].mclk.mclk730.vMCLK_PWRMGT_CNTL = + cpu_to_be32(pi->clk_regs.rv730.mclk_pwrmgt_cntl); + table->initialState.levels[0].mclk.mclk730.vDLL_CNTL = + cpu_to_be32(pi->clk_regs.rv730.dll_cntl); + table->initialState.levels[0].mclk.mclk730.vMPLL_SS = + cpu_to_be32(pi->clk_regs.rv730.mpll_ss); + table->initialState.levels[0].mclk.mclk730.vMPLL_SS2 = + cpu_to_be32(pi->clk_regs.rv730.mpll_ss2); + + table->initialState.levels[0].mclk.mclk730.mclk_value = + cpu_to_be32(initial_state->low.mclk); + + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = + cpu_to_be32(pi->clk_regs.rv730.cg_spll_func_cntl); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = + cpu_to_be32(pi->clk_regs.rv730.cg_spll_func_cntl_2); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = + cpu_to_be32(pi->clk_regs.rv730.cg_spll_func_cntl_3); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM = + cpu_to_be32(pi->clk_regs.rv730.cg_spll_spread_spectrum); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 = + cpu_to_be32(pi->clk_regs.rv730.cg_spll_spread_spectrum_2); + + table->initialState.levels[0].sclk.sclk_value = + cpu_to_be32(initial_state->low.sclk); + + table->initialState.levels[0].arbValue = MC_CG_ARB_FREQ_F0; + + table->initialState.levels[0].seqValue = + rv770_get_seq_value(rdev, &initial_state->low); + + rv770_populate_vddc_value(rdev, + initial_state->low.vddc, + &table->initialState.levels[0].vddc); + rv770_populate_initial_mvdd_value(rdev, + &table->initialState.levels[0].mvdd); + + a_t = CG_R(0xffff) | CG_L(0); + + table->initialState.levels[0].aT = cpu_to_be32(a_t); + + table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp); + + if (pi->boot_in_gen2) + table->initialState.levels[0].gen2PCIE = 1; + else + table->initialState.levels[0].gen2PCIE = 0; + if (initial_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) + table->initialState.levels[0].gen2XSP = 1; + else + table->initialState.levels[0].gen2XSP = 0; + + table->initialState.levels[1] = table->initialState.levels[0]; + table->initialState.levels[2] = table->initialState.levels[0]; + + table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC; + + return 0; +} + +void rv730_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct rv7xx_ps *state = rv770_get_ps(radeon_state); + u32 arb_refresh_rate = 0; + u32 dram_timing = 0; + u32 dram_timing2 = 0; + u32 old_dram_timing = 0; + u32 old_dram_timing2 = 0; + + arb_refresh_rate = RREG32(MC_ARB_RFSH_RATE) & + ~(POWERMODE1_MASK | POWERMODE2_MASK | POWERMODE3_MASK); + arb_refresh_rate |= + (POWERMODE1(rv770_calculate_memory_refresh_rate(rdev, state->low.sclk)) | + POWERMODE2(rv770_calculate_memory_refresh_rate(rdev, state->medium.sclk)) | + POWERMODE3(rv770_calculate_memory_refresh_rate(rdev, state->high.sclk))); + WREG32(MC_ARB_RFSH_RATE, arb_refresh_rate); + + /* save the boot dram timings */ + old_dram_timing = RREG32(MC_ARB_DRAM_TIMING); + old_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2); + + radeon_atom_set_engine_dram_timings(rdev, + state->high.sclk, + state->high.mclk); + + dram_timing = RREG32(MC_ARB_DRAM_TIMING); + dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2); + + WREG32(MC_ARB_DRAM_TIMING_3, dram_timing); + WREG32(MC_ARB_DRAM_TIMING2_3, dram_timing2); + + radeon_atom_set_engine_dram_timings(rdev, + state->medium.sclk, + state->medium.mclk); + + dram_timing = RREG32(MC_ARB_DRAM_TIMING); + dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2); + + WREG32(MC_ARB_DRAM_TIMING_2, dram_timing); + WREG32(MC_ARB_DRAM_TIMING2_2, dram_timing2); + + radeon_atom_set_engine_dram_timings(rdev, + state->low.sclk, + state->low.mclk); + + dram_timing = RREG32(MC_ARB_DRAM_TIMING); + dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2); + + WREG32(MC_ARB_DRAM_TIMING_1, dram_timing); + WREG32(MC_ARB_DRAM_TIMING2_1, dram_timing2); + + /* restore the boot dram timings */ + WREG32(MC_ARB_DRAM_TIMING, old_dram_timing); + WREG32(MC_ARB_DRAM_TIMING2, old_dram_timing2); + +} + +void rv730_start_dpm(struct radeon_device *rdev) +{ + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF); + + WREG32_P(TCI_MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF); + + WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN); +} + +void rv730_stop_dpm(struct radeon_device *rdev) +{ + PPSMC_Result result; + + result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_TwoLevelsDisabled); + + if (result != PPSMC_Result_OK) + DRM_DEBUG("Could not force DPM to low\n"); + + WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN); + + WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF); + + WREG32_P(TCI_MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF); +} + +void rv730_program_dcodt(struct radeon_device *rdev, bool use_dcodt) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 i = use_dcodt ? 0 : 1; + u32 mc4_io_pad_cntl; + + mc4_io_pad_cntl = RREG32(MC4_IO_DQ_PAD_CNTL_D0_I0); + mc4_io_pad_cntl &= 0xFFFFFF00; + mc4_io_pad_cntl |= pi->odt_value_0[i]; + WREG32(MC4_IO_DQ_PAD_CNTL_D0_I0, mc4_io_pad_cntl); + WREG32(MC4_IO_DQ_PAD_CNTL_D0_I1, mc4_io_pad_cntl); + + mc4_io_pad_cntl = RREG32(MC4_IO_QS_PAD_CNTL_D0_I0); + mc4_io_pad_cntl &= 0xFFFFFF00; + mc4_io_pad_cntl |= pi->odt_value_1[i]; + WREG32(MC4_IO_QS_PAD_CNTL_D0_I0, mc4_io_pad_cntl); + WREG32(MC4_IO_QS_PAD_CNTL_D0_I1, mc4_io_pad_cntl); +} + +void rv730_get_odt_values(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 mc4_io_pad_cntl; + + pi->odt_value_0[0] = (u8)0; + pi->odt_value_1[0] = (u8)0x80; + + mc4_io_pad_cntl = RREG32(MC4_IO_DQ_PAD_CNTL_D0_I0); + pi->odt_value_0[1] = (u8)(mc4_io_pad_cntl & 0xff); + + mc4_io_pad_cntl = RREG32(MC4_IO_QS_PAD_CNTL_D0_I0); + pi->odt_value_1[1] = (u8)(mc4_io_pad_cntl & 0xff); +} diff --git a/sys/dev/pci/drm/radeon/rv730d.h b/sys/dev/pci/drm/radeon/rv730d.h new file mode 100644 index 00000000000..f0a7954fb1c --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv730d.h @@ -0,0 +1,165 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef RV730_H +#define RV730_H + +#define CG_SPLL_FUNC_CNTL 0x600 +#define SPLL_RESET (1 << 0) +#define SPLL_SLEEP (1 << 1) +#define SPLL_DIVEN (1 << 2) +#define SPLL_BYPASS_EN (1 << 3) +#define SPLL_REF_DIV(x) ((x) << 4) +#define SPLL_REF_DIV_MASK (0x3f << 4) +#define SPLL_HILEN(x) ((x) << 12) +#define SPLL_HILEN_MASK (0xf << 12) +#define SPLL_LOLEN(x) ((x) << 16) +#define SPLL_LOLEN_MASK (0xf << 16) +#define CG_SPLL_FUNC_CNTL_2 0x604 +#define SCLK_MUX_SEL(x) ((x) << 0) +#define SCLK_MUX_SEL_MASK (0x1ff << 0) +#define CG_SPLL_FUNC_CNTL_3 0x608 +#define SPLL_FB_DIV(x) ((x) << 0) +#define SPLL_FB_DIV_MASK (0x3ffffff << 0) +#define SPLL_DITHEN (1 << 28) + +#define CG_MPLL_FUNC_CNTL 0x624 +#define MPLL_RESET (1 << 0) +#define MPLL_SLEEP (1 << 1) +#define MPLL_DIVEN (1 << 2) +#define MPLL_BYPASS_EN (1 << 3) +#define MPLL_REF_DIV(x) ((x) << 4) +#define MPLL_REF_DIV_MASK (0x3f << 4) +#define MPLL_HILEN(x) ((x) << 12) +#define MPLL_HILEN_MASK (0xf << 12) +#define MPLL_LOLEN(x) ((x) << 16) +#define MPLL_LOLEN_MASK (0xf << 16) +#define CG_MPLL_FUNC_CNTL_2 0x628 +#define MCLK_MUX_SEL(x) ((x) << 0) +#define MCLK_MUX_SEL_MASK (0x1ff << 0) +#define CG_MPLL_FUNC_CNTL_3 0x62c +#define MPLL_FB_DIV(x) ((x) << 0) +#define MPLL_FB_DIV_MASK (0x3ffffff << 0) +#define MPLL_DITHEN (1 << 28) + +#define CG_TCI_MPLL_SPREAD_SPECTRUM 0x634 +#define CG_TCI_MPLL_SPREAD_SPECTRUM_2 0x638 +#define GENERAL_PWRMGT 0x63c +# define GLOBAL_PWRMGT_EN (1 << 0) +# define STATIC_PM_EN (1 << 1) +# define THERMAL_PROTECTION_DIS (1 << 2) +# define THERMAL_PROTECTION_TYPE (1 << 3) +# define ENABLE_GEN2PCIE (1 << 4) +# define ENABLE_GEN2XSP (1 << 5) +# define SW_SMIO_INDEX(x) ((x) << 6) +# define SW_SMIO_INDEX_MASK (3 << 6) +# define LOW_VOLT_D2_ACPI (1 << 8) +# define LOW_VOLT_D3_ACPI (1 << 9) +# define VOLT_PWRMGT_EN (1 << 10) +# define BACKBIAS_PAD_EN (1 << 18) +# define BACKBIAS_VALUE (1 << 19) +# define DYN_SPREAD_SPECTRUM_EN (1 << 23) +# define AC_DC_SW (1 << 24) + +#define SCLK_PWRMGT_CNTL 0x644 +# define SCLK_PWRMGT_OFF (1 << 0) +# define SCLK_LOW_D1 (1 << 1) +# define FIR_RESET (1 << 4) +# define FIR_FORCE_TREND_SEL (1 << 5) +# define FIR_TREND_MODE (1 << 6) +# define DYN_GFX_CLK_OFF_EN (1 << 7) +# define GFX_CLK_FORCE_ON (1 << 8) +# define GFX_CLK_REQUEST_OFF (1 << 9) +# define GFX_CLK_FORCE_OFF (1 << 10) +# define GFX_CLK_OFF_ACPI_D1 (1 << 11) +# define GFX_CLK_OFF_ACPI_D2 (1 << 12) +# define GFX_CLK_OFF_ACPI_D3 (1 << 13) + +#define TCI_MCLK_PWRMGT_CNTL 0x648 +# define MPLL_PWRMGT_OFF (1 << 5) +# define DLL_READY (1 << 6) +# define MC_INT_CNTL (1 << 7) +# define MRDCKA_SLEEP (1 << 8) +# define MRDCKB_SLEEP (1 << 9) +# define MRDCKC_SLEEP (1 << 10) +# define MRDCKD_SLEEP (1 << 11) +# define MRDCKE_SLEEP (1 << 12) +# define MRDCKF_SLEEP (1 << 13) +# define MRDCKG_SLEEP (1 << 14) +# define MRDCKH_SLEEP (1 << 15) +# define MRDCKA_RESET (1 << 16) +# define MRDCKB_RESET (1 << 17) +# define MRDCKC_RESET (1 << 18) +# define MRDCKD_RESET (1 << 19) +# define MRDCKE_RESET (1 << 20) +# define MRDCKF_RESET (1 << 21) +# define MRDCKG_RESET (1 << 22) +# define MRDCKH_RESET (1 << 23) +# define DLL_READY_READ (1 << 24) +# define USE_DISPLAY_GAP (1 << 25) +# define USE_DISPLAY_URGENT_NORMAL (1 << 26) +# define MPLL_TURNOFF_D2 (1 << 28) +#define TCI_DLL_CNTL 0x64c + +#define CG_PG_CNTL 0x858 +# define PWRGATE_ENABLE (1 << 0) + +#define CG_AT 0x6d4 +#define CG_R(x) ((x) << 0) +#define CG_R_MASK (0xffff << 0) +#define CG_L(x) ((x) << 16) +#define CG_L_MASK (0xffff << 16) + +#define CG_SPLL_SPREAD_SPECTRUM 0x790 +#define SSEN (1 << 0) +#define CLK_S(x) ((x) << 4) +#define CLK_S_MASK (0xfff << 4) +#define CG_SPLL_SPREAD_SPECTRUM_2 0x794 +#define CLK_V(x) ((x) << 0) +#define CLK_V_MASK (0x3ffffff << 0) + +#define MC_ARB_DRAM_TIMING 0x2774 +#define MC_ARB_DRAM_TIMING2 0x2778 + +#define MC_ARB_RFSH_RATE 0x27b0 +#define POWERMODE0(x) ((x) << 0) +#define POWERMODE0_MASK (0xff << 0) +#define POWERMODE1(x) ((x) << 8) +#define POWERMODE1_MASK (0xff << 8) +#define POWERMODE2(x) ((x) << 16) +#define POWERMODE2_MASK (0xff << 16) +#define POWERMODE3(x) ((x) << 24) +#define POWERMODE3_MASK (0xff << 24) + +#define MC_ARB_DRAM_TIMING_1 0x27f0 +#define MC_ARB_DRAM_TIMING_2 0x27f4 +#define MC_ARB_DRAM_TIMING_3 0x27f8 +#define MC_ARB_DRAM_TIMING2_1 0x27fc +#define MC_ARB_DRAM_TIMING2_2 0x2800 +#define MC_ARB_DRAM_TIMING2_3 0x2804 + +#define MC4_IO_DQ_PAD_CNTL_D0_I0 0x2978 +#define MC4_IO_DQ_PAD_CNTL_D0_I1 0x297c +#define MC4_IO_QS_PAD_CNTL_D0_I0 0x2980 +#define MC4_IO_QS_PAD_CNTL_D0_I1 0x2984 + +#endif diff --git a/sys/dev/pci/drm/radeon/rv740_dpm.c b/sys/dev/pci/drm/radeon/rv740_dpm.c new file mode 100644 index 00000000000..ace9f804a61 --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv740_dpm.c @@ -0,0 +1,416 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "rv740d.h" +#include "r600_dpm.h" +#include "rv770_dpm.h" +#include "atom.h" + +struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev); + +u32 rv740_get_decoded_reference_divider(u32 encoded_ref) +{ + u32 ref = 0; + + switch (encoded_ref) { + case 0: + ref = 1; + break; + case 16: + ref = 2; + break; + case 17: + ref = 3; + break; + case 18: + ref = 2; + break; + case 19: + ref = 3; + break; + case 20: + ref = 4; + break; + case 21: + ref = 5; + break; + default: + DRM_ERROR("Invalid encoded Reference Divider\n"); + ref = 0; + break; + } + + return ref; +} + +struct dll_speed_setting { + u16 min; + u16 max; + u32 dll_speed; +}; + +static struct dll_speed_setting dll_speed_table[16] = +{ + { 270, 320, 0x0f }, + { 240, 270, 0x0e }, + { 200, 240, 0x0d }, + { 180, 200, 0x0c }, + { 160, 180, 0x0b }, + { 140, 160, 0x0a }, + { 120, 140, 0x09 }, + { 110, 120, 0x08 }, + { 95, 110, 0x07 }, + { 85, 95, 0x06 }, + { 78, 85, 0x05 }, + { 70, 78, 0x04 }, + { 65, 70, 0x03 }, + { 60, 65, 0x02 }, + { 42, 60, 0x01 }, + { 00, 42, 0x00 } +}; + +u32 rv740_get_dll_speed(bool is_gddr5, u32 memory_clock) +{ + int i; + u32 factor; + u16 data_rate; + + if (is_gddr5) + factor = 4; + else + factor = 2; + + data_rate = (u16)(memory_clock * factor / 1000); + + if (data_rate < dll_speed_table[0].max) { + for (i = 0; i < 16; i++) { + if (data_rate > dll_speed_table[i].min && + data_rate <= dll_speed_table[i].max) + return dll_speed_table[i].dll_speed; + } + } + + DRM_DEBUG_KMS("Target MCLK greater than largest MCLK in DLL speed table\n"); + + return 0x0f; +} + +int rv740_populate_sclk_value(struct radeon_device *rdev, u32 engine_clock, + RV770_SMC_SCLK_VALUE *sclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct atom_clock_dividers dividers; + u32 spll_func_cntl = pi->clk_regs.rv770.cg_spll_func_cntl; + u32 spll_func_cntl_2 = pi->clk_regs.rv770.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = pi->clk_regs.rv770.cg_spll_func_cntl_3; + u32 cg_spll_spread_spectrum = pi->clk_regs.rv770.cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2 = pi->clk_regs.rv770.cg_spll_spread_spectrum_2; + u64 tmp; + u32 reference_clock = rdev->clock.spll.reference_freq; + u32 reference_divider; + u32 fbdiv; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + engine_clock, false, ÷rs); + if (ret) + return ret; + + reference_divider = 1 + dividers.ref_div; + + tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16384; + do_div(tmp, reference_clock); + fbdiv = (u32) tmp; + + spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK); + spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div); + spll_func_cntl |= SPLL_PDIV_A(dividers.post_div); + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(2); + + spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK; + spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv); + spll_func_cntl_3 |= SPLL_DITHEN; + + if (pi->sclk_ss) { + struct radeon_atom_ss ss; + u32 vco_freq = engine_clock * dividers.post_div; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_ENGINE_SS, vco_freq)) { + u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate); + u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000); + + cg_spll_spread_spectrum &= ~CLK_S_MASK; + cg_spll_spread_spectrum |= CLK_S(clk_s); + cg_spll_spread_spectrum |= SSEN; + + cg_spll_spread_spectrum_2 &= ~CLK_V_MASK; + cg_spll_spread_spectrum_2 |= CLK_V(clk_v); + } + } + + sclk->sclk_value = cpu_to_be32(engine_clock); + sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl); + sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2); + sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3); + sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(cg_spll_spread_spectrum); + sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(cg_spll_spread_spectrum_2); + + return 0; +} + +int rv740_populate_mclk_value(struct radeon_device *rdev, + u32 engine_clock, u32 memory_clock, + RV7XX_SMC_MCLK_VALUE *mclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 mpll_ad_func_cntl = pi->clk_regs.rv770.mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2 = pi->clk_regs.rv770.mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl = pi->clk_regs.rv770.mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2 = pi->clk_regs.rv770.mpll_dq_func_cntl_2; + u32 mclk_pwrmgt_cntl = pi->clk_regs.rv770.mclk_pwrmgt_cntl; + u32 dll_cntl = pi->clk_regs.rv770.dll_cntl; + u32 mpll_ss1 = pi->clk_regs.rv770.mpll_ss1; + u32 mpll_ss2 = pi->clk_regs.rv770.mpll_ss2; + struct atom_clock_dividers dividers; + u32 ibias; + u32 dll_speed; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, + memory_clock, false, ÷rs); + if (ret) + return ret; + + ibias = rv770_map_clkf_to_ibias(rdev, dividers.whole_fb_div); + + mpll_ad_func_cntl &= ~(CLKR_MASK | + YCLK_POST_DIV_MASK | + CLKF_MASK | + CLKFRAC_MASK | + IBIAS_MASK); + mpll_ad_func_cntl |= CLKR(dividers.ref_div); + mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div); + mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div); + mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div); + mpll_ad_func_cntl |= IBIAS(ibias); + + if (dividers.vco_mode) + mpll_ad_func_cntl_2 |= VCO_MODE; + else + mpll_ad_func_cntl_2 &= ~VCO_MODE; + + if (pi->mem_gddr5) { + mpll_dq_func_cntl &= ~(CLKR_MASK | + YCLK_POST_DIV_MASK | + CLKF_MASK | + CLKFRAC_MASK | + IBIAS_MASK); + mpll_dq_func_cntl |= CLKR(dividers.ref_div); + mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div); + mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div); + mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div); + mpll_dq_func_cntl |= IBIAS(ibias); + + if (dividers.vco_mode) + mpll_dq_func_cntl_2 |= VCO_MODE; + else + mpll_dq_func_cntl_2 &= ~VCO_MODE; + } + + if (pi->mclk_ss) { + struct radeon_atom_ss ss; + u32 vco_freq = memory_clock * dividers.post_div; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_MEMORY_SS, vco_freq)) { + u32 reference_clock = rdev->clock.mpll.reference_freq; + u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div); + u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate); + u32 clk_v = 0x40000 * ss.percentage * + (dividers.whole_fb_div + (dividers.frac_fb_div / 8)) / (clk_s * 10000); + + mpll_ss1 &= ~CLKV_MASK; + mpll_ss1 |= CLKV(clk_v); + + mpll_ss2 &= ~CLKS_MASK; + mpll_ss2 |= CLKS(clk_s); + } + } + + dll_speed = rv740_get_dll_speed(pi->mem_gddr5, + memory_clock); + + mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK; + mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed); + + mclk->mclk770.mclk_value = cpu_to_be32(memory_clock); + mclk->mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl); + mclk->mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2); + mclk->mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl); + mclk->mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2); + mclk->mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + mclk->mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl); + mclk->mclk770.vMPLL_SS = cpu_to_be32(mpll_ss1); + mclk->mclk770.vMPLL_SS2 = cpu_to_be32(mpll_ss2); + + return 0; +} + +void rv740_read_clock_registers(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + pi->clk_regs.rv770.cg_spll_func_cntl = + RREG32(CG_SPLL_FUNC_CNTL); + pi->clk_regs.rv770.cg_spll_func_cntl_2 = + RREG32(CG_SPLL_FUNC_CNTL_2); + pi->clk_regs.rv770.cg_spll_func_cntl_3 = + RREG32(CG_SPLL_FUNC_CNTL_3); + pi->clk_regs.rv770.cg_spll_spread_spectrum = + RREG32(CG_SPLL_SPREAD_SPECTRUM); + pi->clk_regs.rv770.cg_spll_spread_spectrum_2 = + RREG32(CG_SPLL_SPREAD_SPECTRUM_2); + + pi->clk_regs.rv770.mpll_ad_func_cntl = + RREG32(MPLL_AD_FUNC_CNTL); + pi->clk_regs.rv770.mpll_ad_func_cntl_2 = + RREG32(MPLL_AD_FUNC_CNTL_2); + pi->clk_regs.rv770.mpll_dq_func_cntl = + RREG32(MPLL_DQ_FUNC_CNTL); + pi->clk_regs.rv770.mpll_dq_func_cntl_2 = + RREG32(MPLL_DQ_FUNC_CNTL_2); + pi->clk_regs.rv770.mclk_pwrmgt_cntl = + RREG32(MCLK_PWRMGT_CNTL); + pi->clk_regs.rv770.dll_cntl = RREG32(DLL_CNTL); + pi->clk_regs.rv770.mpll_ss1 = RREG32(MPLL_SS1); + pi->clk_regs.rv770.mpll_ss2 = RREG32(MPLL_SS2); +} + +int rv740_populate_smc_acpi_state(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 mpll_ad_func_cntl = pi->clk_regs.rv770.mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2 = pi->clk_regs.rv770.mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl = pi->clk_regs.rv770.mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2 = pi->clk_regs.rv770.mpll_dq_func_cntl_2; + u32 spll_func_cntl = pi->clk_regs.rv770.cg_spll_func_cntl; + u32 spll_func_cntl_2 = pi->clk_regs.rv770.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = pi->clk_regs.rv770.cg_spll_func_cntl_3; + u32 mclk_pwrmgt_cntl = pi->clk_regs.rv770.mclk_pwrmgt_cntl; + u32 dll_cntl = pi->clk_regs.rv770.dll_cntl; + + table->ACPIState = table->initialState; + + table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC; + + if (pi->acpi_vddc) { + rv770_populate_vddc_value(rdev, pi->acpi_vddc, + &table->ACPIState.levels[0].vddc); + table->ACPIState.levels[0].gen2PCIE = + pi->pcie_gen2 ? + pi->acpi_pcie_gen2 : 0; + table->ACPIState.levels[0].gen2XSP = + pi->acpi_pcie_gen2; + } else { + rv770_populate_vddc_value(rdev, pi->min_vddc_in_table, + &table->ACPIState.levels[0].vddc); + table->ACPIState.levels[0].gen2PCIE = 0; + } + + mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN; + + mpll_dq_func_cntl_2 |= BYPASS | BIAS_GEN_PDNB | RESET_EN; + + mclk_pwrmgt_cntl |= (MRDCKA0_RESET | + MRDCKA1_RESET | + MRDCKB0_RESET | + MRDCKB1_RESET | + MRDCKC0_RESET | + MRDCKC1_RESET | + MRDCKD0_RESET | + MRDCKD1_RESET); + + dll_cntl |= (MRDCKA0_BYPASS | + MRDCKA1_BYPASS | + MRDCKB0_BYPASS | + MRDCKB1_BYPASS | + MRDCKC0_BYPASS | + MRDCKC1_BYPASS | + MRDCKD0_BYPASS | + MRDCKD1_BYPASS); + + spll_func_cntl |= SPLL_RESET | SPLL_SLEEP | SPLL_BYPASS_EN; + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(4); + + table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl); + table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2); + table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl); + table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2); + table->ACPIState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + table->ACPIState.levels[0].mclk.mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl); + + table->ACPIState.levels[0].mclk.mclk770.mclk_value = 0; + + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3); + + table->ACPIState.levels[0].sclk.sclk_value = 0; + + table->ACPIState.levels[1] = table->ACPIState.levels[0]; + table->ACPIState.levels[2] = table->ACPIState.levels[0]; + + rv770_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd); + + return 0; +} + +void rv740_enable_mclk_spread_spectrum(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(MPLL_CNTL_MODE, SS_SSEN, ~SS_SSEN); + else + WREG32_P(MPLL_CNTL_MODE, 0, ~SS_SSEN); +} + +u8 rv740_get_mclk_frequency_ratio(u32 memory_clock) +{ + u8 mc_para_index; + + if ((memory_clock < 10000) || (memory_clock > 47500)) + mc_para_index = 0x00; + else + mc_para_index = (u8)((memory_clock - 10000) / 2500); + + return mc_para_index; +} diff --git a/sys/dev/pci/drm/radeon/rv740d.h b/sys/dev/pci/drm/radeon/rv740d.h new file mode 100644 index 00000000000..fe5ab075dc1 --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv740d.h @@ -0,0 +1,117 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef RV740_H +#define RV740_H + +#define CG_SPLL_FUNC_CNTL 0x600 +#define SPLL_RESET (1 << 0) +#define SPLL_SLEEP (1 << 1) +#define SPLL_BYPASS_EN (1 << 3) +#define SPLL_REF_DIV(x) ((x) << 4) +#define SPLL_REF_DIV_MASK (0x3f << 4) +#define SPLL_PDIV_A(x) ((x) << 20) +#define SPLL_PDIV_A_MASK (0x7f << 20) +#define CG_SPLL_FUNC_CNTL_2 0x604 +#define SCLK_MUX_SEL(x) ((x) << 0) +#define SCLK_MUX_SEL_MASK (0x1ff << 0) +#define CG_SPLL_FUNC_CNTL_3 0x608 +#define SPLL_FB_DIV(x) ((x) << 0) +#define SPLL_FB_DIV_MASK (0x3ffffff << 0) +#define SPLL_DITHEN (1 << 28) + +#define MPLL_CNTL_MODE 0x61c +#define SS_SSEN (1 << 24) + +#define MPLL_AD_FUNC_CNTL 0x624 +#define CLKF(x) ((x) << 0) +#define CLKF_MASK (0x7f << 0) +#define CLKR(x) ((x) << 7) +#define CLKR_MASK (0x1f << 7) +#define CLKFRAC(x) ((x) << 12) +#define CLKFRAC_MASK (0x1f << 12) +#define YCLK_POST_DIV(x) ((x) << 17) +#define YCLK_POST_DIV_MASK (3 << 17) +#define IBIAS(x) ((x) << 20) +#define IBIAS_MASK (0x3ff << 20) +#define RESET (1 << 30) +#define PDNB (1 << 31) +#define MPLL_AD_FUNC_CNTL_2 0x628 +#define BYPASS (1 << 19) +#define BIAS_GEN_PDNB (1 << 24) +#define RESET_EN (1 << 25) +#define VCO_MODE (1 << 29) +#define MPLL_DQ_FUNC_CNTL 0x62c +#define MPLL_DQ_FUNC_CNTL_2 0x630 + +#define MCLK_PWRMGT_CNTL 0x648 +#define DLL_SPEED(x) ((x) << 0) +#define DLL_SPEED_MASK (0x1f << 0) +# define MPLL_PWRMGT_OFF (1 << 5) +# define DLL_READY (1 << 6) +# define MC_INT_CNTL (1 << 7) +# define MRDCKA0_SLEEP (1 << 8) +# define MRDCKA1_SLEEP (1 << 9) +# define MRDCKB0_SLEEP (1 << 10) +# define MRDCKB1_SLEEP (1 << 11) +# define MRDCKC0_SLEEP (1 << 12) +# define MRDCKC1_SLEEP (1 << 13) +# define MRDCKD0_SLEEP (1 << 14) +# define MRDCKD1_SLEEP (1 << 15) +# define MRDCKA0_RESET (1 << 16) +# define MRDCKA1_RESET (1 << 17) +# define MRDCKB0_RESET (1 << 18) +# define MRDCKB1_RESET (1 << 19) +# define MRDCKC0_RESET (1 << 20) +# define MRDCKC1_RESET (1 << 21) +# define MRDCKD0_RESET (1 << 22) +# define MRDCKD1_RESET (1 << 23) +# define DLL_READY_READ (1 << 24) +# define USE_DISPLAY_GAP (1 << 25) +# define USE_DISPLAY_URGENT_NORMAL (1 << 26) +# define MPLL_TURNOFF_D2 (1 << 28) +#define DLL_CNTL 0x64c +# define MRDCKA0_BYPASS (1 << 24) +# define MRDCKA1_BYPASS (1 << 25) +# define MRDCKB0_BYPASS (1 << 26) +# define MRDCKB1_BYPASS (1 << 27) +# define MRDCKC0_BYPASS (1 << 28) +# define MRDCKC1_BYPASS (1 << 29) +# define MRDCKD0_BYPASS (1 << 30) +# define MRDCKD1_BYPASS (1 << 31) + +#define CG_SPLL_SPREAD_SPECTRUM 0x790 +#define SSEN (1 << 0) +#define CLK_S(x) ((x) << 4) +#define CLK_S_MASK (0xfff << 4) +#define CG_SPLL_SPREAD_SPECTRUM_2 0x794 +#define CLK_V(x) ((x) << 0) +#define CLK_V_MASK (0x3ffffff << 0) + +#define MPLL_SS1 0x85c +#define CLKV(x) ((x) << 0) +#define CLKV_MASK (0x3ffffff << 0) +#define MPLL_SS2 0x860 +#define CLKS(x) ((x) << 0) +#define CLKS_MASK (0xfff << 0) + +#endif diff --git a/sys/dev/pci/drm/radeon/rv770.c b/sys/dev/pci/drm/radeon/rv770.c index 9d10d783a39..f9abd846a25 100644 --- a/sys/dev/pci/drm/radeon/rv770.c +++ b/sys/dev/pci/drm/radeon/rv770.c @@ -1,4 +1,3 @@ -/* $OpenBSD: rv770.c,v 1.9 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. @@ -29,6 +28,7 @@ #include #include "radeon.h" #include "radeon_asic.h" +#include "radeon_audio.h" #include #include "rv770d.h" #include "atom.h" @@ -40,8 +40,766 @@ static void rv770_gpu_init(struct radeon_device *rdev); void rv770_fini(struct radeon_device *rdev); static void rv770_pcie_gen2_enable(struct radeon_device *rdev); +int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk); -u32 rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) +int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) +{ + unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; + int r; + + /* RV740 uses evergreen uvd clk programming */ + if (rdev->family == CHIP_RV740) + return evergreen_set_uvd_clocks(rdev, vclk, dclk); + + /* bypass vclk and dclk with bclk */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), + ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); + + if (!vclk || !dclk) { + /* keep the Bypass mode, put PLL to sleep */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); + return 0; + } + + r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000, + 43663, 0x03FFFFFE, 1, 30, ~0, + &fb_div, &vclk_div, &dclk_div); + if (r) + return r; + + fb_div |= 1; + vclk_div -= 1; + dclk_div -= 1; + + /* set UPLL_FB_DIV to 0x50000 */ + WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK); + + /* deassert UPLL_RESET and UPLL_SLEEP */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK)); + + /* assert BYPASS EN and FB_DIV[0] <- ??? why? */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); + WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1)); + + r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); + if (r) + return r; + + /* assert PLL_RESET */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); + + /* set the required FB_DIV, REF_DIV, Post divder values */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK); + WREG32_P(CG_UPLL_FUNC_CNTL_2, + UPLL_SW_HILEN(vclk_div >> 1) | + UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) | + UPLL_SW_HILEN2(dclk_div >> 1) | + UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)), + ~UPLL_SW_MASK); + + WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), + ~UPLL_FB_DIV_MASK); + + /* give the PLL some time to settle */ + mdelay(15); + + /* deassert PLL_RESET */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); + + mdelay(15); + + /* deassert BYPASS EN and FB_DIV[0] <- ??? why? */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); + WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1)); + + r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); + if (r) + return r; + + /* switch VCLK and DCLK selection */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), + ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); + + mdelay(100); + + return 0; +} + +static const u32 r7xx_golden_registers[] = +{ + 0x8d00, 0xffffffff, 0x0e0e0074, + 0x8d04, 0xffffffff, 0x013a2b34, + 0x9508, 0xffffffff, 0x00000002, + 0x8b20, 0xffffffff, 0, + 0x88c4, 0xffffffff, 0x000000c2, + 0x28350, 0xffffffff, 0, + 0x9058, 0xffffffff, 0x0fffc40f, + 0x240c, 0xffffffff, 0x00000380, + 0x733c, 0xffffffff, 0x00000002, + 0x2650, 0x00040000, 0, + 0x20bc, 0x00040000, 0, + 0x7300, 0xffffffff, 0x001000f0 +}; + +static const u32 r7xx_golden_dyn_gpr_registers[] = +{ + 0x8db0, 0xffffffff, 0x98989898, + 0x8db4, 0xffffffff, 0x98989898, + 0x8db8, 0xffffffff, 0x98989898, + 0x8dbc, 0xffffffff, 0x98989898, + 0x8dc0, 0xffffffff, 0x98989898, + 0x8dc4, 0xffffffff, 0x98989898, + 0x8dc8, 0xffffffff, 0x98989898, + 0x8dcc, 0xffffffff, 0x98989898, + 0x88c4, 0xffffffff, 0x00000082 +}; + +static const u32 rv770_golden_registers[] = +{ + 0x562c, 0xffffffff, 0, + 0x3f90, 0xffffffff, 0, + 0x9148, 0xffffffff, 0, + 0x3f94, 0xffffffff, 0, + 0x914c, 0xffffffff, 0, + 0x9698, 0x18000000, 0x18000000 +}; + +static const u32 rv770ce_golden_registers[] = +{ + 0x562c, 0xffffffff, 0, + 0x3f90, 0xffffffff, 0x00cc0000, + 0x9148, 0xffffffff, 0x00cc0000, + 0x3f94, 0xffffffff, 0x00cc0000, + 0x914c, 0xffffffff, 0x00cc0000, + 0x9b7c, 0xffffffff, 0x00fa0000, + 0x3f8c, 0xffffffff, 0x00fa0000, + 0x9698, 0x18000000, 0x18000000 +}; + +static const u32 rv770_mgcg_init[] = +{ + 0x8bcc, 0xffffffff, 0x130300f9, + 0x5448, 0xffffffff, 0x100, + 0x55e4, 0xffffffff, 0x100, + 0x160c, 0xffffffff, 0x100, + 0x5644, 0xffffffff, 0x100, + 0xc164, 0xffffffff, 0x100, + 0x8a18, 0xffffffff, 0x100, + 0x897c, 0xffffffff, 0x8000100, + 0x8b28, 0xffffffff, 0x3c000100, + 0x9144, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10000, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10001, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10002, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10003, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x0, + 0x9870, 0xffffffff, 0x100, + 0x8d58, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x0, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x1, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x2, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x3, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x4, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x5, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x6, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x7, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x8, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x9, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x8000, + 0x9490, 0xffffffff, 0x0, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x1, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x2, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x3, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x4, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x5, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x6, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x7, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x8, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x9, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x8000, + 0x9604, 0xffffffff, 0x0, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x1, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x2, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x3, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x4, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x5, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x6, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x7, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x8, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x9, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x80000000, + 0x9030, 0xffffffff, 0x100, + 0x9034, 0xffffffff, 0x100, + 0x9038, 0xffffffff, 0x100, + 0x903c, 0xffffffff, 0x100, + 0x9040, 0xffffffff, 0x100, + 0xa200, 0xffffffff, 0x100, + 0xa204, 0xffffffff, 0x100, + 0xa208, 0xffffffff, 0x100, + 0xa20c, 0xffffffff, 0x100, + 0x971c, 0xffffffff, 0x100, + 0x915c, 0xffffffff, 0x00020001, + 0x9160, 0xffffffff, 0x00040003, + 0x916c, 0xffffffff, 0x00060005, + 0x9170, 0xffffffff, 0x00080007, + 0x9174, 0xffffffff, 0x000a0009, + 0x9178, 0xffffffff, 0x000c000b, + 0x917c, 0xffffffff, 0x000e000d, + 0x9180, 0xffffffff, 0x0010000f, + 0x918c, 0xffffffff, 0x00120011, + 0x9190, 0xffffffff, 0x00140013, + 0x9194, 0xffffffff, 0x00020001, + 0x9198, 0xffffffff, 0x00040003, + 0x919c, 0xffffffff, 0x00060005, + 0x91a8, 0xffffffff, 0x00080007, + 0x91ac, 0xffffffff, 0x000a0009, + 0x91b0, 0xffffffff, 0x000c000b, + 0x91b4, 0xffffffff, 0x000e000d, + 0x91b8, 0xffffffff, 0x0010000f, + 0x91c4, 0xffffffff, 0x00120011, + 0x91c8, 0xffffffff, 0x00140013, + 0x91cc, 0xffffffff, 0x00020001, + 0x91d0, 0xffffffff, 0x00040003, + 0x91d4, 0xffffffff, 0x00060005, + 0x91e0, 0xffffffff, 0x00080007, + 0x91e4, 0xffffffff, 0x000a0009, + 0x91e8, 0xffffffff, 0x000c000b, + 0x91ec, 0xffffffff, 0x00020001, + 0x91f0, 0xffffffff, 0x00040003, + 0x91f4, 0xffffffff, 0x00060005, + 0x9200, 0xffffffff, 0x00080007, + 0x9204, 0xffffffff, 0x000a0009, + 0x9208, 0xffffffff, 0x000c000b, + 0x920c, 0xffffffff, 0x000e000d, + 0x9210, 0xffffffff, 0x0010000f, + 0x921c, 0xffffffff, 0x00120011, + 0x9220, 0xffffffff, 0x00140013, + 0x9224, 0xffffffff, 0x00020001, + 0x9228, 0xffffffff, 0x00040003, + 0x922c, 0xffffffff, 0x00060005, + 0x9238, 0xffffffff, 0x00080007, + 0x923c, 0xffffffff, 0x000a0009, + 0x9240, 0xffffffff, 0x000c000b, + 0x9244, 0xffffffff, 0x000e000d, + 0x9248, 0xffffffff, 0x0010000f, + 0x9254, 0xffffffff, 0x00120011, + 0x9258, 0xffffffff, 0x00140013, + 0x925c, 0xffffffff, 0x00020001, + 0x9260, 0xffffffff, 0x00040003, + 0x9264, 0xffffffff, 0x00060005, + 0x9270, 0xffffffff, 0x00080007, + 0x9274, 0xffffffff, 0x000a0009, + 0x9278, 0xffffffff, 0x000c000b, + 0x927c, 0xffffffff, 0x000e000d, + 0x9280, 0xffffffff, 0x0010000f, + 0x928c, 0xffffffff, 0x00120011, + 0x9290, 0xffffffff, 0x00140013, + 0x9294, 0xffffffff, 0x00020001, + 0x929c, 0xffffffff, 0x00040003, + 0x92a0, 0xffffffff, 0x00060005, + 0x92a4, 0xffffffff, 0x00080007 +}; + +static const u32 rv710_golden_registers[] = +{ + 0x3f90, 0x00ff0000, 0x00fc0000, + 0x9148, 0x00ff0000, 0x00fc0000, + 0x3f94, 0x00ff0000, 0x00fc0000, + 0x914c, 0x00ff0000, 0x00fc0000, + 0xb4c, 0x00000020, 0x00000020, + 0xa180, 0xffffffff, 0x00003f3f +}; + +static const u32 rv710_mgcg_init[] = +{ + 0x8bcc, 0xffffffff, 0x13030040, + 0x5448, 0xffffffff, 0x100, + 0x55e4, 0xffffffff, 0x100, + 0x160c, 0xffffffff, 0x100, + 0x5644, 0xffffffff, 0x100, + 0xc164, 0xffffffff, 0x100, + 0x8a18, 0xffffffff, 0x100, + 0x897c, 0xffffffff, 0x8000100, + 0x8b28, 0xffffffff, 0x3c000100, + 0x9144, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10000, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x0, + 0x9870, 0xffffffff, 0x100, + 0x8d58, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x0, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x1, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x8000, + 0x9490, 0xffffffff, 0x0, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x1, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x8000, + 0x9604, 0xffffffff, 0x0, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x1, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x80000000, + 0x9030, 0xffffffff, 0x100, + 0x9034, 0xffffffff, 0x100, + 0x9038, 0xffffffff, 0x100, + 0x903c, 0xffffffff, 0x100, + 0x9040, 0xffffffff, 0x100, + 0xa200, 0xffffffff, 0x100, + 0xa204, 0xffffffff, 0x100, + 0xa208, 0xffffffff, 0x100, + 0xa20c, 0xffffffff, 0x100, + 0x971c, 0xffffffff, 0x100, + 0x915c, 0xffffffff, 0x00020001, + 0x9174, 0xffffffff, 0x00000003, + 0x9178, 0xffffffff, 0x00050001, + 0x917c, 0xffffffff, 0x00030002, + 0x918c, 0xffffffff, 0x00000004, + 0x9190, 0xffffffff, 0x00070006, + 0x9194, 0xffffffff, 0x00050001, + 0x9198, 0xffffffff, 0x00030002, + 0x91a8, 0xffffffff, 0x00000004, + 0x91ac, 0xffffffff, 0x00070006, + 0x91e8, 0xffffffff, 0x00000001, + 0x9294, 0xffffffff, 0x00000001, + 0x929c, 0xffffffff, 0x00000002, + 0x92a0, 0xffffffff, 0x00040003, + 0x9150, 0xffffffff, 0x4d940000 +}; + +static const u32 rv730_golden_registers[] = +{ + 0x3f90, 0x00ff0000, 0x00f00000, + 0x9148, 0x00ff0000, 0x00f00000, + 0x3f94, 0x00ff0000, 0x00f00000, + 0x914c, 0x00ff0000, 0x00f00000, + 0x900c, 0xffffffff, 0x003b033f, + 0xb4c, 0x00000020, 0x00000020, + 0xa180, 0xffffffff, 0x00003f3f +}; + +static const u32 rv730_mgcg_init[] = +{ + 0x8bcc, 0xffffffff, 0x130300f9, + 0x5448, 0xffffffff, 0x100, + 0x55e4, 0xffffffff, 0x100, + 0x160c, 0xffffffff, 0x100, + 0x5644, 0xffffffff, 0x100, + 0xc164, 0xffffffff, 0x100, + 0x8a18, 0xffffffff, 0x100, + 0x897c, 0xffffffff, 0x8000100, + 0x8b28, 0xffffffff, 0x3c000100, + 0x9144, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10000, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10001, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x0, + 0x9870, 0xffffffff, 0x100, + 0x8d58, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x0, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x1, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x2, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x3, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x4, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x5, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x6, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x7, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x8000, + 0x9490, 0xffffffff, 0x0, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x1, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x2, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x3, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x4, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x5, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x6, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x7, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x8000, + 0x9604, 0xffffffff, 0x0, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x1, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x2, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x3, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x4, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x5, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x6, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x7, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x80000000, + 0x9030, 0xffffffff, 0x100, + 0x9034, 0xffffffff, 0x100, + 0x9038, 0xffffffff, 0x100, + 0x903c, 0xffffffff, 0x100, + 0x9040, 0xffffffff, 0x100, + 0xa200, 0xffffffff, 0x100, + 0xa204, 0xffffffff, 0x100, + 0xa208, 0xffffffff, 0x100, + 0xa20c, 0xffffffff, 0x100, + 0x971c, 0xffffffff, 0x100, + 0x915c, 0xffffffff, 0x00020001, + 0x916c, 0xffffffff, 0x00040003, + 0x9170, 0xffffffff, 0x00000005, + 0x9178, 0xffffffff, 0x00050001, + 0x917c, 0xffffffff, 0x00030002, + 0x918c, 0xffffffff, 0x00000004, + 0x9190, 0xffffffff, 0x00070006, + 0x9194, 0xffffffff, 0x00050001, + 0x9198, 0xffffffff, 0x00030002, + 0x91a8, 0xffffffff, 0x00000004, + 0x91ac, 0xffffffff, 0x00070006, + 0x91b0, 0xffffffff, 0x00050001, + 0x91b4, 0xffffffff, 0x00030002, + 0x91c4, 0xffffffff, 0x00000004, + 0x91c8, 0xffffffff, 0x00070006, + 0x91cc, 0xffffffff, 0x00050001, + 0x91d0, 0xffffffff, 0x00030002, + 0x91e0, 0xffffffff, 0x00000004, + 0x91e4, 0xffffffff, 0x00070006, + 0x91e8, 0xffffffff, 0x00000001, + 0x91ec, 0xffffffff, 0x00050001, + 0x91f0, 0xffffffff, 0x00030002, + 0x9200, 0xffffffff, 0x00000004, + 0x9204, 0xffffffff, 0x00070006, + 0x9208, 0xffffffff, 0x00050001, + 0x920c, 0xffffffff, 0x00030002, + 0x921c, 0xffffffff, 0x00000004, + 0x9220, 0xffffffff, 0x00070006, + 0x9224, 0xffffffff, 0x00050001, + 0x9228, 0xffffffff, 0x00030002, + 0x9238, 0xffffffff, 0x00000004, + 0x923c, 0xffffffff, 0x00070006, + 0x9240, 0xffffffff, 0x00050001, + 0x9244, 0xffffffff, 0x00030002, + 0x9254, 0xffffffff, 0x00000004, + 0x9258, 0xffffffff, 0x00070006, + 0x9294, 0xffffffff, 0x00000001, + 0x929c, 0xffffffff, 0x00000002, + 0x92a0, 0xffffffff, 0x00040003, + 0x92a4, 0xffffffff, 0x00000005 +}; + +static const u32 rv740_golden_registers[] = +{ + 0x88c4, 0xffffffff, 0x00000082, + 0x28a50, 0xfffffffc, 0x00000004, + 0x2650, 0x00040000, 0, + 0x20bc, 0x00040000, 0, + 0x733c, 0xffffffff, 0x00000002, + 0x7300, 0xffffffff, 0x001000f0, + 0x3f90, 0x00ff0000, 0, + 0x9148, 0x00ff0000, 0, + 0x3f94, 0x00ff0000, 0, + 0x914c, 0x00ff0000, 0, + 0x240c, 0xffffffff, 0x00000380, + 0x8a14, 0x00000007, 0x00000007, + 0x8b24, 0xffffffff, 0x00ff0fff, + 0x28a4c, 0xffffffff, 0x00004000, + 0xa180, 0xffffffff, 0x00003f3f, + 0x8d00, 0xffffffff, 0x0e0e003a, + 0x8d04, 0xffffffff, 0x013a0e2a, + 0x8c00, 0xffffffff, 0xe400000f, + 0x8db0, 0xffffffff, 0x98989898, + 0x8db4, 0xffffffff, 0x98989898, + 0x8db8, 0xffffffff, 0x98989898, + 0x8dbc, 0xffffffff, 0x98989898, + 0x8dc0, 0xffffffff, 0x98989898, + 0x8dc4, 0xffffffff, 0x98989898, + 0x8dc8, 0xffffffff, 0x98989898, + 0x8dcc, 0xffffffff, 0x98989898, + 0x9058, 0xffffffff, 0x0fffc40f, + 0x900c, 0xffffffff, 0x003b033f, + 0x28350, 0xffffffff, 0, + 0x8cf0, 0x1fffffff, 0x08e00420, + 0x9508, 0xffffffff, 0x00000002, + 0x88c4, 0xffffffff, 0x000000c2, + 0x9698, 0x18000000, 0x18000000 +}; + +static const u32 rv740_mgcg_init[] = +{ + 0x8bcc, 0xffffffff, 0x13030100, + 0x5448, 0xffffffff, 0x100, + 0x55e4, 0xffffffff, 0x100, + 0x160c, 0xffffffff, 0x100, + 0x5644, 0xffffffff, 0x100, + 0xc164, 0xffffffff, 0x100, + 0x8a18, 0xffffffff, 0x100, + 0x897c, 0xffffffff, 0x100, + 0x8b28, 0xffffffff, 0x100, + 0x9144, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10000, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10001, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10002, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x10003, + 0x9a50, 0xffffffff, 0x100, + 0x9a1c, 0xffffffff, 0x0, + 0x9870, 0xffffffff, 0x100, + 0x8d58, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x0, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x1, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x2, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x3, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x4, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x5, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x6, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x7, + 0x9510, 0xffffffff, 0x100, + 0x9500, 0xffffffff, 0x8000, + 0x9490, 0xffffffff, 0x0, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x1, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x2, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x3, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x4, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x5, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x6, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x7, + 0x949c, 0xffffffff, 0x100, + 0x9490, 0xffffffff, 0x8000, + 0x9604, 0xffffffff, 0x0, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x1, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x2, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x3, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x4, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x5, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x6, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x7, + 0x9654, 0xffffffff, 0x100, + 0x9604, 0xffffffff, 0x80000000, + 0x9030, 0xffffffff, 0x100, + 0x9034, 0xffffffff, 0x100, + 0x9038, 0xffffffff, 0x100, + 0x903c, 0xffffffff, 0x100, + 0x9040, 0xffffffff, 0x100, + 0xa200, 0xffffffff, 0x100, + 0xa204, 0xffffffff, 0x100, + 0xa208, 0xffffffff, 0x100, + 0xa20c, 0xffffffff, 0x100, + 0x971c, 0xffffffff, 0x100, + 0x915c, 0xffffffff, 0x00020001, + 0x9160, 0xffffffff, 0x00040003, + 0x916c, 0xffffffff, 0x00060005, + 0x9170, 0xffffffff, 0x00080007, + 0x9174, 0xffffffff, 0x000a0009, + 0x9178, 0xffffffff, 0x000c000b, + 0x917c, 0xffffffff, 0x000e000d, + 0x9180, 0xffffffff, 0x0010000f, + 0x918c, 0xffffffff, 0x00120011, + 0x9190, 0xffffffff, 0x00140013, + 0x9194, 0xffffffff, 0x00020001, + 0x9198, 0xffffffff, 0x00040003, + 0x919c, 0xffffffff, 0x00060005, + 0x91a8, 0xffffffff, 0x00080007, + 0x91ac, 0xffffffff, 0x000a0009, + 0x91b0, 0xffffffff, 0x000c000b, + 0x91b4, 0xffffffff, 0x000e000d, + 0x91b8, 0xffffffff, 0x0010000f, + 0x91c4, 0xffffffff, 0x00120011, + 0x91c8, 0xffffffff, 0x00140013, + 0x91cc, 0xffffffff, 0x00020001, + 0x91d0, 0xffffffff, 0x00040003, + 0x91d4, 0xffffffff, 0x00060005, + 0x91e0, 0xffffffff, 0x00080007, + 0x91e4, 0xffffffff, 0x000a0009, + 0x91e8, 0xffffffff, 0x000c000b, + 0x91ec, 0xffffffff, 0x00020001, + 0x91f0, 0xffffffff, 0x00040003, + 0x91f4, 0xffffffff, 0x00060005, + 0x9200, 0xffffffff, 0x00080007, + 0x9204, 0xffffffff, 0x000a0009, + 0x9208, 0xffffffff, 0x000c000b, + 0x920c, 0xffffffff, 0x000e000d, + 0x9210, 0xffffffff, 0x0010000f, + 0x921c, 0xffffffff, 0x00120011, + 0x9220, 0xffffffff, 0x00140013, + 0x9224, 0xffffffff, 0x00020001, + 0x9228, 0xffffffff, 0x00040003, + 0x922c, 0xffffffff, 0x00060005, + 0x9238, 0xffffffff, 0x00080007, + 0x923c, 0xffffffff, 0x000a0009, + 0x9240, 0xffffffff, 0x000c000b, + 0x9244, 0xffffffff, 0x000e000d, + 0x9248, 0xffffffff, 0x0010000f, + 0x9254, 0xffffffff, 0x00120011, + 0x9258, 0xffffffff, 0x00140013, + 0x9294, 0xffffffff, 0x00020001, + 0x929c, 0xffffffff, 0x00040003, + 0x92a0, 0xffffffff, 0x00060005, + 0x92a4, 0xffffffff, 0x00080007 +}; + +static void rv770_init_golden_registers(struct radeon_device *rdev) +{ + switch (rdev->family) { + case CHIP_RV770: + radeon_program_register_sequence(rdev, + r7xx_golden_registers, + (const u32)ARRAY_SIZE(r7xx_golden_registers)); + radeon_program_register_sequence(rdev, + r7xx_golden_dyn_gpr_registers, + (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers)); + if (rdev->pdev->device == 0x994e) + radeon_program_register_sequence(rdev, + rv770ce_golden_registers, + (const u32)ARRAY_SIZE(rv770ce_golden_registers)); + else + radeon_program_register_sequence(rdev, + rv770_golden_registers, + (const u32)ARRAY_SIZE(rv770_golden_registers)); + radeon_program_register_sequence(rdev, + rv770_mgcg_init, + (const u32)ARRAY_SIZE(rv770_mgcg_init)); + break; + case CHIP_RV730: + radeon_program_register_sequence(rdev, + r7xx_golden_registers, + (const u32)ARRAY_SIZE(r7xx_golden_registers)); + radeon_program_register_sequence(rdev, + r7xx_golden_dyn_gpr_registers, + (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers)); + radeon_program_register_sequence(rdev, + rv730_golden_registers, + (const u32)ARRAY_SIZE(rv730_golden_registers)); + radeon_program_register_sequence(rdev, + rv730_mgcg_init, + (const u32)ARRAY_SIZE(rv730_mgcg_init)); + break; + case CHIP_RV710: + radeon_program_register_sequence(rdev, + r7xx_golden_registers, + (const u32)ARRAY_SIZE(r7xx_golden_registers)); + radeon_program_register_sequence(rdev, + r7xx_golden_dyn_gpr_registers, + (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers)); + radeon_program_register_sequence(rdev, + rv710_golden_registers, + (const u32)ARRAY_SIZE(rv710_golden_registers)); + radeon_program_register_sequence(rdev, + rv710_mgcg_init, + (const u32)ARRAY_SIZE(rv710_mgcg_init)); + break; + case CHIP_RV740: + radeon_program_register_sequence(rdev, + rv740_golden_registers, + (const u32)ARRAY_SIZE(rv740_golden_registers)); + radeon_program_register_sequence(rdev, + rv740_mgcg_init, + (const u32)ARRAY_SIZE(rv740_mgcg_init)); + break; + default: + break; + } +} + +#define PCIE_BUS_CLK 10000 +#define TCLK (PCIE_BUS_CLK / 10) + +/** + * rv770_get_xclk - get the xclk + * + * @rdev: radeon_device pointer + * + * Returns the reference clock used by the gfx engine + * (r7xx-cayman). + */ +u32 rv770_get_xclk(struct radeon_device *rdev) +{ + u32 reference_clock = rdev->clock.spll.reference_freq; + u32 tmp = RREG32(CG_CLKPIN_CNTL); + + if (tmp & MUX_TCLK_TO_XCLK) + return TCLK; + + if (tmp & XTALIN_DIVIDE) + return reference_clock / 4; + + return reference_clock; +} + +void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset); @@ -75,9 +833,15 @@ u32 rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) /* Unlock the lock, so double-buffering can take place inside vblank */ tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK; WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); +} + +bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id) +{ + struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; /* Return current update_pending status: */ - return RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING; + return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & + AVIVO_D1GRPH_SURFACE_UPDATE_PENDING); } /* get temperature in millidegrees */ @@ -134,7 +898,6 @@ static int rv770_pcie_gart_enable(struct radeon_device *rdev) r = radeon_gart_table_vram_pin(rdev); if (r) return r; - radeon_gart_restore(rdev); /* Setup L2 cache */ WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | @@ -311,7 +1074,8 @@ static void rv770_mc_program(struct radeon_device *rdev) */ void r700_cp_stop(struct radeon_device *rdev) { - radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT)); WREG32(SCRATCH_UMSK, 0); rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; @@ -338,13 +1102,13 @@ static int rv770_cp_load_microcode(struct radeon_device *rdev) mdelay(15); WREG32(GRBM_SOFT_RESET, 0); - fw_data = (const __be32 *)rdev->pfp_fw; + fw_data = (const __be32 *)rdev->pfp_fw->data; WREG32(CP_PFP_UCODE_ADDR, 0); for (i = 0; i < R700_PFP_UCODE_SIZE; i++) WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); WREG32(CP_PFP_UCODE_ADDR, 0); - fw_data = (const __be32 *)rdev->me_fw; + fw_data = (const __be32 *)rdev->me_fw->data; WREG32(CP_ME_RAM_WADDR, 0); for (i = 0; i < R700_PM4_UCODE_SIZE; i++) WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); @@ -363,6 +1127,35 @@ void r700_cp_fini(struct radeon_device *rdev) radeon_scratch_free(rdev, ring->rptr_save_reg); } +void rv770_set_clk_bypass_mode(struct radeon_device *rdev) +{ + u32 tmp, i; + + if (rdev->flags & RADEON_IS_IGP) + return; + + tmp = RREG32(CG_SPLL_FUNC_CNTL_2); + tmp &= SCLK_MUX_SEL_MASK; + tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE; + WREG32(CG_SPLL_FUNC_CNTL_2, tmp); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS) + break; + udelay(1); + } + + tmp &= ~SCLK_MUX_UPDATE; + WREG32(CG_SPLL_FUNC_CNTL_2, tmp); + + tmp = RREG32(MPLL_CNTL_MODE); + if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730)) + tmp &= ~RV730_MPLL_MCLK_SEL; + else + tmp &= ~MPLL_MCLK_SEL; + WREG32(MPLL_CNTL_MODE, tmp); +} + /* * Core functions */ @@ -382,7 +1175,6 @@ static void rv770_gpu_init(struct radeon_device *rdev) u32 hdp_host_path_cntl; u32 sq_dyn_gpr_size_simd_ab_0; u32 gb_tiling_config = 0; - u32 cc_rb_backend_disable = 0; u32 cc_gc_shader_pipe_config = 0; u32 mc_arb_ramcfg; u32 db_debug4, tmp; @@ -516,21 +1308,10 @@ static void rv770_gpu_init(struct radeon_device *rdev) WREG32(SPI_CONFIG_CNTL, 0); } - cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000; - tmp = R7XX_MAX_BACKENDS - r600_count_pipe_bits(cc_rb_backend_disable >> 16); - if (tmp < rdev->config.rv770.max_backends) { - rdev->config.rv770.max_backends = tmp; - } - cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00; - tmp = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R7XX_MAX_PIPES_MASK); - if (tmp < rdev->config.rv770.max_pipes) { - rdev->config.rv770.max_pipes = tmp; - } - tmp = R7XX_MAX_SIMDS - r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK); - if (tmp < rdev->config.rv770.max_simds) { - rdev->config.rv770.max_simds = tmp; - } + tmp = rdev->config.rv770.max_simds - + r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK); + rdev->config.rv770.active_simds = tmp; switch (rdev->config.rv770.max_tile_pipes) { case 1: @@ -550,6 +1331,14 @@ static void rv770_gpu_init(struct radeon_device *rdev) rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes; disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK; + tmp = 0; + for (i = 0; i < rdev->config.rv770.max_backends; i++) + tmp |= (1 << i); + /* if all the backends are disabled, fix it up here */ + if ((disabled_rb_mask & tmp) == tmp) { + for (i = 0; i < rdev->config.rv770.max_backends; i++) + disabled_rb_mask &= ~(1 << i); + } tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT; tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends, R7XX_MAX_BACKENDS, disabled_rb_mask); @@ -584,6 +1373,11 @@ static void rv770_gpu_init(struct radeon_device *rdev) WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff)); WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff)); WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff)); + if (rdev->family == CHIP_RV730) { + WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff)); + WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff)); + WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff)); + } WREG32(CGTS_SYS_TCC_DISABLE, 0); WREG32(CGTS_TCC_DISABLE, 0); @@ -813,7 +1607,7 @@ void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc) } if (rdev->flags & RADEON_IS_AGP) { size_bf = mc->gtt_start; - size_af = 0xFFFFFFFF - mc->gtt_end; + size_af = mc->mc_mask - mc->gtt_end; if (size_bf > size_af) { if (mc->mc_vram_size > size_bf) { dev_warn(rdev->dev, "limiting VRAM\n"); @@ -885,80 +1679,6 @@ static int rv770_mc_init(struct radeon_device *rdev) return 0; } -/** - * rv770_copy_dma - copy pages using the DMA engine - * - * @rdev: radeon_device pointer - * @src_offset: src GPU address - * @dst_offset: dst GPU address - * @num_gpu_pages: number of GPU pages to xfer - * @fence: radeon fence object - * - * Copy GPU paging using the DMA engine (r7xx). - * Used by the radeon ttm implementation to move pages if - * registered as the asic copy callback. - */ -int rv770_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence) -{ - struct radeon_semaphore *sem = NULL; - int ring_index = rdev->asic->copy.dma_ring_index; - struct radeon_ring *ring = &rdev->ring[ring_index]; - u32 size_in_dw, cur_size_in_dw; - int i, num_loops; - int r = 0; - - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return r; - } - - size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; - num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFF); - r = radeon_ring_lock(rdev, ring, num_loops * 5 + 8); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); - return r; - } - - if (radeon_fence_need_sync(*fence, ring->idx)) { - radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring, - ring->idx); - radeon_fence_note_sync(*fence, ring->idx); - } else { - radeon_semaphore_free(rdev, &sem, NULL); - } - - for (i = 0; i < num_loops; i++) { - cur_size_in_dw = size_in_dw; - if (cur_size_in_dw > 0xFFFF) - cur_size_in_dw = 0xFFFF; - size_in_dw -= cur_size_in_dw; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw)); - radeon_ring_write(ring, dst_offset & 0xfffffffc); - radeon_ring_write(ring, src_offset & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); - radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); - src_offset += cur_size_in_dw * 4; - dst_offset += cur_size_in_dw * 4; - } - - r = radeon_fence_emit(rdev, fence, ring->idx); - if (r) { - radeon_ring_unlock_undo(rdev, ring); - return r; - } - - radeon_ring_unlock_commit(rdev, ring); - radeon_semaphore_free(rdev, &sem, *fence); - - return r; -} - static int rv770_startup(struct radeon_device *rdev) { struct radeon_ring *ring; @@ -967,20 +1687,13 @@ static int rv770_startup(struct radeon_device *rdev) /* enable pcie gen2 link */ rv770_pcie_gen2_enable(rdev); - rv770_mc_program(rdev); - - if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { - r = r600_init_microcode(rdev); - if (r) { - DRM_ERROR("Failed to load firmware!\n"); - return r; - } - } - + /* scratch needs to be initialized before MC */ r = r600_vram_scratch_init(rdev); if (r) return r; + rv770_mc_program(rdev); + if (rdev->flags & RADEON_IS_AGP) { rv770_agp_enable(rdev); } else { @@ -990,12 +1703,6 @@ static int rv770_startup(struct radeon_device *rdev) } rv770_gpu_init(rdev); - r = r600_blit_init(rdev); - if (r) { - r600_blit_fini(rdev); - rdev->asic->copy.copy = NULL; - dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); - } /* allocate wb buffer */ r = radeon_wb_init(rdev); @@ -1014,6 +1721,17 @@ static int rv770_startup(struct radeon_device *rdev) return r; } + r = uvd_v2_2_resume(rdev); + if (!r) { + r = radeon_fence_driver_start_ring(rdev, + R600_RING_TYPE_UVD_INDEX); + if (r) + dev_err(rdev->dev, "UVD fences init error (%d).\n", r); + } + + if (r) + rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; + /* Enable IRQ */ if (!rdev->irq.installed) { r = radeon_irq_kms_init(rdev); @@ -1031,15 +1749,13 @@ static int rv770_startup(struct radeon_device *rdev) ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, - R600_CP_RB_RPTR, R600_CP_RB_WPTR, - 0, 0xfffff, RADEON_CP_PACKET2); + RADEON_CP_PACKET2); if (r) return r; ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, - DMA_RB_RPTR, DMA_RB_WPTR, - 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); + DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); if (r) return r; @@ -1054,13 +1770,24 @@ static int rv770_startup(struct radeon_device *rdev) if (r) return r; + ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + if (ring->ring_size) { + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + RADEON_CP_PACKET2); + if (!r) + r = uvd_v1_0_init(rdev); + + if (r) + DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); + } + r = radeon_ib_pool_init(rdev); if (r) { dev_err(rdev->dev, "IB initialization failed (%d).\n", r); return r; } - r = r600_audio_init(rdev); + r = radeon_audio_init(rdev); if (r) { DRM_ERROR("radeon: audio init failed\n"); return r; @@ -1080,6 +1807,12 @@ int rv770_resume(struct radeon_device *rdev) /* post card */ atom_asic_init(rdev->mode_info.atom_context); + /* init golden registers */ + rv770_init_golden_registers(rdev); + + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_resume(rdev); + rdev->accel_working = true; r = rv770_startup(rdev); if (r) { @@ -1094,7 +1827,10 @@ int rv770_resume(struct radeon_device *rdev) int rv770_suspend(struct radeon_device *rdev) { - r600_audio_fini(rdev); + radeon_pm_suspend(rdev); + radeon_audio_fini(rdev); + uvd_v1_0_fini(rdev); + radeon_uvd_suspend(rdev); r700_cp_stop(rdev); r600_dma_stop(rdev); r600_irq_suspend(rdev); @@ -1136,6 +1872,8 @@ int rv770_init(struct radeon_device *rdev) DRM_INFO("GPU not posted. posting now...\n"); atom_asic_init(rdev->mode_info.atom_context); } + /* init golden registers */ + rv770_init_golden_registers(rdev); /* Initialize scratch registers */ r600_scratch_init(rdev); /* Initialize surface registers */ @@ -1160,12 +1898,30 @@ int rv770_init(struct radeon_device *rdev) if (r) return r; + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { + r = r600_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); + r = radeon_uvd_init(rdev); + if (!r) { + rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; + r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], + 4096); + } + rdev->ih.ring_obj = NULL; r600_ih_ring_init(rdev, 64 * 1024); @@ -1192,13 +1948,15 @@ int rv770_init(struct radeon_device *rdev) void rv770_fini(struct radeon_device *rdev) { - r600_blit_fini(rdev); + radeon_pm_fini(rdev); r700_cp_fini(rdev); r600_dma_fini(rdev); r600_irq_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); radeon_irq_kms_fini(rdev); + uvd_v1_0_fini(rdev); + radeon_uvd_fini(rdev); rv770_pcie_gart_fini(rdev); r600_vram_scratch_fini(rdev); radeon_gem_fini(rdev); @@ -1215,7 +1973,6 @@ static void rv770_pcie_gen2_enable(struct radeon_device *rdev) u32 link_width_cntl, lanes, speed_cntl, tmp; u16 link_cntl2; u32 mask; - int ret; if (radeon_pcie_gen2 == 0) return; @@ -1230,33 +1987,32 @@ static void rv770_pcie_gen2_enable(struct radeon_device *rdev) if (ASIC_IS_X2(rdev)) return; - ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); - if (ret != 0) + if (drm_pcie_get_speed_cap_mask(rdev->ddev, &mask)) return; - if (!(mask & DRM_PCIE_SPEED_50)) + if (!(mask & (DRM_PCIE_SPEED_50|DRM_PCIE_SPEED_80))) return; DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); /* advertise upconfig capability */ - link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); + link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); link_width_cntl &= ~LC_UPCONFIGURE_DIS; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); - link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) { lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT; link_width_cntl &= ~(LC_LINK_WIDTH_MASK | LC_RECONFIG_ARC_MISSING_ESCAPE); link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); } else { link_width_cntl |= LC_UPCONFIGURE_DIS; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); } - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) && (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { @@ -1269,29 +2025,29 @@ static void rv770_pcie_gen2_enable(struct radeon_device *rdev) WREG16(0x4088, link_cntl2); WREG32(MM_CFGREGS_CNTL, 0); - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); - speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); speed_cntl |= LC_GEN2_EN_STRAP; - WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); } else { - link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); + link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ if (1) link_width_cntl |= LC_UPCONFIGURE_DIS; else link_width_cntl &= ~LC_UPCONFIGURE_DIS; - WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); } } diff --git a/sys/dev/pci/drm/radeon/rv770_dma.c b/sys/dev/pci/drm/radeon/rv770_dma.c new file mode 100644 index 00000000000..c13bf2cb08e --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv770_dma.c @@ -0,0 +1,94 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "rv770d.h" + +/** + * rv770_copy_dma - copy pages using the DMA engine + * + * @rdev: radeon_device pointer + * @src_offset: src GPU address + * @dst_offset: dst GPU address + * @num_gpu_pages: number of GPU pages to xfer + * @resv: reservation object to sync to + * + * Copy GPU paging using the DMA engine (r7xx). + * Used by the radeon ttm implementation to move pages if + * registered as the asic copy callback. + */ +struct radeon_fence *rv770_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv) +{ + struct radeon_fence *fence; + struct radeon_sync sync; + int ring_index = rdev->asic->copy.dma_ring_index; + struct radeon_ring *ring = &rdev->ring[ring_index]; + u32 size_in_dw, cur_size_in_dw; + int i, num_loops; + int r = 0; + + radeon_sync_create(&sync); + + size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; + num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFF); + r = radeon_ring_lock(rdev, ring, num_loops * 5 + 8); + if (r) { + DRM_ERROR("radeon: moving bo (%d).\n", r); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); + + for (i = 0; i < num_loops; i++) { + cur_size_in_dw = size_in_dw; + if (cur_size_in_dw > 0xFFFF) + cur_size_in_dw = 0xFFFF; + size_in_dw -= cur_size_in_dw; + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw)); + radeon_ring_write(ring, dst_offset & 0xfffffffc); + radeon_ring_write(ring, src_offset & 0xfffffffc); + radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); + radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); + src_offset += cur_size_in_dw * 4; + dst_offset += cur_size_in_dw * 4; + } + + r = radeon_fence_emit(rdev, &fence, ring->idx); + if (r) { + radeon_ring_unlock_undo(rdev, ring); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_ring_unlock_commit(rdev, ring, false); + radeon_sync_free(rdev, &sync, fence); + + return fence; +} diff --git a/sys/dev/pci/drm/radeon/rv770_dpm.c b/sys/dev/pci/drm/radeon/rv770_dpm.c new file mode 100644 index 00000000000..502d5da13ce --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv770_dpm.c @@ -0,0 +1,2585 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "rv770d.h" +#include "r600_dpm.h" +#include "rv770_dpm.h" +#include "cypress_dpm.h" +#include "atom.h" + +#define MC_CG_ARB_FREQ_F0 0x0a +#define MC_CG_ARB_FREQ_F1 0x0b +#define MC_CG_ARB_FREQ_F2 0x0c +#define MC_CG_ARB_FREQ_F3 0x0d + +#define MC_CG_SEQ_DRAMCONF_S0 0x05 +#define MC_CG_SEQ_DRAMCONF_S1 0x06 + +#define PCIE_BUS_CLK 10000 +#define TCLK (PCIE_BUS_CLK / 10) + +#define SMC_RAM_END 0xC000 + +struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps) +{ + struct rv7xx_ps *ps = rps->ps_priv; + + return ps; +} + +struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev) +{ + struct evergreen_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +static void rv770_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + if (enable) { + tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK; + tmp |= LC_HW_VOLTAGE_IF_CONTROL(1); + tmp |= LC_GEN2_EN_STRAP; + } else { + if (!pi->boot_in_gen2) { + tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK; + tmp &= ~LC_GEN2_EN_STRAP; + } + } + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) || + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp); + +} + +static void rv770_enable_l0s(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L0S_INACTIVITY_MASK; + tmp |= LC_L0S_INACTIVITY(3); + WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp); +} + +static void rv770_enable_l1(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL); + tmp &= ~LC_L1_INACTIVITY_MASK; + tmp |= LC_L1_INACTIVITY(4); + tmp &= ~LC_PMI_TO_L1_DIS; + tmp &= ~LC_ASPM_TO_L1_DIS; + WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp); +} + +static void rv770_enable_pll_sleep_in_l1(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L1_INACTIVITY_MASK; + tmp |= LC_L1_INACTIVITY(8); + WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp); + + /* NOTE, this is a PCIE indirect reg, not PCIE PORT */ + tmp = RREG32_PCIE(PCIE_P_CNTL); + tmp |= P_PLL_PWRDN_IN_L1L23; + tmp &= ~P_PLL_BUF_PDNB; + tmp &= ~P_PLL_PDNB; + tmp |= P_ALLOW_PRX_FRONTEND_SHUTOFF; + WREG32_PCIE(PCIE_P_CNTL, tmp); +} + +static void rv770_gfx_clock_gating_enable(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN); + else { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN); + WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON); + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON); + RREG32(GB_TILING_CONFIG); + } +} + +static void rv770_mg_clock_gating_enable(struct radeon_device *rdev, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (enable) { + u32 mgcg_cgtt_local0; + + if (rdev->family == CHIP_RV770) + mgcg_cgtt_local0 = RV770_MGCGTTLOCAL0_DFLT; + else + mgcg_cgtt_local0 = RV7XX_MGCGTTLOCAL0_DFLT; + + WREG32(CG_CGTT_LOCAL_0, mgcg_cgtt_local0); + WREG32(CG_CGTT_LOCAL_1, (RV770_MGCGTTLOCAL1_DFLT & 0xFFFFCFFF)); + + if (pi->mgcgtssm) + WREG32(CGTS_SM_CTRL_REG, RV770_MGCGCGTSSMCTRL_DFLT); + } else { + WREG32(CG_CGTT_LOCAL_0, 0xFFFFFFFF); + WREG32(CG_CGTT_LOCAL_1, 0xFFFFCFFF); + } +} + +void rv770_restore_cgcg(struct radeon_device *rdev) +{ + bool dpm_en = false, cg_en = false; + + if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN) + dpm_en = true; + if (RREG32(SCLK_PWRMGT_CNTL) & DYN_GFX_CLK_OFF_EN) + cg_en = true; + + if (dpm_en && !cg_en) + WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN); +} + +static void rv770_start_dpm(struct radeon_device *rdev) +{ + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF); + + WREG32_P(MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF); + + WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN); +} + +void rv770_stop_dpm(struct radeon_device *rdev) +{ + PPSMC_Result result; + + result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_TwoLevelsDisabled); + + if (result != PPSMC_Result_OK) + DRM_DEBUG("Could not force DPM to low.\n"); + + WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN); + + WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF); + + WREG32_P(MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF); +} + +bool rv770_dpm_enabled(struct radeon_device *rdev) +{ + if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN) + return true; + else + return false; +} + +void rv770_enable_thermal_protection(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS); + else + WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS); +} + +void rv770_enable_acpi_pm(struct radeon_device *rdev) +{ + WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN); +} + +u8 rv770_get_seq_value(struct radeon_device *rdev, + struct rv7xx_pl *pl) +{ + return (pl->flags & ATOM_PPLIB_R600_FLAGS_LOWPOWER) ? + MC_CG_SEQ_DRAMCONF_S0 : MC_CG_SEQ_DRAMCONF_S1; +} + +#if 0 +int rv770_read_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 *value) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + return rv770_read_smc_sram_dword(rdev, + pi->soft_regs_start + reg_offset, + value, pi->sram_end); +} +#endif + +int rv770_write_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 value) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + return rv770_write_smc_sram_dword(rdev, + pi->soft_regs_start + reg_offset, + value, pi->sram_end); +} + +int rv770_populate_smc_t(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + RV770_SMC_SWSTATE *smc_state) +{ + struct rv7xx_ps *state = rv770_get_ps(radeon_state); + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + int i; + int a_n; + int a_d; + u8 l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE]; + u8 r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE]; + u32 a_t; + + l[0] = 0; + r[2] = 100; + + a_n = (int)state->medium.sclk * pi->lmp + + (int)state->low.sclk * (R600_AH_DFLT - pi->rlp); + a_d = (int)state->low.sclk * (100 - (int)pi->rlp) + + (int)state->medium.sclk * pi->lmp; + + l[1] = (u8)(pi->lmp - (int)pi->lmp * a_n / a_d); + r[0] = (u8)(pi->rlp + (100 - (int)pi->rlp) * a_n / a_d); + + a_n = (int)state->high.sclk * pi->lhp + (int)state->medium.sclk * + (R600_AH_DFLT - pi->rmp); + a_d = (int)state->medium.sclk * (100 - (int)pi->rmp) + + (int)state->high.sclk * pi->lhp; + + l[2] = (u8)(pi->lhp - (int)pi->lhp * a_n / a_d); + r[1] = (u8)(pi->rmp + (100 - (int)pi->rmp) * a_n / a_d); + + for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++) { + a_t = CG_R(r[i] * pi->bsp / 200) | CG_L(l[i] * pi->bsp / 200); + smc_state->levels[i].aT = cpu_to_be32(a_t); + } + + a_t = CG_R(r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200) | + CG_L(l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200); + + smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].aT = + cpu_to_be32(a_t); + + return 0; +} + +int rv770_populate_smc_sp(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + RV770_SMC_SWSTATE *smc_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + int i; + + for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++) + smc_state->levels[i].bSP = cpu_to_be32(pi->dsp); + + smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].bSP = + cpu_to_be32(pi->psp); + + return 0; +} + +static void rv770_calculate_fractional_mpll_feedback_divider(u32 memory_clock, + u32 reference_clock, + bool gddr5, + struct atom_clock_dividers *dividers, + u32 *clkf, + u32 *clkfrac) +{ + u32 post_divider, reference_divider, feedback_divider8; + u32 fyclk; + + if (gddr5) + fyclk = (memory_clock * 8) / 2; + else + fyclk = (memory_clock * 4) / 2; + + post_divider = dividers->post_div; + reference_divider = dividers->ref_div; + + feedback_divider8 = + (8 * fyclk * reference_divider * post_divider) / reference_clock; + + *clkf = feedback_divider8 / 8; + *clkfrac = feedback_divider8 % 8; +} + +static int rv770_encode_yclk_post_div(u32 postdiv, u32 *encoded_postdiv) +{ + int ret = 0; + + switch (postdiv) { + case 1: + *encoded_postdiv = 0; + break; + case 2: + *encoded_postdiv = 1; + break; + case 4: + *encoded_postdiv = 2; + break; + case 8: + *encoded_postdiv = 3; + break; + case 16: + *encoded_postdiv = 4; + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +u32 rv770_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf) +{ + if (clkf <= 0x10) + return 0x4B; + if (clkf <= 0x19) + return 0x5B; + if (clkf <= 0x21) + return 0x2B; + if (clkf <= 0x27) + return 0x6C; + if (clkf <= 0x31) + return 0x9D; + return 0xC6; +} + +static int rv770_populate_mclk_value(struct radeon_device *rdev, + u32 engine_clock, u32 memory_clock, + RV7XX_SMC_MCLK_VALUE *mclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u8 encoded_reference_dividers[] = { 0, 16, 17, 20, 21 }; + u32 mpll_ad_func_cntl = + pi->clk_regs.rv770.mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2 = + pi->clk_regs.rv770.mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl = + pi->clk_regs.rv770.mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2 = + pi->clk_regs.rv770.mpll_dq_func_cntl_2; + u32 mclk_pwrmgt_cntl = + pi->clk_regs.rv770.mclk_pwrmgt_cntl; + u32 dll_cntl = pi->clk_regs.rv770.dll_cntl; + struct atom_clock_dividers dividers; + u32 reference_clock = rdev->clock.mpll.reference_freq; + u32 clkf, clkfrac; + u32 postdiv_yclk; + u32 ibias; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, + memory_clock, false, ÷rs); + if (ret) + return ret; + + if ((dividers.ref_div < 1) || (dividers.ref_div > 5)) + return -EINVAL; + + rv770_calculate_fractional_mpll_feedback_divider(memory_clock, reference_clock, + pi->mem_gddr5, + ÷rs, &clkf, &clkfrac); + + ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk); + if (ret) + return ret; + + ibias = rv770_map_clkf_to_ibias(rdev, clkf); + + mpll_ad_func_cntl &= ~(CLKR_MASK | + YCLK_POST_DIV_MASK | + CLKF_MASK | + CLKFRAC_MASK | + IBIAS_MASK); + mpll_ad_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]); + mpll_ad_func_cntl |= YCLK_POST_DIV(postdiv_yclk); + mpll_ad_func_cntl |= CLKF(clkf); + mpll_ad_func_cntl |= CLKFRAC(clkfrac); + mpll_ad_func_cntl |= IBIAS(ibias); + + if (dividers.vco_mode) + mpll_ad_func_cntl_2 |= VCO_MODE; + else + mpll_ad_func_cntl_2 &= ~VCO_MODE; + + if (pi->mem_gddr5) { + rv770_calculate_fractional_mpll_feedback_divider(memory_clock, + reference_clock, + pi->mem_gddr5, + ÷rs, &clkf, &clkfrac); + + ibias = rv770_map_clkf_to_ibias(rdev, clkf); + + ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk); + if (ret) + return ret; + + mpll_dq_func_cntl &= ~(CLKR_MASK | + YCLK_POST_DIV_MASK | + CLKF_MASK | + CLKFRAC_MASK | + IBIAS_MASK); + mpll_dq_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]); + mpll_dq_func_cntl |= YCLK_POST_DIV(postdiv_yclk); + mpll_dq_func_cntl |= CLKF(clkf); + mpll_dq_func_cntl |= CLKFRAC(clkfrac); + mpll_dq_func_cntl |= IBIAS(ibias); + + if (dividers.vco_mode) + mpll_dq_func_cntl_2 |= VCO_MODE; + else + mpll_dq_func_cntl_2 &= ~VCO_MODE; + } + + mclk->mclk770.mclk_value = cpu_to_be32(memory_clock); + mclk->mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl); + mclk->mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2); + mclk->mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl); + mclk->mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2); + mclk->mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + mclk->mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl); + + return 0; +} + +static int rv770_populate_sclk_value(struct radeon_device *rdev, + u32 engine_clock, + RV770_SMC_SCLK_VALUE *sclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct atom_clock_dividers dividers; + u32 spll_func_cntl = + pi->clk_regs.rv770.cg_spll_func_cntl; + u32 spll_func_cntl_2 = + pi->clk_regs.rv770.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = + pi->clk_regs.rv770.cg_spll_func_cntl_3; + u32 cg_spll_spread_spectrum = + pi->clk_regs.rv770.cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2 = + pi->clk_regs.rv770.cg_spll_spread_spectrum_2; + u64 tmp; + u32 reference_clock = rdev->clock.spll.reference_freq; + u32 reference_divider, post_divider; + u32 fbdiv; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + engine_clock, false, ÷rs); + if (ret) + return ret; + + reference_divider = 1 + dividers.ref_div; + + if (dividers.enable_post_div) + post_divider = (0x0f & (dividers.post_div >> 4)) + (0x0f & dividers.post_div) + 2; + else + post_divider = 1; + + tmp = (u64) engine_clock * reference_divider * post_divider * 16384; + do_div(tmp, reference_clock); + fbdiv = (u32) tmp; + + if (dividers.enable_post_div) + spll_func_cntl |= SPLL_DIVEN; + else + spll_func_cntl &= ~SPLL_DIVEN; + spll_func_cntl &= ~(SPLL_HILEN_MASK | SPLL_LOLEN_MASK | SPLL_REF_DIV_MASK); + spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div); + spll_func_cntl |= SPLL_HILEN((dividers.post_div >> 4) & 0xf); + spll_func_cntl |= SPLL_LOLEN(dividers.post_div & 0xf); + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(2); + + spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK; + spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv); + spll_func_cntl_3 |= SPLL_DITHEN; + + if (pi->sclk_ss) { + struct radeon_atom_ss ss; + u32 vco_freq = engine_clock * post_divider; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_ENGINE_SS, vco_freq)) { + u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate); + u32 clk_v = ss.percentage * fbdiv / (clk_s * 10000); + + cg_spll_spread_spectrum &= ~CLKS_MASK; + cg_spll_spread_spectrum |= CLKS(clk_s); + cg_spll_spread_spectrum |= SSEN; + + cg_spll_spread_spectrum_2 &= ~CLKV_MASK; + cg_spll_spread_spectrum_2 |= CLKV(clk_v); + } + } + + sclk->sclk_value = cpu_to_be32(engine_clock); + sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl); + sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2); + sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3); + sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(cg_spll_spread_spectrum); + sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(cg_spll_spread_spectrum_2); + + return 0; +} + +int rv770_populate_vddc_value(struct radeon_device *rdev, u16 vddc, + RV770_SMC_VOLTAGE_VALUE *voltage) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + int i; + + if (!pi->voltage_control) { + voltage->index = 0; + voltage->value = 0; + return 0; + } + + for (i = 0; i < pi->valid_vddc_entries; i++) { + if (vddc <= pi->vddc_table[i].vddc) { + voltage->index = pi->vddc_table[i].vddc_index; + voltage->value = cpu_to_be16(vddc); + break; + } + } + + if (i == pi->valid_vddc_entries) + return -EINVAL; + + return 0; +} + +int rv770_populate_mvdd_value(struct radeon_device *rdev, u32 mclk, + RV770_SMC_VOLTAGE_VALUE *voltage) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (!pi->mvdd_control) { + voltage->index = MVDD_HIGH_INDEX; + voltage->value = cpu_to_be16(MVDD_HIGH_VALUE); + return 0; + } + + if (mclk <= pi->mvdd_split_frequency) { + voltage->index = MVDD_LOW_INDEX; + voltage->value = cpu_to_be16(MVDD_LOW_VALUE); + } else { + voltage->index = MVDD_HIGH_INDEX; + voltage->value = cpu_to_be16(MVDD_HIGH_VALUE); + } + + return 0; +} + +static int rv770_convert_power_level_to_smc(struct radeon_device *rdev, + struct rv7xx_pl *pl, + RV770_SMC_HW_PERFORMANCE_LEVEL *level, + u8 watermark_level) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + int ret; + + level->gen2PCIE = pi->pcie_gen2 ? + ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0; + level->gen2XSP = (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0; + level->backbias = (pl->flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? 1 : 0; + level->displayWatermark = watermark_level; + + if (rdev->family == CHIP_RV740) + ret = rv740_populate_sclk_value(rdev, pl->sclk, + &level->sclk); + else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + ret = rv730_populate_sclk_value(rdev, pl->sclk, + &level->sclk); + else + ret = rv770_populate_sclk_value(rdev, pl->sclk, + &level->sclk); + if (ret) + return ret; + + if (rdev->family == CHIP_RV740) { + if (pi->mem_gddr5) { + if (pl->mclk <= pi->mclk_strobe_mode_threshold) + level->strobeMode = + rv740_get_mclk_frequency_ratio(pl->mclk) | 0x10; + else + level->strobeMode = 0; + + if (pl->mclk > pi->mclk_edc_enable_threshold) + level->mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG; + else + level->mcFlags = 0; + } + ret = rv740_populate_mclk_value(rdev, pl->sclk, + pl->mclk, &level->mclk); + } else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + ret = rv730_populate_mclk_value(rdev, pl->sclk, + pl->mclk, &level->mclk); + else + ret = rv770_populate_mclk_value(rdev, pl->sclk, + pl->mclk, &level->mclk); + if (ret) + return ret; + + ret = rv770_populate_vddc_value(rdev, pl->vddc, + &level->vddc); + if (ret) + return ret; + + ret = rv770_populate_mvdd_value(rdev, pl->mclk, &level->mvdd); + + return ret; +} + +static int rv770_convert_power_state_to_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + RV770_SMC_SWSTATE *smc_state) +{ + struct rv7xx_ps *state = rv770_get_ps(radeon_state); + int ret; + + if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC)) + smc_state->flags |= PPSMC_SWSTATE_FLAG_DC; + + ret = rv770_convert_power_level_to_smc(rdev, + &state->low, + &smc_state->levels[0], + PPSMC_DISPLAY_WATERMARK_LOW); + if (ret) + return ret; + + ret = rv770_convert_power_level_to_smc(rdev, + &state->medium, + &smc_state->levels[1], + PPSMC_DISPLAY_WATERMARK_LOW); + if (ret) + return ret; + + ret = rv770_convert_power_level_to_smc(rdev, + &state->high, + &smc_state->levels[2], + PPSMC_DISPLAY_WATERMARK_HIGH); + if (ret) + return ret; + + smc_state->levels[0].arbValue = MC_CG_ARB_FREQ_F1; + smc_state->levels[1].arbValue = MC_CG_ARB_FREQ_F2; + smc_state->levels[2].arbValue = MC_CG_ARB_FREQ_F3; + + smc_state->levels[0].seqValue = rv770_get_seq_value(rdev, + &state->low); + smc_state->levels[1].seqValue = rv770_get_seq_value(rdev, + &state->medium); + smc_state->levels[2].seqValue = rv770_get_seq_value(rdev, + &state->high); + + rv770_populate_smc_sp(rdev, radeon_state, smc_state); + + return rv770_populate_smc_t(rdev, radeon_state, smc_state); + +} + +u32 rv770_calculate_memory_refresh_rate(struct radeon_device *rdev, + u32 engine_clock) +{ + u32 dram_rows; + u32 dram_refresh_rate; + u32 mc_arb_rfsh_rate; + u32 tmp; + + tmp = (RREG32(MC_ARB_RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT; + dram_rows = 1 << (tmp + 10); + tmp = RREG32(MC_SEQ_MISC0) & 3; + dram_refresh_rate = 1 << (tmp + 3); + mc_arb_rfsh_rate = ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64; + + return mc_arb_rfsh_rate; +} + +static void rv770_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct rv7xx_ps *state = rv770_get_ps(radeon_state); + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 sqm_ratio; + u32 arb_refresh_rate; + u32 high_clock; + + if (state->high.sclk < (state->low.sclk * 0xFF / 0x40)) + high_clock = state->high.sclk; + else + high_clock = (state->low.sclk * 0xFF / 0x40); + + radeon_atom_set_engine_dram_timings(rdev, high_clock, + state->high.mclk); + + sqm_ratio = + STATE0(64 * high_clock / pi->boot_sclk) | + STATE1(64 * high_clock / state->low.sclk) | + STATE2(64 * high_clock / state->medium.sclk) | + STATE3(64 * high_clock / state->high.sclk); + WREG32(MC_ARB_SQM_RATIO, sqm_ratio); + + arb_refresh_rate = + POWERMODE0(rv770_calculate_memory_refresh_rate(rdev, pi->boot_sclk)) | + POWERMODE1(rv770_calculate_memory_refresh_rate(rdev, state->low.sclk)) | + POWERMODE2(rv770_calculate_memory_refresh_rate(rdev, state->medium.sclk)) | + POWERMODE3(rv770_calculate_memory_refresh_rate(rdev, state->high.sclk)); + WREG32(MC_ARB_RFSH_RATE, arb_refresh_rate); +} + +void rv770_enable_backbias(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, BACKBIAS_PAD_EN, ~BACKBIAS_PAD_EN); + else + WREG32_P(GENERAL_PWRMGT, 0, ~(BACKBIAS_VALUE | BACKBIAS_PAD_EN)); +} + +static void rv770_enable_spread_spectrum(struct radeon_device *rdev, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (enable) { + if (pi->sclk_ss) + WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN); + + if (pi->mclk_ss) { + if (rdev->family == CHIP_RV740) + rv740_enable_mclk_spread_spectrum(rdev, true); + } + } else { + WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN); + + WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN); + + WREG32_P(CG_MPLL_SPREAD_SPECTRUM, 0, ~SSEN); + + if (rdev->family == CHIP_RV740) + rv740_enable_mclk_spread_spectrum(rdev, false); + } +} + +static void rv770_program_mpll_timing_parameters(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if ((rdev->family == CHIP_RV770) && !pi->mem_gddr5) { + WREG32(MPLL_TIME, + (MPLL_LOCK_TIME(R600_MPLLLOCKTIME_DFLT * pi->ref_div) | + MPLL_RESET_TIME(R600_MPLLRESETTIME_DFLT))); + } +} + +void rv770_setup_bsp(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 xclk = radeon_get_xclk(rdev); + + r600_calculate_u_and_p(pi->asi, + xclk, + 16, + &pi->bsp, + &pi->bsu); + + r600_calculate_u_and_p(pi->pasi, + xclk, + 16, + &pi->pbsp, + &pi->pbsu); + + pi->dsp = BSP(pi->bsp) | BSU(pi->bsu); + pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu); + + WREG32(CG_BSP, pi->dsp); + +} + +void rv770_program_git(struct radeon_device *rdev) +{ + WREG32_P(CG_GIT, CG_GICST(R600_GICST_DFLT), ~CG_GICST_MASK); +} + +void rv770_program_tp(struct radeon_device *rdev) +{ + int i; + enum r600_td td = R600_TD_DFLT; + + for (i = 0; i < R600_PM_NUMBER_OF_TC; i++) + WREG32(CG_FFCT_0 + (i * 4), (UTC_0(r600_utc[i]) | DTC_0(r600_dtc[i]))); + + if (td == R600_TD_AUTO) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL); + else + WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL); + if (td == R600_TD_UP) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE); + if (td == R600_TD_DOWN) + WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE); +} + +void rv770_program_tpp(struct radeon_device *rdev) +{ + WREG32(CG_TPC, R600_TPC_DFLT); +} + +void rv770_program_sstp(struct radeon_device *rdev) +{ + WREG32(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT))); +} + +void rv770_program_engine_speed_parameters(struct radeon_device *rdev) +{ + WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC); +} + +static void rv770_enable_display_gap(struct radeon_device *rdev) +{ + u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL); + + tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK); + tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) | + DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE)); + WREG32(CG_DISPLAY_GAP_CNTL, tmp); +} + +void rv770_program_vc(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + WREG32(CG_FTV, pi->vrc); +} + +void rv770_clear_vc(struct radeon_device *rdev) +{ + WREG32(CG_FTV, 0); +} + +int rv770_upload_firmware(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + int ret; + + rv770_reset_smc(rdev); + rv770_stop_smc_clock(rdev); + + ret = rv770_load_smc_ucode(rdev, pi->sram_end); + if (ret) + return ret; + + return 0; +} + +static int rv770_populate_smc_acpi_state(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + u32 mpll_ad_func_cntl = + pi->clk_regs.rv770.mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2 = + pi->clk_regs.rv770.mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl = + pi->clk_regs.rv770.mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2 = + pi->clk_regs.rv770.mpll_dq_func_cntl_2; + u32 spll_func_cntl = + pi->clk_regs.rv770.cg_spll_func_cntl; + u32 spll_func_cntl_2 = + pi->clk_regs.rv770.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = + pi->clk_regs.rv770.cg_spll_func_cntl_3; + u32 mclk_pwrmgt_cntl; + u32 dll_cntl; + + table->ACPIState = table->initialState; + + table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC; + + if (pi->acpi_vddc) { + rv770_populate_vddc_value(rdev, pi->acpi_vddc, + &table->ACPIState.levels[0].vddc); + if (pi->pcie_gen2) { + if (pi->acpi_pcie_gen2) + table->ACPIState.levels[0].gen2PCIE = 1; + else + table->ACPIState.levels[0].gen2PCIE = 0; + } else + table->ACPIState.levels[0].gen2PCIE = 0; + if (pi->acpi_pcie_gen2) + table->ACPIState.levels[0].gen2XSP = 1; + else + table->ACPIState.levels[0].gen2XSP = 0; + } else { + rv770_populate_vddc_value(rdev, pi->min_vddc_in_table, + &table->ACPIState.levels[0].vddc); + table->ACPIState.levels[0].gen2PCIE = 0; + } + + + mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN; + + mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN; + + mclk_pwrmgt_cntl = (MRDCKA0_RESET | + MRDCKA1_RESET | + MRDCKB0_RESET | + MRDCKB1_RESET | + MRDCKC0_RESET | + MRDCKC1_RESET | + MRDCKD0_RESET | + MRDCKD1_RESET); + + dll_cntl = 0xff000000; + + spll_func_cntl |= SPLL_RESET | SPLL_SLEEP | SPLL_BYPASS_EN; + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(4); + + table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl); + table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2); + table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl); + table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2); + + table->ACPIState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + table->ACPIState.levels[0].mclk.mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl); + + table->ACPIState.levels[0].mclk.mclk770.mclk_value = 0; + + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3); + + table->ACPIState.levels[0].sclk.sclk_value = 0; + + rv770_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd); + + table->ACPIState.levels[1] = table->ACPIState.levels[0]; + table->ACPIState.levels[2] = table->ACPIState.levels[0]; + + return 0; +} + +int rv770_populate_initial_mvdd_value(struct radeon_device *rdev, + RV770_SMC_VOLTAGE_VALUE *voltage) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if ((pi->s0_vid_lower_smio_cntl & pi->mvdd_mask_low) == + (pi->mvdd_low_smio[MVDD_LOW_INDEX] & pi->mvdd_mask_low) ) { + voltage->index = MVDD_LOW_INDEX; + voltage->value = cpu_to_be16(MVDD_LOW_VALUE); + } else { + voltage->index = MVDD_HIGH_INDEX; + voltage->value = cpu_to_be16(MVDD_HIGH_VALUE); + } + + return 0; +} + +static int rv770_populate_smc_initial_state(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_ps *initial_state = rv770_get_ps(radeon_state); + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 a_t; + + table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = + cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl); + table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = + cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl_2); + table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = + cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl); + table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = + cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl_2); + table->initialState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = + cpu_to_be32(pi->clk_regs.rv770.mclk_pwrmgt_cntl); + table->initialState.levels[0].mclk.mclk770.vDLL_CNTL = + cpu_to_be32(pi->clk_regs.rv770.dll_cntl); + + table->initialState.levels[0].mclk.mclk770.vMPLL_SS = + cpu_to_be32(pi->clk_regs.rv770.mpll_ss1); + table->initialState.levels[0].mclk.mclk770.vMPLL_SS2 = + cpu_to_be32(pi->clk_regs.rv770.mpll_ss2); + + table->initialState.levels[0].mclk.mclk770.mclk_value = + cpu_to_be32(initial_state->low.mclk); + + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_2); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_3); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 = + cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum_2); + + table->initialState.levels[0].sclk.sclk_value = + cpu_to_be32(initial_state->low.sclk); + + table->initialState.levels[0].arbValue = MC_CG_ARB_FREQ_F0; + + table->initialState.levels[0].seqValue = + rv770_get_seq_value(rdev, &initial_state->low); + + rv770_populate_vddc_value(rdev, + initial_state->low.vddc, + &table->initialState.levels[0].vddc); + rv770_populate_initial_mvdd_value(rdev, + &table->initialState.levels[0].mvdd); + + a_t = CG_R(0xffff) | CG_L(0); + table->initialState.levels[0].aT = cpu_to_be32(a_t); + + table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp); + + if (pi->boot_in_gen2) + table->initialState.levels[0].gen2PCIE = 1; + else + table->initialState.levels[0].gen2PCIE = 0; + if (initial_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) + table->initialState.levels[0].gen2XSP = 1; + else + table->initialState.levels[0].gen2XSP = 0; + + if (rdev->family == CHIP_RV740) { + if (pi->mem_gddr5) { + if (initial_state->low.mclk <= pi->mclk_strobe_mode_threshold) + table->initialState.levels[0].strobeMode = + rv740_get_mclk_frequency_ratio(initial_state->low.mclk) | 0x10; + else + table->initialState.levels[0].strobeMode = 0; + + if (initial_state->low.mclk >= pi->mclk_edc_enable_threshold) + table->initialState.levels[0].mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG; + else + table->initialState.levels[0].mcFlags = 0; + } + } + + table->initialState.levels[1] = table->initialState.levels[0]; + table->initialState.levels[2] = table->initialState.levels[0]; + + table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC; + + return 0; +} + +static int rv770_populate_smc_vddc_table(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + int i; + + for (i = 0; i < pi->valid_vddc_entries; i++) { + table->highSMIO[pi->vddc_table[i].vddc_index] = + pi->vddc_table[i].high_smio; + table->lowSMIO[pi->vddc_table[i].vddc_index] = + cpu_to_be32(pi->vddc_table[i].low_smio); + } + + table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_VDDC] = 0; + table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_VDDC] = + cpu_to_be32(pi->vddc_mask_low); + + for (i = 0; + ((i < pi->valid_vddc_entries) && + (pi->max_vddc_in_table > + pi->vddc_table[i].vddc)); + i++); + + table->maxVDDCIndexInPPTable = + pi->vddc_table[i].vddc_index; + + return 0; +} + +static int rv770_populate_smc_mvdd_table(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (pi->mvdd_control) { + table->lowSMIO[MVDD_HIGH_INDEX] |= + cpu_to_be32(pi->mvdd_low_smio[MVDD_HIGH_INDEX]); + table->lowSMIO[MVDD_LOW_INDEX] |= + cpu_to_be32(pi->mvdd_low_smio[MVDD_LOW_INDEX]); + + table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_MVDD] = 0; + table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_MVDD] = + cpu_to_be32(pi->mvdd_mask_low); + } + + return 0; +} + +static int rv770_init_smc_table(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state); + RV770_SMC_STATETABLE *table = &pi->smc_statetable; + int ret; + + memset(table, 0, sizeof(RV770_SMC_STATETABLE)); + + pi->boot_sclk = boot_state->low.sclk; + + rv770_populate_smc_vddc_table(rdev, table); + rv770_populate_smc_mvdd_table(rdev, table); + + switch (rdev->pm.int_thermal_type) { + case THERMAL_TYPE_RV770: + case THERMAL_TYPE_ADT7473_WITH_INTERNAL: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL; + break; + case THERMAL_TYPE_NONE: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE; + break; + case THERMAL_TYPE_EXTERNAL_GPIO: + default: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL; + break; + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) { + table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_DONT_WAIT_FOR_VBLANK_ON_ALERT) + table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_GOTO_BOOT_ON_ALERT) + table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE; + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) + table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC; + + if (pi->mem_gddr5) + table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5; + + if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + ret = rv730_populate_smc_initial_state(rdev, radeon_boot_state, table); + else + ret = rv770_populate_smc_initial_state(rdev, radeon_boot_state, table); + if (ret) + return ret; + + if (rdev->family == CHIP_RV740) + ret = rv740_populate_smc_acpi_state(rdev, table); + else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + ret = rv730_populate_smc_acpi_state(rdev, table); + else + ret = rv770_populate_smc_acpi_state(rdev, table); + if (ret) + return ret; + + table->driverState = table->initialState; + + return rv770_copy_bytes_to_smc(rdev, + pi->state_table_start, + (const u8 *)table, + sizeof(RV770_SMC_STATETABLE), + pi->sram_end); +} + +static int rv770_construct_vddc_table(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u16 min, max, step; + u32 steps = 0; + u8 vddc_index = 0; + u32 i; + + radeon_atom_get_min_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &min); + radeon_atom_get_max_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &max); + radeon_atom_get_voltage_step(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &step); + + steps = (max - min) / step + 1; + + if (steps > MAX_NO_VREG_STEPS) + return -EINVAL; + + for (i = 0; i < steps; i++) { + u32 gpio_pins, gpio_mask; + + pi->vddc_table[i].vddc = (u16)(min + i * step); + radeon_atom_get_voltage_gpio_settings(rdev, + pi->vddc_table[i].vddc, + SET_VOLTAGE_TYPE_ASIC_VDDC, + &gpio_pins, &gpio_mask); + pi->vddc_table[i].low_smio = gpio_pins & gpio_mask; + pi->vddc_table[i].high_smio = 0; + pi->vddc_mask_low = gpio_mask; + if (i > 0) { + if ((pi->vddc_table[i].low_smio != + pi->vddc_table[i - 1].low_smio ) || + (pi->vddc_table[i].high_smio != + pi->vddc_table[i - 1].high_smio)) + vddc_index++; + } + pi->vddc_table[i].vddc_index = vddc_index; + } + + pi->valid_vddc_entries = (u8)steps; + + return 0; +} + +static u32 rv770_get_mclk_split_point(struct atom_memory_info *memory_info) +{ + if (memory_info->mem_type == MEM_TYPE_GDDR3) + return 30000; + + return 0; +} + +static int rv770_get_mvdd_pin_configuration(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 gpio_pins, gpio_mask; + + radeon_atom_get_voltage_gpio_settings(rdev, + MVDD_HIGH_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC, + &gpio_pins, &gpio_mask); + pi->mvdd_mask_low = gpio_mask; + pi->mvdd_low_smio[MVDD_HIGH_INDEX] = + gpio_pins & gpio_mask; + + radeon_atom_get_voltage_gpio_settings(rdev, + MVDD_LOW_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC, + &gpio_pins, &gpio_mask); + pi->mvdd_low_smio[MVDD_LOW_INDEX] = + gpio_pins & gpio_mask; + + return 0; +} + +u8 rv770_get_memory_module_index(struct radeon_device *rdev) +{ + return (u8) ((RREG32(BIOS_SCRATCH_4) >> 16) & 0xff); +} + +static int rv770_get_mvdd_configuration(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u8 memory_module_index; + struct atom_memory_info memory_info; + + memory_module_index = rv770_get_memory_module_index(rdev); + + if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info)) { + pi->mvdd_control = false; + return 0; + } + + pi->mvdd_split_frequency = + rv770_get_mclk_split_point(&memory_info); + + if (pi->mvdd_split_frequency == 0) { + pi->mvdd_control = false; + return 0; + } + + return rv770_get_mvdd_pin_configuration(rdev); +} + +void rv770_enable_voltage_control(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN); + else + WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN); +} + +static void rv770_program_display_gap(struct radeon_device *rdev) +{ + u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL); + + tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK); + if (rdev->pm.dpm.new_active_crtcs & 1) { + tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK); + tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE); + } else if (rdev->pm.dpm.new_active_crtcs & 2) { + tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE); + tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK); + } else { + tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE); + tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE); + } + WREG32(CG_DISPLAY_GAP_CNTL, tmp); +} + +static void rv770_enable_dynamic_pcie_gen2(struct radeon_device *rdev, + bool enable) +{ + rv770_enable_bif_dynamic_pcie_gen2(rdev, enable); + + if (enable) + WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE); + else + WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE); +} + +static void r7xx_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + if ((rdev->family == CHIP_RV730) || + (rdev->family == CHIP_RV710) || + (rdev->family == CHIP_RV740)) + rv730_program_memory_timing_parameters(rdev, radeon_new_state); + else + rv770_program_memory_timing_parameters(rdev, radeon_new_state); +} + +static int rv770_upload_sw_state(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u16 address = pi->state_table_start + + offsetof(RV770_SMC_STATETABLE, driverState); + RV770_SMC_SWSTATE state = { 0 }; + int ret; + + ret = rv770_convert_power_state_to_smc(rdev, radeon_new_state, &state); + if (ret) + return ret; + + return rv770_copy_bytes_to_smc(rdev, address, (const u8 *)&state, + sizeof(RV770_SMC_SWSTATE), + pi->sram_end); +} + +int rv770_halt_smc(struct radeon_device *rdev) +{ + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Halt) != PPSMC_Result_OK) + return -EINVAL; + + if (rv770_wait_for_smc_inactive(rdev) != PPSMC_Result_OK) + return -EINVAL; + + return 0; +} + +int rv770_resume_smc(struct radeon_device *rdev) +{ + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Resume) != PPSMC_Result_OK) + return -EINVAL; + return 0; +} + +int rv770_set_sw_state(struct radeon_device *rdev) +{ + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToSwState) != PPSMC_Result_OK) + DRM_DEBUG("rv770_set_sw_state failed\n"); + return 0; +} + +int rv770_set_boot_state(struct radeon_device *rdev) +{ + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToInitialState) != PPSMC_Result_OK) + return -EINVAL; + return 0; +} + +void rv770_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv7xx_ps *new_state = rv770_get_ps(new_ps); + struct rv7xx_ps *current_state = rv770_get_ps(old_ps); + + if ((new_ps->vclk == old_ps->vclk) && + (new_ps->dclk == old_ps->dclk)) + return; + + if (new_state->high.sclk >= current_state->high.sclk) + return; + + radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk); +} + +void rv770_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps) +{ + struct rv7xx_ps *new_state = rv770_get_ps(new_ps); + struct rv7xx_ps *current_state = rv770_get_ps(old_ps); + + if ((new_ps->vclk == old_ps->vclk) && + (new_ps->dclk == old_ps->dclk)) + return; + + if (new_state->high.sclk < current_state->high.sclk) + return; + + radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk); +} + +int rv770_restrict_performance_levels_before_switch(struct radeon_device *rdev) +{ + if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_NoForcedLevel)) != PPSMC_Result_OK) + return -EINVAL; + + if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_TwoLevelsDisabled)) != PPSMC_Result_OK) + return -EINVAL; + + return 0; +} + +int rv770_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level) +{ + PPSMC_Msg msg; + + if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_ZeroLevelsDisabled) != PPSMC_Result_OK) + return -EINVAL; + msg = PPSMC_MSG_ForceHigh; + } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK) + return -EINVAL; + msg = (PPSMC_Msg)(PPSMC_MSG_TwoLevelsDisabled); + } else { + if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK) + return -EINVAL; + msg = (PPSMC_Msg)(PPSMC_MSG_ZeroLevelsDisabled); + } + + if (rv770_send_msg_to_smc(rdev, msg) != PPSMC_Result_OK) + return -EINVAL; + + rdev->pm.dpm.forced_level = level; + + return 0; +} + +void r7xx_start_smc(struct radeon_device *rdev) +{ + rv770_start_smc(rdev); + rv770_start_smc_clock(rdev); +} + + +void r7xx_stop_smc(struct radeon_device *rdev) +{ + rv770_reset_smc(rdev); + rv770_stop_smc_clock(rdev); +} + +static void rv770_read_clock_registers(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + pi->clk_regs.rv770.cg_spll_func_cntl = + RREG32(CG_SPLL_FUNC_CNTL); + pi->clk_regs.rv770.cg_spll_func_cntl_2 = + RREG32(CG_SPLL_FUNC_CNTL_2); + pi->clk_regs.rv770.cg_spll_func_cntl_3 = + RREG32(CG_SPLL_FUNC_CNTL_3); + pi->clk_regs.rv770.cg_spll_spread_spectrum = + RREG32(CG_SPLL_SPREAD_SPECTRUM); + pi->clk_regs.rv770.cg_spll_spread_spectrum_2 = + RREG32(CG_SPLL_SPREAD_SPECTRUM_2); + pi->clk_regs.rv770.mpll_ad_func_cntl = + RREG32(MPLL_AD_FUNC_CNTL); + pi->clk_regs.rv770.mpll_ad_func_cntl_2 = + RREG32(MPLL_AD_FUNC_CNTL_2); + pi->clk_regs.rv770.mpll_dq_func_cntl = + RREG32(MPLL_DQ_FUNC_CNTL); + pi->clk_regs.rv770.mpll_dq_func_cntl_2 = + RREG32(MPLL_DQ_FUNC_CNTL_2); + pi->clk_regs.rv770.mclk_pwrmgt_cntl = + RREG32(MCLK_PWRMGT_CNTL); + pi->clk_regs.rv770.dll_cntl = RREG32(DLL_CNTL); +} + +static void r7xx_read_clock_registers(struct radeon_device *rdev) +{ + if (rdev->family == CHIP_RV740) + rv740_read_clock_registers(rdev); + else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + rv730_read_clock_registers(rdev); + else + rv770_read_clock_registers(rdev); +} + +void rv770_read_voltage_smio_registers(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + pi->s0_vid_lower_smio_cntl = + RREG32(S0_VID_LOWER_SMIO_CNTL); +} + +void rv770_reset_smio_status(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 sw_smio_index, vid_smio_cntl; + + sw_smio_index = + (RREG32(GENERAL_PWRMGT) & SW_SMIO_INDEX_MASK) >> SW_SMIO_INDEX_SHIFT; + switch (sw_smio_index) { + case 3: + vid_smio_cntl = RREG32(S3_VID_LOWER_SMIO_CNTL); + break; + case 2: + vid_smio_cntl = RREG32(S2_VID_LOWER_SMIO_CNTL); + break; + case 1: + vid_smio_cntl = RREG32(S1_VID_LOWER_SMIO_CNTL); + break; + case 0: + return; + default: + vid_smio_cntl = pi->s0_vid_lower_smio_cntl; + break; + } + + WREG32(S0_VID_LOWER_SMIO_CNTL, vid_smio_cntl); + WREG32_P(GENERAL_PWRMGT, SW_SMIO_INDEX(0), ~SW_SMIO_INDEX_MASK); +} + +void rv770_get_memory_type(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 tmp; + + tmp = RREG32(MC_SEQ_MISC0); + + if (((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT) == + MC_SEQ_MISC0_GDDR5_VALUE) + pi->mem_gddr5 = true; + else + pi->mem_gddr5 = false; + +} + +void rv770_get_pcie_gen2_status(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 tmp; + + tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + + if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) && + (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) + pi->pcie_gen2 = true; + else + pi->pcie_gen2 = false; + + if (pi->pcie_gen2) { + if (tmp & LC_CURRENT_DATA_RATE) + pi->boot_in_gen2 = true; + else + pi->boot_in_gen2 = false; + } else + pi->boot_in_gen2 = false; +} + +#if 0 +static int rv770_enter_ulp_state(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (pi->gfx_clock_gating) { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN); + WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON); + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON); + RREG32(GB_TILING_CONFIG); + } + + WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower), + ~HOST_SMC_MSG_MASK); + + udelay(7000); + + return 0; +} + +static int rv770_exit_ulp_state(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + int i; + + WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_ResumeFromMinimumPower), + ~HOST_SMC_MSG_MASK); + + udelay(7000); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (((RREG32(SMC_MSG) & HOST_SMC_RESP_MASK) >> HOST_SMC_RESP_SHIFT) == 1) + break; + udelay(1000); + } + + if (pi->gfx_clock_gating) + WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN); + + return 0; +} +#endif + +static void rv770_get_mclk_odt_threshold(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u8 memory_module_index; + struct atom_memory_info memory_info; + + pi->mclk_odt_threshold = 0; + + if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) { + memory_module_index = rv770_get_memory_module_index(rdev); + + if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info)) + return; + + if (memory_info.mem_type == MEM_TYPE_DDR2 || + memory_info.mem_type == MEM_TYPE_DDR3) + pi->mclk_odt_threshold = 30000; + } +} + +void rv770_get_max_vddc(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u16 vddc; + + if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc)) + pi->max_vddc = 0; + else + pi->max_vddc = vddc; +} + +void rv770_program_response_times(struct radeon_device *rdev) +{ + u32 voltage_response_time, backbias_response_time; + u32 acpi_delay_time, vbi_time_out; + u32 vddc_dly, bb_dly, acpi_dly, vbi_dly; + u32 reference_clock; + + voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time; + backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time; + + if (voltage_response_time == 0) + voltage_response_time = 1000; + + if (backbias_response_time == 0) + backbias_response_time = 1000; + + acpi_delay_time = 15000; + vbi_time_out = 100000; + + reference_clock = radeon_get_xclk(rdev); + + vddc_dly = (voltage_response_time * reference_clock) / 1600; + bb_dly = (backbias_response_time * reference_clock) / 1600; + acpi_dly = (acpi_delay_time * reference_clock) / 1600; + vbi_dly = (vbi_time_out * reference_clock) / 1600; + + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_delay_vreg, vddc_dly); + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_delay_bbias, bb_dly); + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_delay_acpi, acpi_dly); + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly); +#if 0 + /* XXX look up hw revision */ + if (WEKIVA_A21) + rv770_write_smc_soft_register(rdev, + RV770_SMC_SOFT_REGISTER_baby_step_timer, + 0x10); +#endif +} + +static void rv770_program_dcodt_before_state_switch(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state); + struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state); + bool current_use_dc = false; + bool new_use_dc = false; + + if (pi->mclk_odt_threshold == 0) + return; + + if (current_state->high.mclk <= pi->mclk_odt_threshold) + current_use_dc = true; + + if (new_state->high.mclk <= pi->mclk_odt_threshold) + new_use_dc = true; + + if (current_use_dc == new_use_dc) + return; + + if (!current_use_dc && new_use_dc) + return; + + if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + rv730_program_dcodt(rdev, new_use_dc); +} + +static void rv770_program_dcodt_after_state_switch(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state); + struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state); + bool current_use_dc = false; + bool new_use_dc = false; + + if (pi->mclk_odt_threshold == 0) + return; + + if (current_state->high.mclk <= pi->mclk_odt_threshold) + current_use_dc = true; + + if (new_state->high.mclk <= pi->mclk_odt_threshold) + new_use_dc = true; + + if (current_use_dc == new_use_dc) + return; + + if (current_use_dc && !new_use_dc) + return; + + if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + rv730_program_dcodt(rdev, new_use_dc); +} + +static void rv770_retrieve_odt_values(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (pi->mclk_odt_threshold == 0) + return; + + if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + rv730_get_odt_values(rdev); +} + +static void rv770_set_dpm_event_sources(struct radeon_device *rdev, u32 sources) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + bool want_thermal_protection; + enum radeon_dpm_event_src dpm_event_src; + + switch (sources) { + case 0: + default: + want_thermal_protection = false; + break; + case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL; + break; + + case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL; + break; + + case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) | + (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL; + break; + } + + if (want_thermal_protection) { + WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK); + if (pi->thermal_protection) + WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS); + } else { + WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS); + } +} + +void rv770_enable_auto_throttle_source(struct radeon_device *rdev, + enum radeon_dpm_auto_throttle_src source, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (enable) { + if (!(pi->active_auto_throttle_sources & (1 << source))) { + pi->active_auto_throttle_sources |= 1 << source; + rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources); + } + } else { + if (pi->active_auto_throttle_sources & (1 << source)) { + pi->active_auto_throttle_sources &= ~(1 << source); + rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources); + } + } +} + +static int rv770_set_thermal_temperature_range(struct radeon_device *rdev, + int min_temp, int max_temp) +{ + int low_temp = 0 * 1000; + int high_temp = 255 * 1000; + + if (low_temp < min_temp) + low_temp = min_temp; + if (high_temp > max_temp) + high_temp = max_temp; + if (high_temp < low_temp) { + DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp); + return -EINVAL; + } + + WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK); + WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK); + WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK); + + rdev->pm.dpm.thermal.min_temp = low_temp; + rdev->pm.dpm.thermal.max_temp = high_temp; + + return 0; +} + +int rv770_dpm_enable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + int ret; + + if (pi->gfx_clock_gating) + rv770_restore_cgcg(rdev); + + if (rv770_dpm_enabled(rdev)) + return -EINVAL; + + if (pi->voltage_control) { + rv770_enable_voltage_control(rdev, true); + ret = rv770_construct_vddc_table(rdev); + if (ret) { + DRM_ERROR("rv770_construct_vddc_table failed\n"); + return ret; + } + } + + if (pi->dcodt) + rv770_retrieve_odt_values(rdev); + + if (pi->mvdd_control) { + ret = rv770_get_mvdd_configuration(rdev); + if (ret) { + DRM_ERROR("rv770_get_mvdd_configuration failed\n"); + return ret; + } + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS) + rv770_enable_backbias(rdev, true); + + rv770_enable_spread_spectrum(rdev, true); + + if (pi->thermal_protection) + rv770_enable_thermal_protection(rdev, true); + + rv770_program_mpll_timing_parameters(rdev); + rv770_setup_bsp(rdev); + rv770_program_git(rdev); + rv770_program_tp(rdev); + rv770_program_tpp(rdev); + rv770_program_sstp(rdev); + rv770_program_engine_speed_parameters(rdev); + rv770_enable_display_gap(rdev); + rv770_program_vc(rdev); + + if (pi->dynamic_pcie_gen2) + rv770_enable_dynamic_pcie_gen2(rdev, true); + + ret = rv770_upload_firmware(rdev); + if (ret) { + DRM_ERROR("rv770_upload_firmware failed\n"); + return ret; + } + ret = rv770_init_smc_table(rdev, boot_ps); + if (ret) { + DRM_ERROR("rv770_init_smc_table failed\n"); + return ret; + } + + rv770_program_response_times(rdev); + r7xx_start_smc(rdev); + + if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + rv730_start_dpm(rdev); + else + rv770_start_dpm(rdev); + + if (pi->gfx_clock_gating) + rv770_gfx_clock_gating_enable(rdev, true); + + if (pi->mg_clock_gating) + rv770_mg_clock_gating_enable(rdev, true); + + rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + + return 0; +} + +int rv770_dpm_late_enable(struct radeon_device *rdev) +{ + int ret; + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + PPSMC_Result result; + + ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) + return ret; + rdev->irq.dpm_thermal = true; + radeon_irq_set(rdev); + result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt); + + if (result != PPSMC_Result_OK) + DRM_DEBUG_KMS("Could not enable thermal interrupts.\n"); + } + + return 0; +} + +void rv770_dpm_disable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (!rv770_dpm_enabled(rdev)) + return; + + rv770_clear_vc(rdev); + + if (pi->thermal_protection) + rv770_enable_thermal_protection(rdev, false); + + rv770_enable_spread_spectrum(rdev, false); + + if (pi->dynamic_pcie_gen2) + rv770_enable_dynamic_pcie_gen2(rdev, false); + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + rdev->irq.dpm_thermal = false; + radeon_irq_set(rdev); + } + + if (pi->gfx_clock_gating) + rv770_gfx_clock_gating_enable(rdev, false); + + if (pi->mg_clock_gating) + rv770_mg_clock_gating_enable(rdev, false); + + if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) + rv730_stop_dpm(rdev); + else + rv770_stop_dpm(rdev); + + r7xx_stop_smc(rdev); + rv770_reset_smio_status(rdev); +} + +int rv770_dpm_set_power_state(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps; + struct radeon_ps *old_ps = rdev->pm.dpm.current_ps; + int ret; + + ret = rv770_restrict_performance_levels_before_switch(rdev); + if (ret) { + DRM_ERROR("rv770_restrict_performance_levels_before_switch failed\n"); + return ret; + } + rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps); + ret = rv770_halt_smc(rdev); + if (ret) { + DRM_ERROR("rv770_halt_smc failed\n"); + return ret; + } + ret = rv770_upload_sw_state(rdev, new_ps); + if (ret) { + DRM_ERROR("rv770_upload_sw_state failed\n"); + return ret; + } + r7xx_program_memory_timing_parameters(rdev, new_ps); + if (pi->dcodt) + rv770_program_dcodt_before_state_switch(rdev, new_ps, old_ps); + ret = rv770_resume_smc(rdev); + if (ret) { + DRM_ERROR("rv770_resume_smc failed\n"); + return ret; + } + ret = rv770_set_sw_state(rdev); + if (ret) { + DRM_ERROR("rv770_set_sw_state failed\n"); + return ret; + } + if (pi->dcodt) + rv770_program_dcodt_after_state_switch(rdev, new_ps, old_ps); + rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps); + + return 0; +} + +#if 0 +void rv770_dpm_reset_asic(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + + rv770_restrict_performance_levels_before_switch(rdev); + if (pi->dcodt) + rv770_program_dcodt_before_state_switch(rdev, boot_ps, boot_ps); + rv770_set_boot_state(rdev); + if (pi->dcodt) + rv770_program_dcodt_after_state_switch(rdev, boot_ps, boot_ps); +} +#endif + +void rv770_dpm_setup_asic(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + r7xx_read_clock_registers(rdev); + rv770_read_voltage_smio_registers(rdev); + rv770_get_memory_type(rdev); + if (pi->dcodt) + rv770_get_mclk_odt_threshold(rdev); + rv770_get_pcie_gen2_status(rdev); + + rv770_enable_acpi_pm(rdev); + + if (radeon_aspm != 0) { + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s) + rv770_enable_l0s(rdev); + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1) + rv770_enable_l1(rdev); + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1) + rv770_enable_pll_sleep_in_l1(rdev); + } +} + +void rv770_dpm_display_configuration_changed(struct radeon_device *rdev) +{ + rv770_program_display_gap(rdev); +} + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; +}; + +union pplib_clock_info { + struct _ATOM_PPLIB_R600_CLOCK_INFO r600; + struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; + struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; + struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; +}; + +union pplib_power_state { + struct _ATOM_PPLIB_STATE v1; + struct _ATOM_PPLIB_STATE_V2 v2; +}; + +static void rv7xx_parse_pplib_non_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info, + u8 table_rev) +{ + rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); + rps->class = le16_to_cpu(non_clock_info->usClassification); + rps->class2 = le16_to_cpu(non_clock_info->usClassification2); + + if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) { + rps->vclk = le32_to_cpu(non_clock_info->ulVCLK); + rps->dclk = le32_to_cpu(non_clock_info->ulDCLK); + } else { + rps->vclk = 0; + rps->dclk = 0; + } + + if (r600_is_uvd_state(rps->class, rps->class2)) { + if ((rps->vclk == 0) || (rps->dclk == 0)) { + rps->vclk = RV770_DEFAULT_VCLK_FREQ; + rps->dclk = RV770_DEFAULT_DCLK_FREQ; + } + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) + rdev->pm.dpm.boot_ps = rps; + if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + rdev->pm.dpm.uvd_ps = rps; +} + +static void rv7xx_parse_pplib_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, int index, + union pplib_clock_info *clock_info) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct rv7xx_ps *ps = rv770_get_ps(rps); + u32 sclk, mclk; + struct rv7xx_pl *pl; + + switch (index) { + case 0: + pl = &ps->low; + break; + case 1: + pl = &ps->medium; + break; + case 2: + default: + pl = &ps->high; + break; + } + + if (rdev->family >= CHIP_CEDAR) { + sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow); + sclk |= clock_info->evergreen.ucEngineClockHigh << 16; + mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow); + mclk |= clock_info->evergreen.ucMemoryClockHigh << 16; + + pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC); + pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI); + pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags); + } else { + sclk = le16_to_cpu(clock_info->r600.usEngineClockLow); + sclk |= clock_info->r600.ucEngineClockHigh << 16; + mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow); + mclk |= clock_info->r600.ucMemoryClockHigh << 16; + + pl->vddc = le16_to_cpu(clock_info->r600.usVDDC); + pl->flags = le32_to_cpu(clock_info->r600.ulFlags); + } + + pl->mclk = mclk; + pl->sclk = sclk; + + /* patch up vddc if necessary */ + if (pl->vddc == 0xff01) { + if (pi->max_vddc) + pl->vddc = pi->max_vddc; + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) { + pi->acpi_vddc = pl->vddc; + if (rdev->family >= CHIP_CEDAR) + eg_pi->acpi_vddci = pl->vddci; + if (ps->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) + pi->acpi_pcie_gen2 = true; + else + pi->acpi_pcie_gen2 = false; + } + + if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) { + if (rdev->family >= CHIP_BARTS) { + eg_pi->ulv.supported = true; + eg_pi->ulv.pl = pl; + } + } + + if (pi->min_vddc_in_table > pl->vddc) + pi->min_vddc_in_table = pl->vddc; + + if (pi->max_vddc_in_table < pl->vddc) + pi->max_vddc_in_table = pl->vddc; + + /* patch up boot state */ + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { + u16 vddc, vddci, mvdd; + radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd); + pl->mclk = rdev->clock.default_mclk; + pl->sclk = rdev->clock.default_sclk; + pl->vddc = vddc; + pl->vddci = vddci; + } + + if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == + ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) { + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci; + } +} + +int rv7xx_parse_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; + union pplib_power_state *power_state; + int i, j; + union pplib_clock_info *clock_info; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + struct rv7xx_ps *ps; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) * + power_info->pplib.ucNumStates, GFP_KERNEL); + if (!rdev->pm.dpm.ps) + return -ENOMEM; + + for (i = 0; i < power_info->pplib.ucNumStates; i++) { + power_state = (union pplib_power_state *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usStateArrayOffset) + + i * power_info->pplib.ucStateEntrySize); + non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) + + (power_state->v1.ucNonClockStateIndex * + power_info->pplib.ucNonClockSize)); + if (power_info->pplib.ucStateEntrySize - 1) { + u8 *idx; + ps = kzalloc(sizeof(struct rv7xx_ps), GFP_KERNEL); + if (ps == NULL) { + kfree(rdev->pm.dpm.ps); + return -ENOMEM; + } + rdev->pm.dpm.ps[i].ps_priv = ps; + rv7xx_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i], + non_clock_info, + power_info->pplib.ucNonClockSize); + idx = (u8 *)&power_state->v1.ucClockStateIndices[0]; + for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) { + clock_info = (union pplib_clock_info *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) + + (idx[j] * power_info->pplib.ucClockInfoSize)); + rv7xx_parse_pplib_clock_info(rdev, + &rdev->pm.dpm.ps[i], j, + clock_info); + } + } + } + rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates; + return 0; +} + +void rv770_get_engine_memory_ss(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct radeon_atom_ss ss; + + pi->sclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_ENGINE_SS, 0); + pi->mclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_MEMORY_SS, 0); + + if (pi->sclk_ss || pi->mclk_ss) + pi->dynamic_ss = true; + else + pi->dynamic_ss = false; +} + +int rv770_dpm_init(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi; + struct atom_clock_dividers dividers; + int ret; + + pi = kzalloc(sizeof(struct rv7xx_power_info), GFP_KERNEL); + if (pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = pi; + + rv770_get_max_vddc(rdev); + + pi->acpi_vddc = 0; + pi->min_vddc_in_table = 0; + pi->max_vddc_in_table = 0; + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = rv7xx_parse_power_table(rdev); + if (ret) + return ret; + + if (rdev->pm.dpm.voltage_response_time == 0) + rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT; + if (rdev->pm.dpm.backbias_response_time == 0) + rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + 0, false, ÷rs); + if (ret) + pi->ref_div = dividers.ref_div + 1; + else + pi->ref_div = R600_REFERENCEDIVIDER_DFLT; + + pi->mclk_strobe_mode_threshold = 30000; + pi->mclk_edc_enable_threshold = 30000; + + pi->rlp = RV770_RLP_DFLT; + pi->rmp = RV770_RMP_DFLT; + pi->lhp = RV770_LHP_DFLT; + pi->lmp = RV770_LMP_DFLT; + + pi->voltage_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0); + + pi->mvdd_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0); + + rv770_get_engine_memory_ss(rdev); + + pi->asi = RV770_ASI_DFLT; + pi->pasi = RV770_HASI_DFLT; + pi->vrc = RV770_VRC_DFLT; + + pi->power_gating = false; + + pi->gfx_clock_gating = true; + + pi->mg_clock_gating = true; + pi->mgcgtssm = true; + + pi->dynamic_pcie_gen2 = true; + + if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE) + pi->thermal_protection = true; + else + pi->thermal_protection = false; + + pi->display_gap = true; + + if (rdev->flags & RADEON_IS_MOBILITY) + pi->dcodt = true; + else + pi->dcodt = false; + + pi->ulps = true; + + pi->mclk_stutter_mode_threshold = 0; + + pi->sram_end = SMC_RAM_END; + pi->state_table_start = RV770_SMC_TABLE_ADDRESS; + pi->soft_regs_start = RV770_SMC_SOFT_REGISTERS_START; + + return 0; +} + +void rv770_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct rv7xx_ps *ps = rv770_get_ps(rps); + struct rv7xx_pl *pl; + + r600_dpm_print_class_info(rps->class, rps->class2); + r600_dpm_print_cap_info(rps->caps); + printk("\tuvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + if (rdev->family >= CHIP_CEDAR) { + pl = &ps->low; + printk("\t\tpower level 0 sclk: %u mclk: %u vddc: %u vddci: %u\n", + pl->sclk, pl->mclk, pl->vddc, pl->vddci); + pl = &ps->medium; + printk("\t\tpower level 1 sclk: %u mclk: %u vddc: %u vddci: %u\n", + pl->sclk, pl->mclk, pl->vddc, pl->vddci); + pl = &ps->high; + printk("\t\tpower level 2 sclk: %u mclk: %u vddc: %u vddci: %u\n", + pl->sclk, pl->mclk, pl->vddc, pl->vddci); + } else { + pl = &ps->low; + printk("\t\tpower level 0 sclk: %u mclk: %u vddc: %u\n", + pl->sclk, pl->mclk, pl->vddc); + pl = &ps->medium; + printk("\t\tpower level 1 sclk: %u mclk: %u vddc: %u\n", + pl->sclk, pl->mclk, pl->vddc); + pl = &ps->high; + printk("\t\tpower level 2 sclk: %u mclk: %u vddc: %u\n", + pl->sclk, pl->mclk, pl->vddc); + } + r600_dpm_print_ps_status(rdev, rps); +} + +void rv770_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct radeon_ps *rps = rdev->pm.dpm.current_ps; + struct rv7xx_ps *ps = rv770_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >> + CURRENT_PROFILE_INDEX_SHIFT; + + if (current_index > 2) { + seq_printf(m, "invalid dpm profile %d\n", current_index); + } else { + if (current_index == 0) + pl = &ps->low; + else if (current_index == 1) + pl = &ps->medium; + else /* current_index == 2 */ + pl = &ps->high; + seq_printf(m, "uvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + if (rdev->family >= CHIP_CEDAR) { + seq_printf(m, "power level %d sclk: %u mclk: %u vddc: %u vddci: %u\n", + current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci); + } else { + seq_printf(m, "power level %d sclk: %u mclk: %u vddc: %u\n", + current_index, pl->sclk, pl->mclk, pl->vddc); + } + } +} + +u32 rv770_dpm_get_current_sclk(struct radeon_device *rdev) +{ + struct radeon_ps *rps = rdev->pm.dpm.current_ps; + struct rv7xx_ps *ps = rv770_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >> + CURRENT_PROFILE_INDEX_SHIFT; + + if (current_index > 2) { + return 0; + } else { + if (current_index == 0) + pl = &ps->low; + else if (current_index == 1) + pl = &ps->medium; + else /* current_index == 2 */ + pl = &ps->high; + return pl->sclk; + } +} + +u32 rv770_dpm_get_current_mclk(struct radeon_device *rdev) +{ + struct radeon_ps *rps = rdev->pm.dpm.current_ps; + struct rv7xx_ps *ps = rv770_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >> + CURRENT_PROFILE_INDEX_SHIFT; + + if (current_index > 2) { + return 0; + } else { + if (current_index == 0) + pl = &ps->low; + else if (current_index == 1) + pl = &ps->medium; + else /* current_index == 2 */ + pl = &ps->high; + return pl->mclk; + } +} + +void rv770_dpm_fini(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); +} + +u32 rv770_dpm_get_sclk(struct radeon_device *rdev, bool low) +{ + struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps); + + if (low) + return requested_state->low.sclk; + else + return requested_state->high.sclk; +} + +u32 rv770_dpm_get_mclk(struct radeon_device *rdev, bool low) +{ + struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps); + + if (low) + return requested_state->low.mclk; + else + return requested_state->high.mclk; +} + +bool rv770_dpm_vblank_too_short(struct radeon_device *rdev) +{ + u32 vblank_time = r600_dpm_get_vblank_time(rdev); + u32 switch_limit = 200; /* 300 */ + + /* RV770 */ + /* mclk switching doesn't seem to work reliably on desktop RV770s */ + if ((rdev->family == CHIP_RV770) && + !(rdev->flags & RADEON_IS_MOBILITY)) + switch_limit = 0xffffffff; /* disable mclk switching */ + + if (vblank_time < switch_limit) + return true; + else + return false; + +} diff --git a/sys/dev/pci/drm/radeon/rv770_dpm.h b/sys/dev/pci/drm/radeon/rv770_dpm.h new file mode 100644 index 00000000000..d12beab7f3e --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv770_dpm.h @@ -0,0 +1,284 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __RV770_DPM_H__ +#define __RV770_DPM_H__ + +#include "rv770_smc.h" + +struct rv770_clock_registers { + u32 cg_spll_func_cntl; + u32 cg_spll_func_cntl_2; + u32 cg_spll_func_cntl_3; + u32 cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2; + u32 mpll_ad_func_cntl; + u32 mpll_ad_func_cntl_2; + u32 mpll_dq_func_cntl; + u32 mpll_dq_func_cntl_2; + u32 mclk_pwrmgt_cntl; + u32 dll_cntl; + u32 mpll_ss1; + u32 mpll_ss2; +}; + +struct rv730_clock_registers { + u32 cg_spll_func_cntl; + u32 cg_spll_func_cntl_2; + u32 cg_spll_func_cntl_3; + u32 cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2; + u32 mclk_pwrmgt_cntl; + u32 dll_cntl; + u32 mpll_func_cntl; + u32 mpll_func_cntl2; + u32 mpll_func_cntl3; + u32 mpll_ss; + u32 mpll_ss2; +}; + +union r7xx_clock_registers { + struct rv770_clock_registers rv770; + struct rv730_clock_registers rv730; +}; + +struct vddc_table_entry { + u16 vddc; + u8 vddc_index; + u8 high_smio; + u32 low_smio; +}; + +#define MAX_NO_OF_MVDD_VALUES 2 +#define MAX_NO_VREG_STEPS 32 + +struct rv7xx_power_info { + /* flags */ + bool mem_gddr5; + bool pcie_gen2; + bool dynamic_pcie_gen2; + bool acpi_pcie_gen2; + bool boot_in_gen2; + bool voltage_control; /* vddc */ + bool mvdd_control; + bool sclk_ss; + bool mclk_ss; + bool dynamic_ss; + bool gfx_clock_gating; + bool mg_clock_gating; + bool mgcgtssm; + bool power_gating; + bool thermal_protection; + bool display_gap; + bool dcodt; + bool ulps; + /* registers */ + union r7xx_clock_registers clk_regs; + u32 s0_vid_lower_smio_cntl; + /* voltage */ + u32 vddc_mask_low; + u32 mvdd_mask_low; + u32 mvdd_split_frequency; + u32 mvdd_low_smio[MAX_NO_OF_MVDD_VALUES]; + u16 max_vddc; + u16 max_vddc_in_table; + u16 min_vddc_in_table; + struct vddc_table_entry vddc_table[MAX_NO_VREG_STEPS]; + u8 valid_vddc_entries; + /* dc odt */ + u32 mclk_odt_threshold; + u8 odt_value_0[2]; + u8 odt_value_1[2]; + /* stored values */ + u32 boot_sclk; + u16 acpi_vddc; + u32 ref_div; + u32 active_auto_throttle_sources; + u32 mclk_stutter_mode_threshold; + u32 mclk_strobe_mode_threshold; + u32 mclk_edc_enable_threshold; + u32 bsp; + u32 bsu; + u32 pbsp; + u32 pbsu; + u32 dsp; + u32 psp; + u32 asi; + u32 pasi; + u32 vrc; + u32 restricted_levels; + u32 rlp; + u32 rmp; + u32 lhp; + u32 lmp; + /* smc offsets */ + u16 state_table_start; + u16 soft_regs_start; + u16 sram_end; + /* scratch structs */ + RV770_SMC_STATETABLE smc_statetable; +}; + +struct rv7xx_pl { + u32 sclk; + u32 mclk; + u16 vddc; + u16 vddci; /* eg+ only */ + u32 flags; + enum radeon_pcie_gen pcie_gen; /* si+ only */ +}; + +struct rv7xx_ps { + struct rv7xx_pl high; + struct rv7xx_pl medium; + struct rv7xx_pl low; + bool dc_compatible; +}; + +#define RV770_RLP_DFLT 10 +#define RV770_RMP_DFLT 25 +#define RV770_LHP_DFLT 25 +#define RV770_LMP_DFLT 10 +#define RV770_VRC_DFLT 0x003f +#define RV770_ASI_DFLT 1000 +#define RV770_HASI_DFLT 200000 +#define RV770_MGCGTTLOCAL0_DFLT 0x00100000 +#define RV7XX_MGCGTTLOCAL0_DFLT 0 +#define RV770_MGCGTTLOCAL1_DFLT 0xFFFF0000 +#define RV770_MGCGCGTSSMCTRL_DFLT 0x55940000 + +#define MVDD_LOW_INDEX 0 +#define MVDD_HIGH_INDEX 1 + +#define MVDD_LOW_VALUE 0 +#define MVDD_HIGH_VALUE 0xffff + +#define RV770_DEFAULT_VCLK_FREQ 53300 /* 10 khz */ +#define RV770_DEFAULT_DCLK_FREQ 40000 /* 10 khz */ + +/* rv730/rv710 */ +int rv730_populate_sclk_value(struct radeon_device *rdev, + u32 engine_clock, + RV770_SMC_SCLK_VALUE *sclk); +int rv730_populate_mclk_value(struct radeon_device *rdev, + u32 engine_clock, u32 memory_clock, + LPRV7XX_SMC_MCLK_VALUE mclk); +void rv730_read_clock_registers(struct radeon_device *rdev); +int rv730_populate_smc_acpi_state(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table); +int rv730_populate_smc_initial_state(struct radeon_device *rdev, + struct radeon_ps *radeon_initial_state, + RV770_SMC_STATETABLE *table); +void rv730_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_state); +void rv730_power_gating_enable(struct radeon_device *rdev, + bool enable); +void rv730_start_dpm(struct radeon_device *rdev); +void rv730_stop_dpm(struct radeon_device *rdev); +void rv730_program_dcodt(struct radeon_device *rdev, bool use_dcodt); +void rv730_get_odt_values(struct radeon_device *rdev); + +/* rv740 */ +int rv740_populate_sclk_value(struct radeon_device *rdev, u32 engine_clock, + RV770_SMC_SCLK_VALUE *sclk); +int rv740_populate_mclk_value(struct radeon_device *rdev, + u32 engine_clock, u32 memory_clock, + RV7XX_SMC_MCLK_VALUE *mclk); +void rv740_read_clock_registers(struct radeon_device *rdev); +int rv740_populate_smc_acpi_state(struct radeon_device *rdev, + RV770_SMC_STATETABLE *table); +void rv740_enable_mclk_spread_spectrum(struct radeon_device *rdev, + bool enable); +u8 rv740_get_mclk_frequency_ratio(u32 memory_clock); +u32 rv740_get_dll_speed(bool is_gddr5, u32 memory_clock); +u32 rv740_get_decoded_reference_divider(u32 encoded_ref); + +/* rv770 */ +u32 rv770_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf); +int rv770_populate_vddc_value(struct radeon_device *rdev, u16 vddc, + RV770_SMC_VOLTAGE_VALUE *voltage); +int rv770_populate_mvdd_value(struct radeon_device *rdev, u32 mclk, + RV770_SMC_VOLTAGE_VALUE *voltage); +u8 rv770_get_seq_value(struct radeon_device *rdev, + struct rv7xx_pl *pl); +int rv770_populate_initial_mvdd_value(struct radeon_device *rdev, + RV770_SMC_VOLTAGE_VALUE *voltage); +u32 rv770_calculate_memory_refresh_rate(struct radeon_device *rdev, + u32 engine_clock); +void rv770_program_response_times(struct radeon_device *rdev); +int rv770_populate_smc_sp(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + RV770_SMC_SWSTATE *smc_state); +int rv770_populate_smc_t(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + RV770_SMC_SWSTATE *smc_state); +void rv770_read_voltage_smio_registers(struct radeon_device *rdev); +void rv770_get_memory_type(struct radeon_device *rdev); +void r7xx_start_smc(struct radeon_device *rdev); +u8 rv770_get_memory_module_index(struct radeon_device *rdev); +void rv770_get_max_vddc(struct radeon_device *rdev); +void rv770_get_pcie_gen2_status(struct radeon_device *rdev); +void rv770_enable_acpi_pm(struct radeon_device *rdev); +void rv770_restore_cgcg(struct radeon_device *rdev); +bool rv770_dpm_enabled(struct radeon_device *rdev); +void rv770_enable_voltage_control(struct radeon_device *rdev, + bool enable); +void rv770_enable_backbias(struct radeon_device *rdev, + bool enable); +void rv770_enable_thermal_protection(struct radeon_device *rdev, + bool enable); +void rv770_enable_auto_throttle_source(struct radeon_device *rdev, + enum radeon_dpm_auto_throttle_src source, + bool enable); +void rv770_setup_bsp(struct radeon_device *rdev); +void rv770_program_git(struct radeon_device *rdev); +void rv770_program_tp(struct radeon_device *rdev); +void rv770_program_tpp(struct radeon_device *rdev); +void rv770_program_sstp(struct radeon_device *rdev); +void rv770_program_engine_speed_parameters(struct radeon_device *rdev); +void rv770_program_vc(struct radeon_device *rdev); +void rv770_clear_vc(struct radeon_device *rdev); +int rv770_upload_firmware(struct radeon_device *rdev); +void rv770_stop_dpm(struct radeon_device *rdev); +void r7xx_stop_smc(struct radeon_device *rdev); +void rv770_reset_smio_status(struct radeon_device *rdev); +int rv770_restrict_performance_levels_before_switch(struct radeon_device *rdev); +int rv770_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level); +int rv770_halt_smc(struct radeon_device *rdev); +int rv770_resume_smc(struct radeon_device *rdev); +int rv770_set_sw_state(struct radeon_device *rdev); +int rv770_set_boot_state(struct radeon_device *rdev); +int rv7xx_parse_power_table(struct radeon_device *rdev); +void rv770_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps); +void rv770_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_ps, + struct radeon_ps *old_ps); +void rv770_get_engine_memory_ss(struct radeon_device *rdev); + +/* smc */ +int rv770_write_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 value); + +#endif diff --git a/sys/dev/pci/drm/radeon/rv770_smc.c b/sys/dev/pci/drm/radeon/rv770_smc.c new file mode 100644 index 00000000000..7ac88f8389f --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv770_smc.c @@ -0,0 +1,630 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "rv770d.h" +#include "rv770_dpm.h" +#include "rv770_smc.h" +#include "atom.h" +#include "radeon_ucode.h" + +#define FIRST_SMC_INT_VECT_REG 0xFFD8 +#define FIRST_INT_VECT_S19 0xFFC0 + +static const u8 rv770_smc_int_vectors[] = +{ + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x0C, 0xD7, + 0x08, 0x2B, 0x08, 0x10, + 0x03, 0x51, 0x03, 0x51, + 0x03, 0x51, 0x03, 0x51 +}; + +static const u8 rv730_smc_int_vectors[] = +{ + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x08, 0x15, + 0x08, 0x15, 0x0C, 0xBB, + 0x08, 0x30, 0x08, 0x15, + 0x03, 0x56, 0x03, 0x56, + 0x03, 0x56, 0x03, 0x56 +}; + +static const u8 rv710_smc_int_vectors[] = +{ + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x08, 0x04, + 0x08, 0x04, 0x0C, 0xCB, + 0x08, 0x1F, 0x08, 0x04, + 0x03, 0x51, 0x03, 0x51, + 0x03, 0x51, 0x03, 0x51 +}; + +static const u8 rv740_smc_int_vectors[] = +{ + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x08, 0x10, + 0x08, 0x10, 0x0C, 0xD7, + 0x08, 0x2B, 0x08, 0x10, + 0x03, 0x51, 0x03, 0x51, + 0x03, 0x51, 0x03, 0x51 +}; + +static const u8 cedar_smc_int_vectors[] = +{ + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x11, 0x8B, + 0x0B, 0x20, 0x0B, 0x05, + 0x04, 0xF6, 0x04, 0xF6, + 0x04, 0xF6, 0x04, 0xF6 +}; + +static const u8 redwood_smc_int_vectors[] = +{ + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x11, 0x8B, + 0x0B, 0x20, 0x0B, 0x05, + 0x04, 0xF6, 0x04, 0xF6, + 0x04, 0xF6, 0x04, 0xF6 +}; + +static const u8 juniper_smc_int_vectors[] = +{ + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x11, 0x8B, + 0x0B, 0x20, 0x0B, 0x05, + 0x04, 0xF6, 0x04, 0xF6, + 0x04, 0xF6, 0x04, 0xF6 +}; + +static const u8 cypress_smc_int_vectors[] = +{ + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x0B, 0x05, + 0x0B, 0x05, 0x11, 0x8B, + 0x0B, 0x20, 0x0B, 0x05, + 0x04, 0xF6, 0x04, 0xF6, + 0x04, 0xF6, 0x04, 0xF6 +}; + +static const u8 barts_smc_int_vectors[] = +{ + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x12, 0xAA, + 0x0C, 0x2F, 0x15, 0xF6, + 0x15, 0xF6, 0x05, 0x0A, + 0x05, 0x0A, 0x05, 0x0A +}; + +static const u8 turks_smc_int_vectors[] = +{ + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x12, 0xAA, + 0x0C, 0x2F, 0x15, 0xF6, + 0x15, 0xF6, 0x05, 0x0A, + 0x05, 0x0A, 0x05, 0x0A +}; + +static const u8 caicos_smc_int_vectors[] = +{ + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x0C, 0x14, + 0x0C, 0x14, 0x12, 0xAA, + 0x0C, 0x2F, 0x15, 0xF6, + 0x15, 0xF6, 0x05, 0x0A, + 0x05, 0x0A, 0x05, 0x0A +}; + +static const u8 cayman_smc_int_vectors[] = +{ + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x12, 0x05, + 0x12, 0x05, 0x18, 0xEA, + 0x12, 0x20, 0x1C, 0x34, + 0x1C, 0x34, 0x08, 0x72, + 0x08, 0x72, 0x08, 0x72 +}; + +static int rv770_set_smc_sram_address(struct radeon_device *rdev, + u16 smc_address, u16 limit) +{ + u32 addr; + + if (smc_address & 3) + return -EINVAL; + if ((smc_address + 3) > limit) + return -EINVAL; + + addr = smc_address; + addr |= SMC_SRAM_AUTO_INC_DIS; + + WREG32(SMC_SRAM_ADDR, addr); + + return 0; +} + +int rv770_copy_bytes_to_smc(struct radeon_device *rdev, + u16 smc_start_address, const u8 *src, + u16 byte_count, u16 limit) +{ + unsigned long flags; + u32 data, original_data, extra_shift; + u16 addr; + int ret = 0; + + if (smc_start_address & 3) + return -EINVAL; + if ((smc_start_address + byte_count) > limit) + return -EINVAL; + + addr = smc_start_address; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + while (byte_count >= 4) { + /* SMC address space is BE */ + data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3]; + + ret = rv770_set_smc_sram_address(rdev, addr, limit); + if (ret) + goto done; + + WREG32(SMC_SRAM_DATA, data); + + src += 4; + byte_count -= 4; + addr += 4; + } + + /* RMW for final bytes */ + if (byte_count > 0) { + data = 0; + + ret = rv770_set_smc_sram_address(rdev, addr, limit); + if (ret) + goto done; + + original_data = RREG32(SMC_SRAM_DATA); + + extra_shift = 8 * (4 - byte_count); + + while (byte_count > 0) { + /* SMC address space is BE */ + data = (data << 8) + *src++; + byte_count--; + } + + data <<= extra_shift; + + data |= (original_data & ~((~0UL) << extra_shift)); + + ret = rv770_set_smc_sram_address(rdev, addr, limit); + if (ret) + goto done; + + WREG32(SMC_SRAM_DATA, data); + } + +done: + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return ret; +} + +static int rv770_program_interrupt_vectors(struct radeon_device *rdev, + u32 smc_first_vector, const u8 *src, + u32 byte_count) +{ + u32 tmp, i; + + if (byte_count % 4) + return -EINVAL; + + if (smc_first_vector < FIRST_SMC_INT_VECT_REG) { + tmp = FIRST_SMC_INT_VECT_REG - smc_first_vector; + + if (tmp > byte_count) + return 0; + + byte_count -= tmp; + src += tmp; + smc_first_vector = FIRST_SMC_INT_VECT_REG; + } + + for (i = 0; i < byte_count; i += 4) { + /* SMC address space is BE */ + tmp = (src[i] << 24) | (src[i + 1] << 16) | (src[i + 2] << 8) | src[i + 3]; + + WREG32(SMC_ISR_FFD8_FFDB + i, tmp); + } + + return 0; +} + +void rv770_start_smc(struct radeon_device *rdev) +{ + WREG32_P(SMC_IO, SMC_RST_N, ~SMC_RST_N); +} + +void rv770_reset_smc(struct radeon_device *rdev) +{ + WREG32_P(SMC_IO, 0, ~SMC_RST_N); +} + +void rv770_stop_smc_clock(struct radeon_device *rdev) +{ + WREG32_P(SMC_IO, 0, ~SMC_CLK_EN); +} + +void rv770_start_smc_clock(struct radeon_device *rdev) +{ + WREG32_P(SMC_IO, SMC_CLK_EN, ~SMC_CLK_EN); +} + +bool rv770_is_smc_running(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32(SMC_IO); + + if ((tmp & SMC_RST_N) && (tmp & SMC_CLK_EN)) + return true; + else + return false; +} + +PPSMC_Result rv770_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg) +{ + u32 tmp; + int i; + PPSMC_Result result; + + if (!rv770_is_smc_running(rdev)) + return PPSMC_Result_Failed; + + WREG32_P(SMC_MSG, HOST_SMC_MSG(msg), ~HOST_SMC_MSG_MASK); + + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = RREG32(SMC_MSG) & HOST_SMC_RESP_MASK; + tmp >>= HOST_SMC_RESP_SHIFT; + if (tmp != 0) + break; + udelay(1); + } + + tmp = RREG32(SMC_MSG) & HOST_SMC_RESP_MASK; + tmp >>= HOST_SMC_RESP_SHIFT; + + result = (PPSMC_Result)tmp; + return result; +} + +PPSMC_Result rv770_wait_for_smc_inactive(struct radeon_device *rdev) +{ + int i; + PPSMC_Result result = PPSMC_Result_OK; + + if (!rv770_is_smc_running(rdev)) + return result; + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(SMC_IO) & SMC_STOP_MODE) + break; + udelay(1); + } + + return result; +} + +static void rv770_clear_smc_sram(struct radeon_device *rdev, u16 limit) +{ + unsigned long flags; + u16 i; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + for (i = 0; i < limit; i += 4) { + rv770_set_smc_sram_address(rdev, i, limit); + WREG32(SMC_SRAM_DATA, 0); + } + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); +} + +int rv770_load_smc_ucode(struct radeon_device *rdev, + u16 limit) +{ + int ret; + const u8 *int_vect; + u16 int_vect_start_address; + u16 int_vect_size; + const u8 *ucode_data; + u16 ucode_start_address; + u16 ucode_size; + + if (!rdev->smc_fw) + return -EINVAL; + + rv770_clear_smc_sram(rdev, limit); + + switch (rdev->family) { + case CHIP_RV770: + ucode_start_address = RV770_SMC_UCODE_START; + ucode_size = RV770_SMC_UCODE_SIZE; + int_vect = (const u8 *)&rv770_smc_int_vectors; + int_vect_start_address = RV770_SMC_INT_VECTOR_START; + int_vect_size = RV770_SMC_INT_VECTOR_SIZE; + break; + case CHIP_RV730: + ucode_start_address = RV730_SMC_UCODE_START; + ucode_size = RV730_SMC_UCODE_SIZE; + int_vect = (const u8 *)&rv730_smc_int_vectors; + int_vect_start_address = RV730_SMC_INT_VECTOR_START; + int_vect_size = RV730_SMC_INT_VECTOR_SIZE; + break; + case CHIP_RV710: + ucode_start_address = RV710_SMC_UCODE_START; + ucode_size = RV710_SMC_UCODE_SIZE; + int_vect = (const u8 *)&rv710_smc_int_vectors; + int_vect_start_address = RV710_SMC_INT_VECTOR_START; + int_vect_size = RV710_SMC_INT_VECTOR_SIZE; + break; + case CHIP_RV740: + ucode_start_address = RV740_SMC_UCODE_START; + ucode_size = RV740_SMC_UCODE_SIZE; + int_vect = (const u8 *)&rv740_smc_int_vectors; + int_vect_start_address = RV740_SMC_INT_VECTOR_START; + int_vect_size = RV740_SMC_INT_VECTOR_SIZE; + break; + case CHIP_CEDAR: + ucode_start_address = CEDAR_SMC_UCODE_START; + ucode_size = CEDAR_SMC_UCODE_SIZE; + int_vect = (const u8 *)&cedar_smc_int_vectors; + int_vect_start_address = CEDAR_SMC_INT_VECTOR_START; + int_vect_size = CEDAR_SMC_INT_VECTOR_SIZE; + break; + case CHIP_REDWOOD: + ucode_start_address = REDWOOD_SMC_UCODE_START; + ucode_size = REDWOOD_SMC_UCODE_SIZE; + int_vect = (const u8 *)&redwood_smc_int_vectors; + int_vect_start_address = REDWOOD_SMC_INT_VECTOR_START; + int_vect_size = REDWOOD_SMC_INT_VECTOR_SIZE; + break; + case CHIP_JUNIPER: + ucode_start_address = JUNIPER_SMC_UCODE_START; + ucode_size = JUNIPER_SMC_UCODE_SIZE; + int_vect = (const u8 *)&juniper_smc_int_vectors; + int_vect_start_address = JUNIPER_SMC_INT_VECTOR_START; + int_vect_size = JUNIPER_SMC_INT_VECTOR_SIZE; + break; + case CHIP_CYPRESS: + case CHIP_HEMLOCK: + ucode_start_address = CYPRESS_SMC_UCODE_START; + ucode_size = CYPRESS_SMC_UCODE_SIZE; + int_vect = (const u8 *)&cypress_smc_int_vectors; + int_vect_start_address = CYPRESS_SMC_INT_VECTOR_START; + int_vect_size = CYPRESS_SMC_INT_VECTOR_SIZE; + break; + case CHIP_BARTS: + ucode_start_address = BARTS_SMC_UCODE_START; + ucode_size = BARTS_SMC_UCODE_SIZE; + int_vect = (const u8 *)&barts_smc_int_vectors; + int_vect_start_address = BARTS_SMC_INT_VECTOR_START; + int_vect_size = BARTS_SMC_INT_VECTOR_SIZE; + break; + case CHIP_TURKS: + ucode_start_address = TURKS_SMC_UCODE_START; + ucode_size = TURKS_SMC_UCODE_SIZE; + int_vect = (const u8 *)&turks_smc_int_vectors; + int_vect_start_address = TURKS_SMC_INT_VECTOR_START; + int_vect_size = TURKS_SMC_INT_VECTOR_SIZE; + break; + case CHIP_CAICOS: + ucode_start_address = CAICOS_SMC_UCODE_START; + ucode_size = CAICOS_SMC_UCODE_SIZE; + int_vect = (const u8 *)&caicos_smc_int_vectors; + int_vect_start_address = CAICOS_SMC_INT_VECTOR_START; + int_vect_size = CAICOS_SMC_INT_VECTOR_SIZE; + break; + case CHIP_CAYMAN: + ucode_start_address = CAYMAN_SMC_UCODE_START; + ucode_size = CAYMAN_SMC_UCODE_SIZE; + int_vect = (const u8 *)&cayman_smc_int_vectors; + int_vect_start_address = CAYMAN_SMC_INT_VECTOR_START; + int_vect_size = CAYMAN_SMC_INT_VECTOR_SIZE; + break; + default: + DRM_ERROR("unknown asic in smc ucode loader\n"); + BUG(); + } + + /* load the ucode */ + ucode_data = (const u8 *)rdev->smc_fw->data; + ret = rv770_copy_bytes_to_smc(rdev, ucode_start_address, + ucode_data, ucode_size, limit); + if (ret) + return ret; + + /* set up the int vectors */ + ret = rv770_program_interrupt_vectors(rdev, int_vect_start_address, + int_vect, int_vect_size); + if (ret) + return ret; + + return 0; +} + +int rv770_read_smc_sram_dword(struct radeon_device *rdev, + u16 smc_address, u32 *value, u16 limit) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + ret = rv770_set_smc_sram_address(rdev, smc_address, limit); + if (ret == 0) + *value = RREG32(SMC_SRAM_DATA); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return ret; +} + +int rv770_write_smc_sram_dword(struct radeon_device *rdev, + u16 smc_address, u32 value, u16 limit) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + ret = rv770_set_smc_sram_address(rdev, smc_address, limit); + if (ret == 0) + WREG32(SMC_SRAM_DATA, value); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return ret; +} diff --git a/sys/dev/pci/drm/radeon/rv770_smc.h b/sys/dev/pci/drm/radeon/rv770_smc.h new file mode 100644 index 00000000000..3b2c963c488 --- /dev/null +++ b/sys/dev/pci/drm/radeon/rv770_smc.h @@ -0,0 +1,207 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __RV770_SMC_H__ +#define __RV770_SMC_H__ + +#include "ppsmc.h" + +#pragma pack(push, 1) + +#define RV770_SMC_TABLE_ADDRESS 0xB000 + +#define RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE 3 + +struct RV770_SMC_SCLK_VALUE +{ + uint32_t vCG_SPLL_FUNC_CNTL; + uint32_t vCG_SPLL_FUNC_CNTL_2; + uint32_t vCG_SPLL_FUNC_CNTL_3; + uint32_t vCG_SPLL_SPREAD_SPECTRUM; + uint32_t vCG_SPLL_SPREAD_SPECTRUM_2; + uint32_t sclk_value; +}; + +typedef struct RV770_SMC_SCLK_VALUE RV770_SMC_SCLK_VALUE; + +struct RV770_SMC_MCLK_VALUE +{ + uint32_t vMPLL_AD_FUNC_CNTL; + uint32_t vMPLL_AD_FUNC_CNTL_2; + uint32_t vMPLL_DQ_FUNC_CNTL; + uint32_t vMPLL_DQ_FUNC_CNTL_2; + uint32_t vMCLK_PWRMGT_CNTL; + uint32_t vDLL_CNTL; + uint32_t vMPLL_SS; + uint32_t vMPLL_SS2; + uint32_t mclk_value; +}; + +typedef struct RV770_SMC_MCLK_VALUE RV770_SMC_MCLK_VALUE; + + +struct RV730_SMC_MCLK_VALUE +{ + uint32_t vMCLK_PWRMGT_CNTL; + uint32_t vDLL_CNTL; + uint32_t vMPLL_FUNC_CNTL; + uint32_t vMPLL_FUNC_CNTL2; + uint32_t vMPLL_FUNC_CNTL3; + uint32_t vMPLL_SS; + uint32_t vMPLL_SS2; + uint32_t mclk_value; +}; + +typedef struct RV730_SMC_MCLK_VALUE RV730_SMC_MCLK_VALUE; + +struct RV770_SMC_VOLTAGE_VALUE +{ + uint16_t value; + uint8_t index; + uint8_t padding; +}; + +typedef struct RV770_SMC_VOLTAGE_VALUE RV770_SMC_VOLTAGE_VALUE; + +union RV7XX_SMC_MCLK_VALUE +{ + RV770_SMC_MCLK_VALUE mclk770; + RV730_SMC_MCLK_VALUE mclk730; +}; + +typedef union RV7XX_SMC_MCLK_VALUE RV7XX_SMC_MCLK_VALUE, *LPRV7XX_SMC_MCLK_VALUE; + +struct RV770_SMC_HW_PERFORMANCE_LEVEL +{ + uint8_t arbValue; + union{ + uint8_t seqValue; + uint8_t ACIndex; + }; + uint8_t displayWatermark; + uint8_t gen2PCIE; + uint8_t gen2XSP; + uint8_t backbias; + uint8_t strobeMode; + uint8_t mcFlags; + uint32_t aT; + uint32_t bSP; + RV770_SMC_SCLK_VALUE sclk; + RV7XX_SMC_MCLK_VALUE mclk; + RV770_SMC_VOLTAGE_VALUE vddc; + RV770_SMC_VOLTAGE_VALUE mvdd; + RV770_SMC_VOLTAGE_VALUE vddci; + uint8_t reserved1; + uint8_t reserved2; + uint8_t stateFlags; + uint8_t padding; +}; + +#define SMC_STROBE_RATIO 0x0F +#define SMC_STROBE_ENABLE 0x10 + +#define SMC_MC_EDC_RD_FLAG 0x01 +#define SMC_MC_EDC_WR_FLAG 0x02 +#define SMC_MC_RTT_ENABLE 0x04 +#define SMC_MC_STUTTER_EN 0x08 + +typedef struct RV770_SMC_HW_PERFORMANCE_LEVEL RV770_SMC_HW_PERFORMANCE_LEVEL; + +struct RV770_SMC_SWSTATE +{ + uint8_t flags; + uint8_t padding1; + uint8_t padding2; + uint8_t padding3; + RV770_SMC_HW_PERFORMANCE_LEVEL levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE]; +}; + +typedef struct RV770_SMC_SWSTATE RV770_SMC_SWSTATE; + +#define RV770_SMC_VOLTAGEMASK_VDDC 0 +#define RV770_SMC_VOLTAGEMASK_MVDD 1 +#define RV770_SMC_VOLTAGEMASK_VDDCI 2 +#define RV770_SMC_VOLTAGEMASK_MAX 4 + +struct RV770_SMC_VOLTAGEMASKTABLE +{ + uint8_t highMask[RV770_SMC_VOLTAGEMASK_MAX]; + uint32_t lowMask[RV770_SMC_VOLTAGEMASK_MAX]; +}; + +typedef struct RV770_SMC_VOLTAGEMASKTABLE RV770_SMC_VOLTAGEMASKTABLE; + +#define MAX_NO_VREG_STEPS 32 + +struct RV770_SMC_STATETABLE +{ + uint8_t thermalProtectType; + uint8_t systemFlags; + uint8_t maxVDDCIndexInPPTable; + uint8_t extraFlags; + uint8_t highSMIO[MAX_NO_VREG_STEPS]; + uint32_t lowSMIO[MAX_NO_VREG_STEPS]; + RV770_SMC_VOLTAGEMASKTABLE voltageMaskTable; + RV770_SMC_SWSTATE initialState; + RV770_SMC_SWSTATE ACPIState; + RV770_SMC_SWSTATE driverState; + RV770_SMC_SWSTATE ULVState; +}; + +typedef struct RV770_SMC_STATETABLE RV770_SMC_STATETABLE; + +#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP 0x01 + +#pragma pack(pop) + +#define RV770_SMC_SOFT_REGISTERS_START 0x104 + +#define RV770_SMC_SOFT_REGISTER_mclk_chg_timeout 0x0 +#define RV770_SMC_SOFT_REGISTER_baby_step_timer 0x8 +#define RV770_SMC_SOFT_REGISTER_delay_bbias 0xC +#define RV770_SMC_SOFT_REGISTER_delay_vreg 0x10 +#define RV770_SMC_SOFT_REGISTER_delay_acpi 0x2C +#define RV770_SMC_SOFT_REGISTER_seq_index 0x64 +#define RV770_SMC_SOFT_REGISTER_mvdd_chg_time 0x68 +#define RV770_SMC_SOFT_REGISTER_mclk_switch_lim 0x78 +#define RV770_SMC_SOFT_REGISTER_mc_block_delay 0x90 +#define RV770_SMC_SOFT_REGISTER_uvd_enabled 0x9C +#define RV770_SMC_SOFT_REGISTER_is_asic_lombok 0xA0 + +int rv770_copy_bytes_to_smc(struct radeon_device *rdev, + u16 smc_start_address, const u8 *src, + u16 byte_count, u16 limit); +void rv770_start_smc(struct radeon_device *rdev); +void rv770_reset_smc(struct radeon_device *rdev); +void rv770_stop_smc_clock(struct radeon_device *rdev); +void rv770_start_smc_clock(struct radeon_device *rdev); +bool rv770_is_smc_running(struct radeon_device *rdev); +PPSMC_Result rv770_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg); +PPSMC_Result rv770_wait_for_smc_inactive(struct radeon_device *rdev); +int rv770_read_smc_sram_dword(struct radeon_device *rdev, + u16 smc_address, u32 *value, u16 limit); +int rv770_write_smc_sram_dword(struct radeon_device *rdev, + u16 smc_address, u32 value, u16 limit); +int rv770_load_smc_ucode(struct radeon_device *rdev, + u16 limit); + +#endif diff --git a/sys/dev/pci/drm/radeon/rv770d.h b/sys/dev/pci/drm/radeon/rv770d.h index 338c540defc..9ef2064b1c9 100644 --- a/sys/dev/pci/drm/radeon/rv770d.h +++ b/sys/dev/pci/drm/radeon/rv770d.h @@ -1,4 +1,3 @@ -/* $OpenBSD: rv770d.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2009 Advanced Micro Devices, Inc. * Copyright 2009 Red Hat Inc. @@ -39,6 +38,277 @@ #define R7XX_MAX_PIPES 8 #define R7XX_MAX_PIPES_MASK 0xff +/* discrete uvd clocks */ +#define CG_UPLL_FUNC_CNTL 0x718 +# define UPLL_RESET_MASK 0x00000001 +# define UPLL_SLEEP_MASK 0x00000002 +# define UPLL_BYPASS_EN_MASK 0x00000004 +# define UPLL_CTLREQ_MASK 0x00000008 +# define UPLL_REF_DIV(x) ((x) << 16) +# define UPLL_REF_DIV_MASK 0x003F0000 +# define UPLL_CTLACK_MASK 0x40000000 +# define UPLL_CTLACK2_MASK 0x80000000 +#define CG_UPLL_FUNC_CNTL_2 0x71c +# define UPLL_SW_HILEN(x) ((x) << 0) +# define UPLL_SW_LOLEN(x) ((x) << 4) +# define UPLL_SW_HILEN2(x) ((x) << 8) +# define UPLL_SW_LOLEN2(x) ((x) << 12) +# define UPLL_SW_MASK 0x0000FFFF +# define VCLK_SRC_SEL(x) ((x) << 20) +# define VCLK_SRC_SEL_MASK 0x01F00000 +# define DCLK_SRC_SEL(x) ((x) << 25) +# define DCLK_SRC_SEL_MASK 0x3E000000 +#define CG_UPLL_FUNC_CNTL_3 0x720 +# define UPLL_FB_DIV(x) ((x) << 0) +# define UPLL_FB_DIV_MASK 0x01FFFFFF + +/* pm registers */ +#define SMC_SRAM_ADDR 0x200 +#define SMC_SRAM_AUTO_INC_DIS (1 << 16) +#define SMC_SRAM_DATA 0x204 +#define SMC_IO 0x208 +#define SMC_RST_N (1 << 0) +#define SMC_STOP_MODE (1 << 2) +#define SMC_CLK_EN (1 << 11) +#define SMC_MSG 0x20c +#define HOST_SMC_MSG(x) ((x) << 0) +#define HOST_SMC_MSG_MASK (0xff << 0) +#define HOST_SMC_MSG_SHIFT 0 +#define HOST_SMC_RESP(x) ((x) << 8) +#define HOST_SMC_RESP_MASK (0xff << 8) +#define HOST_SMC_RESP_SHIFT 8 +#define SMC_HOST_MSG(x) ((x) << 16) +#define SMC_HOST_MSG_MASK (0xff << 16) +#define SMC_HOST_MSG_SHIFT 16 +#define SMC_HOST_RESP(x) ((x) << 24) +#define SMC_HOST_RESP_MASK (0xff << 24) +#define SMC_HOST_RESP_SHIFT 24 + +#define SMC_ISR_FFD8_FFDB 0x218 + +#define CG_SPLL_FUNC_CNTL 0x600 +#define SPLL_RESET (1 << 0) +#define SPLL_SLEEP (1 << 1) +#define SPLL_DIVEN (1 << 2) +#define SPLL_BYPASS_EN (1 << 3) +#define SPLL_REF_DIV(x) ((x) << 4) +#define SPLL_REF_DIV_MASK (0x3f << 4) +#define SPLL_HILEN(x) ((x) << 12) +#define SPLL_HILEN_MASK (0xf << 12) +#define SPLL_LOLEN(x) ((x) << 16) +#define SPLL_LOLEN_MASK (0xf << 16) +#define CG_SPLL_FUNC_CNTL_2 0x604 +#define SCLK_MUX_SEL(x) ((x) << 0) +#define SCLK_MUX_SEL_MASK (0x1ff << 0) +#define SCLK_MUX_UPDATE (1 << 26) +#define CG_SPLL_FUNC_CNTL_3 0x608 +#define SPLL_FB_DIV(x) ((x) << 0) +#define SPLL_FB_DIV_MASK (0x3ffffff << 0) +#define SPLL_DITHEN (1 << 28) +#define CG_SPLL_STATUS 0x60c +#define SPLL_CHG_STATUS (1 << 1) + +#define SPLL_CNTL_MODE 0x610 +#define SPLL_DIV_SYNC (1 << 5) + +#define MPLL_CNTL_MODE 0x61c +# define MPLL_MCLK_SEL (1 << 11) +# define RV730_MPLL_MCLK_SEL (1 << 25) + +#define MPLL_AD_FUNC_CNTL 0x624 +#define CLKF(x) ((x) << 0) +#define CLKF_MASK (0x7f << 0) +#define CLKR(x) ((x) << 7) +#define CLKR_MASK (0x1f << 7) +#define CLKFRAC(x) ((x) << 12) +#define CLKFRAC_MASK (0x1f << 12) +#define YCLK_POST_DIV(x) ((x) << 17) +#define YCLK_POST_DIV_MASK (3 << 17) +#define IBIAS(x) ((x) << 20) +#define IBIAS_MASK (0x3ff << 20) +#define RESET (1 << 30) +#define PDNB (1 << 31) +#define MPLL_AD_FUNC_CNTL_2 0x628 +#define BYPASS (1 << 19) +#define BIAS_GEN_PDNB (1 << 24) +#define RESET_EN (1 << 25) +#define VCO_MODE (1 << 29) +#define MPLL_DQ_FUNC_CNTL 0x62c +#define MPLL_DQ_FUNC_CNTL_2 0x630 + +#define GENERAL_PWRMGT 0x63c +# define GLOBAL_PWRMGT_EN (1 << 0) +# define STATIC_PM_EN (1 << 1) +# define THERMAL_PROTECTION_DIS (1 << 2) +# define THERMAL_PROTECTION_TYPE (1 << 3) +# define ENABLE_GEN2PCIE (1 << 4) +# define ENABLE_GEN2XSP (1 << 5) +# define SW_SMIO_INDEX(x) ((x) << 6) +# define SW_SMIO_INDEX_MASK (3 << 6) +# define SW_SMIO_INDEX_SHIFT 6 +# define LOW_VOLT_D2_ACPI (1 << 8) +# define LOW_VOLT_D3_ACPI (1 << 9) +# define VOLT_PWRMGT_EN (1 << 10) +# define BACKBIAS_PAD_EN (1 << 18) +# define BACKBIAS_VALUE (1 << 19) +# define DYN_SPREAD_SPECTRUM_EN (1 << 23) +# define AC_DC_SW (1 << 24) + +#define CG_TPC 0x640 +#define SCLK_PWRMGT_CNTL 0x644 +# define SCLK_PWRMGT_OFF (1 << 0) +# define SCLK_LOW_D1 (1 << 1) +# define FIR_RESET (1 << 4) +# define FIR_FORCE_TREND_SEL (1 << 5) +# define FIR_TREND_MODE (1 << 6) +# define DYN_GFX_CLK_OFF_EN (1 << 7) +# define GFX_CLK_FORCE_ON (1 << 8) +# define GFX_CLK_REQUEST_OFF (1 << 9) +# define GFX_CLK_FORCE_OFF (1 << 10) +# define GFX_CLK_OFF_ACPI_D1 (1 << 11) +# define GFX_CLK_OFF_ACPI_D2 (1 << 12) +# define GFX_CLK_OFF_ACPI_D3 (1 << 13) +#define MCLK_PWRMGT_CNTL 0x648 +# define DLL_SPEED(x) ((x) << 0) +# define DLL_SPEED_MASK (0x1f << 0) +# define MPLL_PWRMGT_OFF (1 << 5) +# define DLL_READY (1 << 6) +# define MC_INT_CNTL (1 << 7) +# define MRDCKA0_SLEEP (1 << 8) +# define MRDCKA1_SLEEP (1 << 9) +# define MRDCKB0_SLEEP (1 << 10) +# define MRDCKB1_SLEEP (1 << 11) +# define MRDCKC0_SLEEP (1 << 12) +# define MRDCKC1_SLEEP (1 << 13) +# define MRDCKD0_SLEEP (1 << 14) +# define MRDCKD1_SLEEP (1 << 15) +# define MRDCKA0_RESET (1 << 16) +# define MRDCKA1_RESET (1 << 17) +# define MRDCKB0_RESET (1 << 18) +# define MRDCKB1_RESET (1 << 19) +# define MRDCKC0_RESET (1 << 20) +# define MRDCKC1_RESET (1 << 21) +# define MRDCKD0_RESET (1 << 22) +# define MRDCKD1_RESET (1 << 23) +# define DLL_READY_READ (1 << 24) +# define USE_DISPLAY_GAP (1 << 25) +# define USE_DISPLAY_URGENT_NORMAL (1 << 26) +# define MPLL_TURNOFF_D2 (1 << 28) +#define DLL_CNTL 0x64c +# define MRDCKA0_BYPASS (1 << 24) +# define MRDCKA1_BYPASS (1 << 25) +# define MRDCKB0_BYPASS (1 << 26) +# define MRDCKB1_BYPASS (1 << 27) +# define MRDCKC0_BYPASS (1 << 28) +# define MRDCKC1_BYPASS (1 << 29) +# define MRDCKD0_BYPASS (1 << 30) +# define MRDCKD1_BYPASS (1 << 31) + +#define MPLL_TIME 0x654 +# define MPLL_LOCK_TIME(x) ((x) << 0) +# define MPLL_LOCK_TIME_MASK (0xffff << 0) +# define MPLL_RESET_TIME(x) ((x) << 16) +# define MPLL_RESET_TIME_MASK (0xffff << 16) + +#define CG_CLKPIN_CNTL 0x660 +# define MUX_TCLK_TO_XCLK (1 << 8) +# define XTALIN_DIVIDE (1 << 9) + +#define TARGET_AND_CURRENT_PROFILE_INDEX 0x66c +# define CURRENT_PROFILE_INDEX_MASK (0xf << 4) +# define CURRENT_PROFILE_INDEX_SHIFT 4 + +#define S0_VID_LOWER_SMIO_CNTL 0x678 +#define S1_VID_LOWER_SMIO_CNTL 0x67c +#define S2_VID_LOWER_SMIO_CNTL 0x680 +#define S3_VID_LOWER_SMIO_CNTL 0x684 + +#define CG_FTV 0x690 +#define CG_FFCT_0 0x694 +# define UTC_0(x) ((x) << 0) +# define UTC_0_MASK (0x3ff << 0) +# define DTC_0(x) ((x) << 10) +# define DTC_0_MASK (0x3ff << 10) + +#define CG_BSP 0x6d0 +# define BSP(x) ((x) << 0) +# define BSP_MASK (0xffff << 0) +# define BSU(x) ((x) << 16) +# define BSU_MASK (0xf << 16) +#define CG_AT 0x6d4 +# define CG_R(x) ((x) << 0) +# define CG_R_MASK (0xffff << 0) +# define CG_L(x) ((x) << 16) +# define CG_L_MASK (0xffff << 16) +#define CG_GIT 0x6d8 +# define CG_GICST(x) ((x) << 0) +# define CG_GICST_MASK (0xffff << 0) +# define CG_GIPOT(x) ((x) << 16) +# define CG_GIPOT_MASK (0xffff << 16) + +#define CG_SSP 0x6e8 +# define SST(x) ((x) << 0) +# define SST_MASK (0xffff << 0) +# define SSTU(x) ((x) << 16) +# define SSTU_MASK (0xf << 16) + +#define CG_DISPLAY_GAP_CNTL 0x714 +# define DISP1_GAP(x) ((x) << 0) +# define DISP1_GAP_MASK (3 << 0) +# define DISP2_GAP(x) ((x) << 2) +# define DISP2_GAP_MASK (3 << 2) +# define VBI_TIMER_COUNT(x) ((x) << 4) +# define VBI_TIMER_COUNT_MASK (0x3fff << 4) +# define VBI_TIMER_UNIT(x) ((x) << 20) +# define VBI_TIMER_UNIT_MASK (7 << 20) +# define DISP1_GAP_MCHG(x) ((x) << 24) +# define DISP1_GAP_MCHG_MASK (3 << 24) +# define DISP2_GAP_MCHG(x) ((x) << 26) +# define DISP2_GAP_MCHG_MASK (3 << 26) + +#define CG_SPLL_SPREAD_SPECTRUM 0x790 +#define SSEN (1 << 0) +#define CLKS(x) ((x) << 4) +#define CLKS_MASK (0xfff << 4) +#define CG_SPLL_SPREAD_SPECTRUM_2 0x794 +#define CLKV(x) ((x) << 0) +#define CLKV_MASK (0x3ffffff << 0) +#define CG_MPLL_SPREAD_SPECTRUM 0x798 +#define CG_UPLL_SPREAD_SPECTRUM 0x79c +# define SSEN_MASK 0x00000001 + +#define CG_CGTT_LOCAL_0 0x7d0 +#define CG_CGTT_LOCAL_1 0x7d4 + +#define BIOS_SCRATCH_4 0x1734 + +#define MC_SEQ_MISC0 0x2a00 +#define MC_SEQ_MISC0_GDDR5_SHIFT 28 +#define MC_SEQ_MISC0_GDDR5_MASK 0xf0000000 +#define MC_SEQ_MISC0_GDDR5_VALUE 5 + +#define MC_ARB_SQM_RATIO 0x2770 +#define STATE0(x) ((x) << 0) +#define STATE0_MASK (0xff << 0) +#define STATE1(x) ((x) << 8) +#define STATE1_MASK (0xff << 8) +#define STATE2(x) ((x) << 16) +#define STATE2_MASK (0xff << 16) +#define STATE3(x) ((x) << 24) +#define STATE3_MASK (0xff << 24) + +#define MC_ARB_RFSH_RATE 0x27b0 +#define POWERMODE0(x) ((x) << 0) +#define POWERMODE0_MASK (0xff << 0) +#define POWERMODE1(x) ((x) << 8) +#define POWERMODE1_MASK (0xff << 8) +#define POWERMODE2(x) ((x) << 16) +#define POWERMODE2_MASK (0xff << 16) +#define POWERMODE3(x) ((x) << 24) +#define POWERMODE3_MASK (0xff << 24) + +#define CGTS_SM_CTRL_REG 0x9150 + /* Registers */ #define CB_COLOR0_BASE 0x28040 #define CB_COLOR1_BASE 0x28044 @@ -63,8 +333,8 @@ #define CONFIG_MEMSIZE 0x5428 #define CP_ME_CNTL 0x86D8 -#define CP_ME_HALT (1<<28) -#define CP_PFP_HALT (1<<26) +#define CP_ME_HALT (1 << 28) +#define CP_PFP_HALT (1 << 26) #define CP_ME_RAM_DATA 0xC160 #define CP_ME_RAM_RADDR 0xC158 #define CP_ME_RAM_WADDR 0xC15C @@ -113,6 +383,11 @@ #define DMA_TILING_CONFIG 0x3ec8 #define DMA_TILING_CONFIG2 0xd0b8 +/* RV730 only */ +#define UVD_UDEC_TILING_CONFIG 0xef40 +#define UVD_UDEC_DB_TILING_CONFIG 0xef44 +#define UVD_UDEC_DBW_TILING_CONFIG 0xef48 + #define GC_USER_SHADER_PIPE_CONFIG 0x8954 #define INACTIVE_QD_PIPES(x) ((x) << 8) #define INACTIVE_QD_PIPES_MASK 0x0000FF00 @@ -129,6 +404,23 @@ #define GUI_ACTIVE (1<<31) #define GRBM_STATUS2 0x8014 +#define CG_THERMAL_CTRL 0x72C +#define DPM_EVENT_SRC(x) ((x) << 0) +#define DPM_EVENT_SRC_MASK (7 << 0) +#define DIG_THERM_DPM(x) ((x) << 14) +#define DIG_THERM_DPM_MASK 0x003FC000 +#define DIG_THERM_DPM_SHIFT 14 + +#define CG_THERMAL_INT 0x734 +#define DIG_THERM_INTH(x) ((x) << 8) +#define DIG_THERM_INTH_MASK 0x0000FF00 +#define DIG_THERM_INTH_SHIFT 8 +#define DIG_THERM_INTL(x) ((x) << 16) +#define DIG_THERM_INTL_MASK 0x00FF0000 +#define DIG_THERM_INTL_SHIFT 16 +#define THERM_INT_MASK_HIGH (1 << 24) +#define THERM_INT_MASK_LOW (1 << 25) + #define CG_MULT_THERMAL_STATUS 0x740 #define ASIC_T(x) ((x) << 16) #define ASIC_T_MASK 0x3FF0000 @@ -567,7 +859,7 @@ #define AFMT_VBI_PACKET_CONTROL 0x7608 # define AFMT_GENERIC0_UPDATE (1 << 2) #define AFMT_INFOFRAME_CONTROL0 0x760c -# define AFMT_AUDIO_INFO_SOURCE (1 << 6) /* 0 - sound block; 1 - hmdi regs */ +# define AFMT_AUDIO_INFO_SOURCE (1 << 6) /* 0 - sound block; 1 - hdmi regs */ # define AFMT_AUDIO_INFO_UPDATE (1 << 7) # define AFMT_MPEG_INFO_UPDATE (1 << 10) #define AFMT_GENERIC0_7 0x7610 @@ -630,7 +922,22 @@ #define D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x691c #define D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x611c -/* PCIE link stuff */ +/* PCIE indirect regs */ +#define PCIE_P_CNTL 0x40 +# define P_PLL_PWRDN_IN_L1L23 (1 << 3) +# define P_PLL_BUF_PDNB (1 << 4) +# define P_PLL_PDNB (1 << 9) +# define P_ALLOW_PRX_FRONTEND_SHUTOFF (1 << 12) +/* PCIE PORT regs */ +#define PCIE_LC_CNTL 0xa0 +# define LC_L0S_INACTIVITY(x) ((x) << 8) +# define LC_L0S_INACTIVITY_MASK (0xf << 8) +# define LC_L0S_INACTIVITY_SHIFT 8 +# define LC_L1_INACTIVITY(x) ((x) << 12) +# define LC_L1_INACTIVITY_MASK (0xf << 12) +# define LC_L1_INACTIVITY_SHIFT 12 +# define LC_PMI_TO_L1_DIS (1 << 16) +# define LC_ASPM_TO_L1_DIS (1 << 24) #define PCIE_LC_TRAINING_CNTL 0xa1 /* PCIE_P */ #define PCIE_LC_LINK_WIDTH_CNTL 0xa2 /* PCIE_P */ # define LC_LINK_WIDTH_SHIFT 0 @@ -658,6 +965,9 @@ # define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK (0x3 << 8) # define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT 3 # define LC_CURRENT_DATA_RATE (1 << 11) +# define LC_HW_VOLTAGE_IF_CONTROL(x) ((x) << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_MASK (3 << 12) +# define LC_HW_VOLTAGE_IF_CONTROL_SHIFT 12 # define LC_VOLTAGE_TIMER_SEL_MASK (0xf << 14) # define LC_CLR_FAILED_SPD_CHANGE_CNT (1 << 21) # define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 23) @@ -668,4 +978,37 @@ # define TARGET_LINK_SPEED_MASK (0xf << 0) # define SELECTABLE_DEEMPHASIS (1 << 6) +/* + * PM4 + */ +#define PACKET0(reg, n) ((RADEON_PACKET_TYPE0 << 30) | \ + (((reg) >> 2) & 0xFFFF) | \ + ((n) & 0x3FFF) << 16) +#define PACKET3(op, n) ((RADEON_PACKET_TYPE3 << 30) | \ + (((op) & 0xFF) << 8) | \ + ((n) & 0x3FFF) << 16) + +/* UVD */ +#define UVD_SEMA_ADDR_LOW 0xef00 +#define UVD_SEMA_ADDR_HIGH 0xef04 +#define UVD_SEMA_CMD 0xef08 +#define UVD_GPCOM_VCPU_CMD 0xef0c +#define UVD_GPCOM_VCPU_DATA0 0xef10 +#define UVD_GPCOM_VCPU_DATA1 0xef14 + +#define UVD_LMI_EXT40_ADDR 0xf498 +#define UVD_VCPU_CHIP_ID 0xf4d4 +#define UVD_VCPU_CACHE_OFFSET0 0xf4d8 +#define UVD_VCPU_CACHE_SIZE0 0xf4dc +#define UVD_VCPU_CACHE_OFFSET1 0xf4e0 +#define UVD_VCPU_CACHE_SIZE1 0xf4e4 +#define UVD_VCPU_CACHE_OFFSET2 0xf4e8 +#define UVD_VCPU_CACHE_SIZE2 0xf4ec +#define UVD_LMI_ADDR_EXT 0xf594 + +#define UVD_RBC_RB_RPTR 0xf690 +#define UVD_RBC_RB_WPTR 0xf694 + +#define UVD_CONTEXT_ID 0xf6f4 + #endif diff --git a/sys/dev/pci/drm/radeon/si.c b/sys/dev/pci/drm/radeon/si.c index 5edbbde1e5e..3d47942e94b 100644 --- a/sys/dev/pci/drm/radeon/si.c +++ b/sys/dev/pci/drm/radeon/si.c @@ -1,4 +1,3 @@ -/* $OpenBSD: si.c,v 1.25 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2011 Advanced Micro Devices, Inc. * @@ -25,39 +24,1299 @@ #include #include "radeon.h" #include "radeon_asic.h" +#include "radeon_audio.h" #include #include "sid.h" #include "atom.h" #include "si_blit_shaders.h" +#include "clearstate_si.h" +#include "radeon_ucode.h" -#define SI_PFP_UCODE_SIZE 2144 -#define SI_PM4_UCODE_SIZE 2144 -#define SI_CE_UCODE_SIZE 2144 -#define SI_RLC_UCODE_SIZE 2048 -#define SI_MC_UCODE_SIZE 7769 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin"); MODULE_FIRMWARE("radeon/TAHITI_me.bin"); MODULE_FIRMWARE("radeon/TAHITI_ce.bin"); MODULE_FIRMWARE("radeon/TAHITI_mc.bin"); +MODULE_FIRMWARE("radeon/TAHITI_mc2.bin"); MODULE_FIRMWARE("radeon/TAHITI_rlc.bin"); +MODULE_FIRMWARE("radeon/TAHITI_smc.bin"); + +MODULE_FIRMWARE("radeon/tahiti_pfp.bin"); +MODULE_FIRMWARE("radeon/tahiti_me.bin"); +MODULE_FIRMWARE("radeon/tahiti_ce.bin"); +MODULE_FIRMWARE("radeon/tahiti_mc.bin"); +MODULE_FIRMWARE("radeon/tahiti_rlc.bin"); +MODULE_FIRMWARE("radeon/tahiti_smc.bin"); + MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin"); MODULE_FIRMWARE("radeon/PITCAIRN_me.bin"); MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin"); MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin"); +MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin"); MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin"); +MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin"); + +MODULE_FIRMWARE("radeon/pitcairn_pfp.bin"); +MODULE_FIRMWARE("radeon/pitcairn_me.bin"); +MODULE_FIRMWARE("radeon/pitcairn_ce.bin"); +MODULE_FIRMWARE("radeon/pitcairn_mc.bin"); +MODULE_FIRMWARE("radeon/pitcairn_rlc.bin"); +MODULE_FIRMWARE("radeon/pitcairn_smc.bin"); + MODULE_FIRMWARE("radeon/VERDE_pfp.bin"); MODULE_FIRMWARE("radeon/VERDE_me.bin"); MODULE_FIRMWARE("radeon/VERDE_ce.bin"); MODULE_FIRMWARE("radeon/VERDE_mc.bin"); +MODULE_FIRMWARE("radeon/VERDE_mc2.bin"); MODULE_FIRMWARE("radeon/VERDE_rlc.bin"); - +MODULE_FIRMWARE("radeon/VERDE_smc.bin"); + +MODULE_FIRMWARE("radeon/verde_pfp.bin"); +MODULE_FIRMWARE("radeon/verde_me.bin"); +MODULE_FIRMWARE("radeon/verde_ce.bin"); +MODULE_FIRMWARE("radeon/verde_mc.bin"); +MODULE_FIRMWARE("radeon/verde_rlc.bin"); +MODULE_FIRMWARE("radeon/verde_smc.bin"); + +MODULE_FIRMWARE("radeon/OLAND_pfp.bin"); +MODULE_FIRMWARE("radeon/OLAND_me.bin"); +MODULE_FIRMWARE("radeon/OLAND_ce.bin"); +MODULE_FIRMWARE("radeon/OLAND_mc.bin"); +MODULE_FIRMWARE("radeon/OLAND_mc2.bin"); +MODULE_FIRMWARE("radeon/OLAND_rlc.bin"); +MODULE_FIRMWARE("radeon/OLAND_smc.bin"); + +MODULE_FIRMWARE("radeon/oland_pfp.bin"); +MODULE_FIRMWARE("radeon/oland_me.bin"); +MODULE_FIRMWARE("radeon/oland_ce.bin"); +MODULE_FIRMWARE("radeon/oland_mc.bin"); +MODULE_FIRMWARE("radeon/oland_rlc.bin"); +MODULE_FIRMWARE("radeon/oland_smc.bin"); + +MODULE_FIRMWARE("radeon/HAINAN_pfp.bin"); +MODULE_FIRMWARE("radeon/HAINAN_me.bin"); +MODULE_FIRMWARE("radeon/HAINAN_ce.bin"); +MODULE_FIRMWARE("radeon/HAINAN_mc.bin"); +MODULE_FIRMWARE("radeon/HAINAN_mc2.bin"); +MODULE_FIRMWARE("radeon/HAINAN_rlc.bin"); +MODULE_FIRMWARE("radeon/HAINAN_smc.bin"); + +MODULE_FIRMWARE("radeon/hainan_pfp.bin"); +MODULE_FIRMWARE("radeon/hainan_me.bin"); +MODULE_FIRMWARE("radeon/hainan_ce.bin"); +MODULE_FIRMWARE("radeon/hainan_mc.bin"); +MODULE_FIRMWARE("radeon/hainan_rlc.bin"); +MODULE_FIRMWARE("radeon/hainan_smc.bin"); + +static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); +static void si_pcie_gen3_enable(struct radeon_device *rdev); +static void si_program_aspm(struct radeon_device *rdev); +extern void sumo_rlc_fini(struct radeon_device *rdev); +extern int sumo_rlc_init(struct radeon_device *rdev); extern int r600_ih_ring_alloc(struct radeon_device *rdev); extern void r600_ih_ring_fini(struct radeon_device *rdev); extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev); extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev); +extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev); +extern bool evergreen_is_display_hung(struct radeon_device *rdev); +static void si_enable_gui_idle_interrupt(struct radeon_device *rdev, + bool enable); +static void si_init_pg(struct radeon_device *rdev); +static void si_init_cg(struct radeon_device *rdev); +static void si_fini_pg(struct radeon_device *rdev); +static void si_fini_cg(struct radeon_device *rdev); +static void si_rlc_stop(struct radeon_device *rdev); + +static const u32 verde_rlc_save_restore_register_list[] = +{ + (0x8000 << 16) | (0x98f4 >> 2), + 0x00000000, + (0x8040 << 16) | (0x98f4 >> 2), + 0x00000000, + (0x8000 << 16) | (0xe80 >> 2), + 0x00000000, + (0x8040 << 16) | (0xe80 >> 2), + 0x00000000, + (0x8000 << 16) | (0x89bc >> 2), + 0x00000000, + (0x8040 << 16) | (0x89bc >> 2), + 0x00000000, + (0x8000 << 16) | (0x8c1c >> 2), + 0x00000000, + (0x8040 << 16) | (0x8c1c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x98f0 >> 2), + 0x00000000, + (0x9c00 << 16) | (0xe7c >> 2), + 0x00000000, + (0x8000 << 16) | (0x9148 >> 2), + 0x00000000, + (0x8040 << 16) | (0x9148 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9150 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x897c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8d8c >> 2), + 0x00000000, + (0x9c00 << 16) | (0xac54 >> 2), + 0X00000000, + 0x3, + (0x9c00 << 16) | (0x98f8 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9910 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9914 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9918 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x991c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9920 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9924 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9928 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x992c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9930 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9934 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9938 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x993c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9940 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9944 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9948 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x994c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9950 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9954 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9958 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x995c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9960 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9964 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9968 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x996c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9970 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9974 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9978 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x997c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9980 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9984 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9988 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x998c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8c00 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8c14 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8c04 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8c08 >> 2), + 0x00000000, + (0x8000 << 16) | (0x9b7c >> 2), + 0x00000000, + (0x8040 << 16) | (0x9b7c >> 2), + 0x00000000, + (0x8000 << 16) | (0xe84 >> 2), + 0x00000000, + (0x8040 << 16) | (0xe84 >> 2), + 0x00000000, + (0x8000 << 16) | (0x89c0 >> 2), + 0x00000000, + (0x8040 << 16) | (0x89c0 >> 2), + 0x00000000, + (0x8000 << 16) | (0x914c >> 2), + 0x00000000, + (0x8040 << 16) | (0x914c >> 2), + 0x00000000, + (0x8000 << 16) | (0x8c20 >> 2), + 0x00000000, + (0x8040 << 16) | (0x8c20 >> 2), + 0x00000000, + (0x8000 << 16) | (0x9354 >> 2), + 0x00000000, + (0x8040 << 16) | (0x9354 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9060 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9364 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9100 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x913c >> 2), + 0x00000000, + (0x8000 << 16) | (0x90e0 >> 2), + 0x00000000, + (0x8000 << 16) | (0x90e4 >> 2), + 0x00000000, + (0x8000 << 16) | (0x90e8 >> 2), + 0x00000000, + (0x8040 << 16) | (0x90e0 >> 2), + 0x00000000, + (0x8040 << 16) | (0x90e4 >> 2), + 0x00000000, + (0x8040 << 16) | (0x90e8 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8bcc >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8b24 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x88c4 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8e50 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8c0c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8e58 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8e5c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9508 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x950c >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9494 >> 2), + 0x00000000, + (0x9c00 << 16) | (0xac0c >> 2), + 0x00000000, + (0x9c00 << 16) | (0xac10 >> 2), + 0x00000000, + (0x9c00 << 16) | (0xac14 >> 2), + 0x00000000, + (0x9c00 << 16) | (0xae00 >> 2), + 0x00000000, + (0x9c00 << 16) | (0xac08 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x88d4 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x88c8 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x88cc >> 2), + 0x00000000, + (0x9c00 << 16) | (0x89b0 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8b10 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x8a14 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9830 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9834 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9838 >> 2), + 0x00000000, + (0x9c00 << 16) | (0x9a10 >> 2), + 0x00000000, + (0x8000 << 16) | (0x9870 >> 2), + 0x00000000, + (0x8000 << 16) | (0x9874 >> 2), + 0x00000000, + (0x8001 << 16) | (0x9870 >> 2), + 0x00000000, + (0x8001 << 16) | (0x9874 >> 2), + 0x00000000, + (0x8040 << 16) | (0x9870 >> 2), + 0x00000000, + (0x8040 << 16) | (0x9874 >> 2), + 0x00000000, + (0x8041 << 16) | (0x9870 >> 2), + 0x00000000, + (0x8041 << 16) | (0x9874 >> 2), + 0x00000000, + 0x00000000 +}; + +static const u32 tahiti_golden_rlc_registers[] = +{ + 0xc424, 0xffffffff, 0x00601005, + 0xc47c, 0xffffffff, 0x10104040, + 0xc488, 0xffffffff, 0x0100000a, + 0xc314, 0xffffffff, 0x00000800, + 0xc30c, 0xffffffff, 0x800000f4, + 0xf4a8, 0xffffffff, 0x00000000 +}; + +static const u32 tahiti_golden_registers[] = +{ + 0x9a10, 0x00010000, 0x00018208, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0x0002021c, 0x00020200, + 0xc78, 0x00000080, 0x00000000, + 0xd030, 0x000300c0, 0x00800040, + 0xd830, 0x000300c0, 0x00800040, + 0x5bb0, 0x000000f0, 0x00000070, + 0x5bc0, 0x00200000, 0x50100000, + 0x7030, 0x31000311, 0x00000011, + 0x277c, 0x00000003, 0x000007ff, + 0x240c, 0x000007ff, 0x00000000, + 0x8a14, 0xf000001f, 0x00000007, + 0x8b24, 0xffffffff, 0x00ffffff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x4e000000, + 0x28350, 0x3f3f3fff, 0x2a00126a, + 0x30, 0x000000ff, 0x0040, + 0x34, 0x00000040, 0x00004040, + 0x9100, 0x07ffffff, 0x03000000, + 0x8e88, 0x01ff1f3f, 0x00000000, + 0x8e84, 0x01ff1f3f, 0x00000000, + 0x9060, 0x0000007f, 0x00000020, + 0x9508, 0x00010000, 0x00010000, + 0xac14, 0x00000200, 0x000002fb, + 0xac10, 0xffffffff, 0x0000543b, + 0xac0c, 0xffffffff, 0xa9210876, + 0x88d0, 0xffffffff, 0x000fff40, + 0x88d4, 0x0000001f, 0x00000010, + 0x1410, 0x20000000, 0x20fffed8, + 0x15c0, 0x000c0fc0, 0x000c0400 +}; + +static const u32 tahiti_golden_registers2[] = +{ + 0xc64, 0x00000001, 0x00000001 +}; + +static const u32 pitcairn_golden_rlc_registers[] = +{ + 0xc424, 0xffffffff, 0x00601004, + 0xc47c, 0xffffffff, 0x10102020, + 0xc488, 0xffffffff, 0x01000020, + 0xc314, 0xffffffff, 0x00000800, + 0xc30c, 0xffffffff, 0x800000a4 +}; + +static const u32 pitcairn_golden_registers[] = +{ + 0x9a10, 0x00010000, 0x00018208, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0x0002021c, 0x00020200, + 0xc78, 0x00000080, 0x00000000, + 0xd030, 0x000300c0, 0x00800040, + 0xd830, 0x000300c0, 0x00800040, + 0x5bb0, 0x000000f0, 0x00000070, + 0x5bc0, 0x00200000, 0x50100000, + 0x7030, 0x31000311, 0x00000011, + 0x2ae4, 0x00073ffe, 0x000022a2, + 0x240c, 0x000007ff, 0x00000000, + 0x8a14, 0xf000001f, 0x00000007, + 0x8b24, 0xffffffff, 0x00ffffff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x4e000000, + 0x28350, 0x3f3f3fff, 0x2a00126a, + 0x30, 0x000000ff, 0x0040, + 0x34, 0x00000040, 0x00004040, + 0x9100, 0x07ffffff, 0x03000000, + 0x9060, 0x0000007f, 0x00000020, + 0x9508, 0x00010000, 0x00010000, + 0xac14, 0x000003ff, 0x000000f7, + 0xac10, 0xffffffff, 0x00000000, + 0xac0c, 0xffffffff, 0x32761054, + 0x88d4, 0x0000001f, 0x00000010, + 0x15c0, 0x000c0fc0, 0x000c0400 +}; + +static const u32 verde_golden_rlc_registers[] = +{ + 0xc424, 0xffffffff, 0x033f1005, + 0xc47c, 0xffffffff, 0x10808020, + 0xc488, 0xffffffff, 0x00800008, + 0xc314, 0xffffffff, 0x00001000, + 0xc30c, 0xffffffff, 0x80010014 +}; + +static const u32 verde_golden_registers[] = +{ + 0x9a10, 0x00010000, 0x00018208, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0x0002021c, 0x00020200, + 0xc78, 0x00000080, 0x00000000, + 0xd030, 0x000300c0, 0x00800040, + 0xd030, 0x000300c0, 0x00800040, + 0xd830, 0x000300c0, 0x00800040, + 0xd830, 0x000300c0, 0x00800040, + 0x5bb0, 0x000000f0, 0x00000070, + 0x5bc0, 0x00200000, 0x50100000, + 0x7030, 0x31000311, 0x00000011, + 0x2ae4, 0x00073ffe, 0x000022a2, + 0x2ae4, 0x00073ffe, 0x000022a2, + 0x2ae4, 0x00073ffe, 0x000022a2, + 0x240c, 0x000007ff, 0x00000000, + 0x240c, 0x000007ff, 0x00000000, + 0x240c, 0x000007ff, 0x00000000, + 0x8a14, 0xf000001f, 0x00000007, + 0x8a14, 0xf000001f, 0x00000007, + 0x8a14, 0xf000001f, 0x00000007, + 0x8b24, 0xffffffff, 0x00ffffff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x4e000000, + 0x28350, 0x3f3f3fff, 0x0000124a, + 0x28350, 0x3f3f3fff, 0x0000124a, + 0x28350, 0x3f3f3fff, 0x0000124a, + 0x30, 0x000000ff, 0x0040, + 0x34, 0x00000040, 0x00004040, + 0x9100, 0x07ffffff, 0x03000000, + 0x9100, 0x07ffffff, 0x03000000, + 0x8e88, 0x01ff1f3f, 0x00000000, + 0x8e88, 0x01ff1f3f, 0x00000000, + 0x8e88, 0x01ff1f3f, 0x00000000, + 0x8e84, 0x01ff1f3f, 0x00000000, + 0x8e84, 0x01ff1f3f, 0x00000000, + 0x8e84, 0x01ff1f3f, 0x00000000, + 0x9060, 0x0000007f, 0x00000020, + 0x9508, 0x00010000, 0x00010000, + 0xac14, 0x000003ff, 0x00000003, + 0xac14, 0x000003ff, 0x00000003, + 0xac14, 0x000003ff, 0x00000003, + 0xac10, 0xffffffff, 0x00000000, + 0xac10, 0xffffffff, 0x00000000, + 0xac10, 0xffffffff, 0x00000000, + 0xac0c, 0xffffffff, 0x00001032, + 0xac0c, 0xffffffff, 0x00001032, + 0xac0c, 0xffffffff, 0x00001032, + 0x88d4, 0x0000001f, 0x00000010, + 0x88d4, 0x0000001f, 0x00000010, + 0x88d4, 0x0000001f, 0x00000010, + 0x15c0, 0x000c0fc0, 0x000c0400 +}; + +static const u32 oland_golden_rlc_registers[] = +{ + 0xc424, 0xffffffff, 0x00601005, + 0xc47c, 0xffffffff, 0x10104040, + 0xc488, 0xffffffff, 0x0100000a, + 0xc314, 0xffffffff, 0x00000800, + 0xc30c, 0xffffffff, 0x800000f4 +}; + +static const u32 oland_golden_registers[] = +{ + 0x9a10, 0x00010000, 0x00018208, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0x0002021c, 0x00020200, + 0xc78, 0x00000080, 0x00000000, + 0xd030, 0x000300c0, 0x00800040, + 0xd830, 0x000300c0, 0x00800040, + 0x5bb0, 0x000000f0, 0x00000070, + 0x5bc0, 0x00200000, 0x50100000, + 0x7030, 0x31000311, 0x00000011, + 0x2ae4, 0x00073ffe, 0x000022a2, + 0x240c, 0x000007ff, 0x00000000, + 0x8a14, 0xf000001f, 0x00000007, + 0x8b24, 0xffffffff, 0x00ffffff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x4e000000, + 0x28350, 0x3f3f3fff, 0x00000082, + 0x30, 0x000000ff, 0x0040, + 0x34, 0x00000040, 0x00004040, + 0x9100, 0x07ffffff, 0x03000000, + 0x9060, 0x0000007f, 0x00000020, + 0x9508, 0x00010000, 0x00010000, + 0xac14, 0x000003ff, 0x000000f3, + 0xac10, 0xffffffff, 0x00000000, + 0xac0c, 0xffffffff, 0x00003210, + 0x88d4, 0x0000001f, 0x00000010, + 0x15c0, 0x000c0fc0, 0x000c0400 +}; + +static const u32 hainan_golden_registers[] = +{ + 0x9a10, 0x00010000, 0x00018208, + 0x9830, 0xffffffff, 0x00000000, + 0x9834, 0xf00fffff, 0x00000400, + 0x9838, 0x0002021c, 0x00020200, + 0xd0c0, 0xff000fff, 0x00000100, + 0xd030, 0x000300c0, 0x00800040, + 0xd8c0, 0xff000fff, 0x00000100, + 0xd830, 0x000300c0, 0x00800040, + 0x2ae4, 0x00073ffe, 0x000022a2, + 0x240c, 0x000007ff, 0x00000000, + 0x8a14, 0xf000001f, 0x00000007, + 0x8b24, 0xffffffff, 0x00ffffff, + 0x8b10, 0x0000ff0f, 0x00000000, + 0x28a4c, 0x07ffffff, 0x4e000000, + 0x28350, 0x3f3f3fff, 0x00000000, + 0x30, 0x000000ff, 0x0040, + 0x34, 0x00000040, 0x00004040, + 0x9100, 0x03e00000, 0x03600000, + 0x9060, 0x0000007f, 0x00000020, + 0x9508, 0x00010000, 0x00010000, + 0xac14, 0x000003ff, 0x000000f1, + 0xac10, 0xffffffff, 0x00000000, + 0xac0c, 0xffffffff, 0x00003210, + 0x88d4, 0x0000001f, 0x00000010, + 0x15c0, 0x000c0fc0, 0x000c0400 +}; + +static const u32 hainan_golden_registers2[] = +{ + 0x98f8, 0xffffffff, 0x02010001 +}; + +static const u32 tahiti_mgcg_cgcg_init[] = +{ + 0xc400, 0xffffffff, 0xfffffffc, + 0x802c, 0xffffffff, 0xe0000000, + 0x9a60, 0xffffffff, 0x00000100, + 0x92a4, 0xffffffff, 0x00000100, + 0xc164, 0xffffffff, 0x00000100, + 0x9774, 0xffffffff, 0x00000100, + 0x8984, 0xffffffff, 0x06000100, + 0x8a18, 0xffffffff, 0x00000100, + 0x92a0, 0xffffffff, 0x00000100, + 0xc380, 0xffffffff, 0x00000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x8d88, 0xffffffff, 0x00000100, + 0x8d8c, 0xffffffff, 0x00000100, + 0x9030, 0xffffffff, 0x00000100, + 0x9034, 0xffffffff, 0x00000100, + 0x9038, 0xffffffff, 0x00000100, + 0x903c, 0xffffffff, 0x00000100, + 0xad80, 0xffffffff, 0x00000100, + 0xac54, 0xffffffff, 0x00000100, + 0x897c, 0xffffffff, 0x06000100, + 0x9868, 0xffffffff, 0x00000100, + 0x9510, 0xffffffff, 0x00000100, + 0xaf04, 0xffffffff, 0x00000100, + 0xae04, 0xffffffff, 0x00000100, + 0x949c, 0xffffffff, 0x00000100, + 0x802c, 0xffffffff, 0xe0000000, + 0x9160, 0xffffffff, 0x00010000, + 0x9164, 0xffffffff, 0x00030002, + 0x9168, 0xffffffff, 0x00040007, + 0x916c, 0xffffffff, 0x00060005, + 0x9170, 0xffffffff, 0x00090008, + 0x9174, 0xffffffff, 0x00020001, + 0x9178, 0xffffffff, 0x00040003, + 0x917c, 0xffffffff, 0x00000007, + 0x9180, 0xffffffff, 0x00060005, + 0x9184, 0xffffffff, 0x00090008, + 0x9188, 0xffffffff, 0x00030002, + 0x918c, 0xffffffff, 0x00050004, + 0x9190, 0xffffffff, 0x00000008, + 0x9194, 0xffffffff, 0x00070006, + 0x9198, 0xffffffff, 0x000a0009, + 0x919c, 0xffffffff, 0x00040003, + 0x91a0, 0xffffffff, 0x00060005, + 0x91a4, 0xffffffff, 0x00000009, + 0x91a8, 0xffffffff, 0x00080007, + 0x91ac, 0xffffffff, 0x000b000a, + 0x91b0, 0xffffffff, 0x00050004, + 0x91b4, 0xffffffff, 0x00070006, + 0x91b8, 0xffffffff, 0x0008000b, + 0x91bc, 0xffffffff, 0x000a0009, + 0x91c0, 0xffffffff, 0x000d000c, + 0x91c4, 0xffffffff, 0x00060005, + 0x91c8, 0xffffffff, 0x00080007, + 0x91cc, 0xffffffff, 0x0000000b, + 0x91d0, 0xffffffff, 0x000a0009, + 0x91d4, 0xffffffff, 0x000d000c, + 0x91d8, 0xffffffff, 0x00070006, + 0x91dc, 0xffffffff, 0x00090008, + 0x91e0, 0xffffffff, 0x0000000c, + 0x91e4, 0xffffffff, 0x000b000a, + 0x91e8, 0xffffffff, 0x000e000d, + 0x91ec, 0xffffffff, 0x00080007, + 0x91f0, 0xffffffff, 0x000a0009, + 0x91f4, 0xffffffff, 0x0000000d, + 0x91f8, 0xffffffff, 0x000c000b, + 0x91fc, 0xffffffff, 0x000f000e, + 0x9200, 0xffffffff, 0x00090008, + 0x9204, 0xffffffff, 0x000b000a, + 0x9208, 0xffffffff, 0x000c000f, + 0x920c, 0xffffffff, 0x000e000d, + 0x9210, 0xffffffff, 0x00110010, + 0x9214, 0xffffffff, 0x000a0009, + 0x9218, 0xffffffff, 0x000c000b, + 0x921c, 0xffffffff, 0x0000000f, + 0x9220, 0xffffffff, 0x000e000d, + 0x9224, 0xffffffff, 0x00110010, + 0x9228, 0xffffffff, 0x000b000a, + 0x922c, 0xffffffff, 0x000d000c, + 0x9230, 0xffffffff, 0x00000010, + 0x9234, 0xffffffff, 0x000f000e, + 0x9238, 0xffffffff, 0x00120011, + 0x923c, 0xffffffff, 0x000c000b, + 0x9240, 0xffffffff, 0x000e000d, + 0x9244, 0xffffffff, 0x00000011, + 0x9248, 0xffffffff, 0x0010000f, + 0x924c, 0xffffffff, 0x00130012, + 0x9250, 0xffffffff, 0x000d000c, + 0x9254, 0xffffffff, 0x000f000e, + 0x9258, 0xffffffff, 0x00100013, + 0x925c, 0xffffffff, 0x00120011, + 0x9260, 0xffffffff, 0x00150014, + 0x9264, 0xffffffff, 0x000e000d, + 0x9268, 0xffffffff, 0x0010000f, + 0x926c, 0xffffffff, 0x00000013, + 0x9270, 0xffffffff, 0x00120011, + 0x9274, 0xffffffff, 0x00150014, + 0x9278, 0xffffffff, 0x000f000e, + 0x927c, 0xffffffff, 0x00110010, + 0x9280, 0xffffffff, 0x00000014, + 0x9284, 0xffffffff, 0x00130012, + 0x9288, 0xffffffff, 0x00160015, + 0x928c, 0xffffffff, 0x0010000f, + 0x9290, 0xffffffff, 0x00120011, + 0x9294, 0xffffffff, 0x00000015, + 0x9298, 0xffffffff, 0x00140013, + 0x929c, 0xffffffff, 0x00170016, + 0x9150, 0xffffffff, 0x96940200, + 0x8708, 0xffffffff, 0x00900100, + 0xc478, 0xffffffff, 0x00000080, + 0xc404, 0xffffffff, 0x0020003f, + 0x30, 0xffffffff, 0x0000001c, + 0x34, 0x000f0000, 0x000f0000, + 0x160c, 0xffffffff, 0x00000100, + 0x1024, 0xffffffff, 0x00000100, + 0x102c, 0x00000101, 0x00000000, + 0x20a8, 0xffffffff, 0x00000104, + 0x264c, 0x000c0000, 0x000c0000, + 0x2648, 0x000c0000, 0x000c0000, + 0x55e4, 0xff000fff, 0x00000100, + 0x55e8, 0x00000001, 0x00000001, + 0x2f50, 0x00000001, 0x00000001, + 0x30cc, 0xc0000fff, 0x00000104, + 0xc1e4, 0x00000001, 0x00000001, + 0xd0c0, 0xfffffff0, 0x00000100, + 0xd8c0, 0xfffffff0, 0x00000100 +}; + +static const u32 pitcairn_mgcg_cgcg_init[] = +{ + 0xc400, 0xffffffff, 0xfffffffc, + 0x802c, 0xffffffff, 0xe0000000, + 0x9a60, 0xffffffff, 0x00000100, + 0x92a4, 0xffffffff, 0x00000100, + 0xc164, 0xffffffff, 0x00000100, + 0x9774, 0xffffffff, 0x00000100, + 0x8984, 0xffffffff, 0x06000100, + 0x8a18, 0xffffffff, 0x00000100, + 0x92a0, 0xffffffff, 0x00000100, + 0xc380, 0xffffffff, 0x00000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x8d88, 0xffffffff, 0x00000100, + 0x8d8c, 0xffffffff, 0x00000100, + 0x9030, 0xffffffff, 0x00000100, + 0x9034, 0xffffffff, 0x00000100, + 0x9038, 0xffffffff, 0x00000100, + 0x903c, 0xffffffff, 0x00000100, + 0xad80, 0xffffffff, 0x00000100, + 0xac54, 0xffffffff, 0x00000100, + 0x897c, 0xffffffff, 0x06000100, + 0x9868, 0xffffffff, 0x00000100, + 0x9510, 0xffffffff, 0x00000100, + 0xaf04, 0xffffffff, 0x00000100, + 0xae04, 0xffffffff, 0x00000100, + 0x949c, 0xffffffff, 0x00000100, + 0x802c, 0xffffffff, 0xe0000000, + 0x9160, 0xffffffff, 0x00010000, + 0x9164, 0xffffffff, 0x00030002, + 0x9168, 0xffffffff, 0x00040007, + 0x916c, 0xffffffff, 0x00060005, + 0x9170, 0xffffffff, 0x00090008, + 0x9174, 0xffffffff, 0x00020001, + 0x9178, 0xffffffff, 0x00040003, + 0x917c, 0xffffffff, 0x00000007, + 0x9180, 0xffffffff, 0x00060005, + 0x9184, 0xffffffff, 0x00090008, + 0x9188, 0xffffffff, 0x00030002, + 0x918c, 0xffffffff, 0x00050004, + 0x9190, 0xffffffff, 0x00000008, + 0x9194, 0xffffffff, 0x00070006, + 0x9198, 0xffffffff, 0x000a0009, + 0x919c, 0xffffffff, 0x00040003, + 0x91a0, 0xffffffff, 0x00060005, + 0x91a4, 0xffffffff, 0x00000009, + 0x91a8, 0xffffffff, 0x00080007, + 0x91ac, 0xffffffff, 0x000b000a, + 0x91b0, 0xffffffff, 0x00050004, + 0x91b4, 0xffffffff, 0x00070006, + 0x91b8, 0xffffffff, 0x0008000b, + 0x91bc, 0xffffffff, 0x000a0009, + 0x91c0, 0xffffffff, 0x000d000c, + 0x9200, 0xffffffff, 0x00090008, + 0x9204, 0xffffffff, 0x000b000a, + 0x9208, 0xffffffff, 0x000c000f, + 0x920c, 0xffffffff, 0x000e000d, + 0x9210, 0xffffffff, 0x00110010, + 0x9214, 0xffffffff, 0x000a0009, + 0x9218, 0xffffffff, 0x000c000b, + 0x921c, 0xffffffff, 0x0000000f, + 0x9220, 0xffffffff, 0x000e000d, + 0x9224, 0xffffffff, 0x00110010, + 0x9228, 0xffffffff, 0x000b000a, + 0x922c, 0xffffffff, 0x000d000c, + 0x9230, 0xffffffff, 0x00000010, + 0x9234, 0xffffffff, 0x000f000e, + 0x9238, 0xffffffff, 0x00120011, + 0x923c, 0xffffffff, 0x000c000b, + 0x9240, 0xffffffff, 0x000e000d, + 0x9244, 0xffffffff, 0x00000011, + 0x9248, 0xffffffff, 0x0010000f, + 0x924c, 0xffffffff, 0x00130012, + 0x9250, 0xffffffff, 0x000d000c, + 0x9254, 0xffffffff, 0x000f000e, + 0x9258, 0xffffffff, 0x00100013, + 0x925c, 0xffffffff, 0x00120011, + 0x9260, 0xffffffff, 0x00150014, + 0x9150, 0xffffffff, 0x96940200, + 0x8708, 0xffffffff, 0x00900100, + 0xc478, 0xffffffff, 0x00000080, + 0xc404, 0xffffffff, 0x0020003f, + 0x30, 0xffffffff, 0x0000001c, + 0x34, 0x000f0000, 0x000f0000, + 0x160c, 0xffffffff, 0x00000100, + 0x1024, 0xffffffff, 0x00000100, + 0x102c, 0x00000101, 0x00000000, + 0x20a8, 0xffffffff, 0x00000104, + 0x55e4, 0xff000fff, 0x00000100, + 0x55e8, 0x00000001, 0x00000001, + 0x2f50, 0x00000001, 0x00000001, + 0x30cc, 0xc0000fff, 0x00000104, + 0xc1e4, 0x00000001, 0x00000001, + 0xd0c0, 0xfffffff0, 0x00000100, + 0xd8c0, 0xfffffff0, 0x00000100 +}; + +static const u32 verde_mgcg_cgcg_init[] = +{ + 0xc400, 0xffffffff, 0xfffffffc, + 0x802c, 0xffffffff, 0xe0000000, + 0x9a60, 0xffffffff, 0x00000100, + 0x92a4, 0xffffffff, 0x00000100, + 0xc164, 0xffffffff, 0x00000100, + 0x9774, 0xffffffff, 0x00000100, + 0x8984, 0xffffffff, 0x06000100, + 0x8a18, 0xffffffff, 0x00000100, + 0x92a0, 0xffffffff, 0x00000100, + 0xc380, 0xffffffff, 0x00000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x8d88, 0xffffffff, 0x00000100, + 0x8d8c, 0xffffffff, 0x00000100, + 0x9030, 0xffffffff, 0x00000100, + 0x9034, 0xffffffff, 0x00000100, + 0x9038, 0xffffffff, 0x00000100, + 0x903c, 0xffffffff, 0x00000100, + 0xad80, 0xffffffff, 0x00000100, + 0xac54, 0xffffffff, 0x00000100, + 0x897c, 0xffffffff, 0x06000100, + 0x9868, 0xffffffff, 0x00000100, + 0x9510, 0xffffffff, 0x00000100, + 0xaf04, 0xffffffff, 0x00000100, + 0xae04, 0xffffffff, 0x00000100, + 0x949c, 0xffffffff, 0x00000100, + 0x802c, 0xffffffff, 0xe0000000, + 0x9160, 0xffffffff, 0x00010000, + 0x9164, 0xffffffff, 0x00030002, + 0x9168, 0xffffffff, 0x00040007, + 0x916c, 0xffffffff, 0x00060005, + 0x9170, 0xffffffff, 0x00090008, + 0x9174, 0xffffffff, 0x00020001, + 0x9178, 0xffffffff, 0x00040003, + 0x917c, 0xffffffff, 0x00000007, + 0x9180, 0xffffffff, 0x00060005, + 0x9184, 0xffffffff, 0x00090008, + 0x9188, 0xffffffff, 0x00030002, + 0x918c, 0xffffffff, 0x00050004, + 0x9190, 0xffffffff, 0x00000008, + 0x9194, 0xffffffff, 0x00070006, + 0x9198, 0xffffffff, 0x000a0009, + 0x919c, 0xffffffff, 0x00040003, + 0x91a0, 0xffffffff, 0x00060005, + 0x91a4, 0xffffffff, 0x00000009, + 0x91a8, 0xffffffff, 0x00080007, + 0x91ac, 0xffffffff, 0x000b000a, + 0x91b0, 0xffffffff, 0x00050004, + 0x91b4, 0xffffffff, 0x00070006, + 0x91b8, 0xffffffff, 0x0008000b, + 0x91bc, 0xffffffff, 0x000a0009, + 0x91c0, 0xffffffff, 0x000d000c, + 0x9200, 0xffffffff, 0x00090008, + 0x9204, 0xffffffff, 0x000b000a, + 0x9208, 0xffffffff, 0x000c000f, + 0x920c, 0xffffffff, 0x000e000d, + 0x9210, 0xffffffff, 0x00110010, + 0x9214, 0xffffffff, 0x000a0009, + 0x9218, 0xffffffff, 0x000c000b, + 0x921c, 0xffffffff, 0x0000000f, + 0x9220, 0xffffffff, 0x000e000d, + 0x9224, 0xffffffff, 0x00110010, + 0x9228, 0xffffffff, 0x000b000a, + 0x922c, 0xffffffff, 0x000d000c, + 0x9230, 0xffffffff, 0x00000010, + 0x9234, 0xffffffff, 0x000f000e, + 0x9238, 0xffffffff, 0x00120011, + 0x923c, 0xffffffff, 0x000c000b, + 0x9240, 0xffffffff, 0x000e000d, + 0x9244, 0xffffffff, 0x00000011, + 0x9248, 0xffffffff, 0x0010000f, + 0x924c, 0xffffffff, 0x00130012, + 0x9250, 0xffffffff, 0x000d000c, + 0x9254, 0xffffffff, 0x000f000e, + 0x9258, 0xffffffff, 0x00100013, + 0x925c, 0xffffffff, 0x00120011, + 0x9260, 0xffffffff, 0x00150014, + 0x9150, 0xffffffff, 0x96940200, + 0x8708, 0xffffffff, 0x00900100, + 0xc478, 0xffffffff, 0x00000080, + 0xc404, 0xffffffff, 0x0020003f, + 0x30, 0xffffffff, 0x0000001c, + 0x34, 0x000f0000, 0x000f0000, + 0x160c, 0xffffffff, 0x00000100, + 0x1024, 0xffffffff, 0x00000100, + 0x102c, 0x00000101, 0x00000000, + 0x20a8, 0xffffffff, 0x00000104, + 0x264c, 0x000c0000, 0x000c0000, + 0x2648, 0x000c0000, 0x000c0000, + 0x55e4, 0xff000fff, 0x00000100, + 0x55e8, 0x00000001, 0x00000001, + 0x2f50, 0x00000001, 0x00000001, + 0x30cc, 0xc0000fff, 0x00000104, + 0xc1e4, 0x00000001, 0x00000001, + 0xd0c0, 0xfffffff0, 0x00000100, + 0xd8c0, 0xfffffff0, 0x00000100 +}; + +static const u32 oland_mgcg_cgcg_init[] = +{ + 0xc400, 0xffffffff, 0xfffffffc, + 0x802c, 0xffffffff, 0xe0000000, + 0x9a60, 0xffffffff, 0x00000100, + 0x92a4, 0xffffffff, 0x00000100, + 0xc164, 0xffffffff, 0x00000100, + 0x9774, 0xffffffff, 0x00000100, + 0x8984, 0xffffffff, 0x06000100, + 0x8a18, 0xffffffff, 0x00000100, + 0x92a0, 0xffffffff, 0x00000100, + 0xc380, 0xffffffff, 0x00000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x8d88, 0xffffffff, 0x00000100, + 0x8d8c, 0xffffffff, 0x00000100, + 0x9030, 0xffffffff, 0x00000100, + 0x9034, 0xffffffff, 0x00000100, + 0x9038, 0xffffffff, 0x00000100, + 0x903c, 0xffffffff, 0x00000100, + 0xad80, 0xffffffff, 0x00000100, + 0xac54, 0xffffffff, 0x00000100, + 0x897c, 0xffffffff, 0x06000100, + 0x9868, 0xffffffff, 0x00000100, + 0x9510, 0xffffffff, 0x00000100, + 0xaf04, 0xffffffff, 0x00000100, + 0xae04, 0xffffffff, 0x00000100, + 0x949c, 0xffffffff, 0x00000100, + 0x802c, 0xffffffff, 0xe0000000, + 0x9160, 0xffffffff, 0x00010000, + 0x9164, 0xffffffff, 0x00030002, + 0x9168, 0xffffffff, 0x00040007, + 0x916c, 0xffffffff, 0x00060005, + 0x9170, 0xffffffff, 0x00090008, + 0x9174, 0xffffffff, 0x00020001, + 0x9178, 0xffffffff, 0x00040003, + 0x917c, 0xffffffff, 0x00000007, + 0x9180, 0xffffffff, 0x00060005, + 0x9184, 0xffffffff, 0x00090008, + 0x9188, 0xffffffff, 0x00030002, + 0x918c, 0xffffffff, 0x00050004, + 0x9190, 0xffffffff, 0x00000008, + 0x9194, 0xffffffff, 0x00070006, + 0x9198, 0xffffffff, 0x000a0009, + 0x919c, 0xffffffff, 0x00040003, + 0x91a0, 0xffffffff, 0x00060005, + 0x91a4, 0xffffffff, 0x00000009, + 0x91a8, 0xffffffff, 0x00080007, + 0x91ac, 0xffffffff, 0x000b000a, + 0x91b0, 0xffffffff, 0x00050004, + 0x91b4, 0xffffffff, 0x00070006, + 0x91b8, 0xffffffff, 0x0008000b, + 0x91bc, 0xffffffff, 0x000a0009, + 0x91c0, 0xffffffff, 0x000d000c, + 0x91c4, 0xffffffff, 0x00060005, + 0x91c8, 0xffffffff, 0x00080007, + 0x91cc, 0xffffffff, 0x0000000b, + 0x91d0, 0xffffffff, 0x000a0009, + 0x91d4, 0xffffffff, 0x000d000c, + 0x9150, 0xffffffff, 0x96940200, + 0x8708, 0xffffffff, 0x00900100, + 0xc478, 0xffffffff, 0x00000080, + 0xc404, 0xffffffff, 0x0020003f, + 0x30, 0xffffffff, 0x0000001c, + 0x34, 0x000f0000, 0x000f0000, + 0x160c, 0xffffffff, 0x00000100, + 0x1024, 0xffffffff, 0x00000100, + 0x102c, 0x00000101, 0x00000000, + 0x20a8, 0xffffffff, 0x00000104, + 0x264c, 0x000c0000, 0x000c0000, + 0x2648, 0x000c0000, 0x000c0000, + 0x55e4, 0xff000fff, 0x00000100, + 0x55e8, 0x00000001, 0x00000001, + 0x2f50, 0x00000001, 0x00000001, + 0x30cc, 0xc0000fff, 0x00000104, + 0xc1e4, 0x00000001, 0x00000001, + 0xd0c0, 0xfffffff0, 0x00000100, + 0xd8c0, 0xfffffff0, 0x00000100 +}; + +static const u32 hainan_mgcg_cgcg_init[] = +{ + 0xc400, 0xffffffff, 0xfffffffc, + 0x802c, 0xffffffff, 0xe0000000, + 0x9a60, 0xffffffff, 0x00000100, + 0x92a4, 0xffffffff, 0x00000100, + 0xc164, 0xffffffff, 0x00000100, + 0x9774, 0xffffffff, 0x00000100, + 0x8984, 0xffffffff, 0x06000100, + 0x8a18, 0xffffffff, 0x00000100, + 0x92a0, 0xffffffff, 0x00000100, + 0xc380, 0xffffffff, 0x00000100, + 0x8b28, 0xffffffff, 0x00000100, + 0x9144, 0xffffffff, 0x00000100, + 0x8d88, 0xffffffff, 0x00000100, + 0x8d8c, 0xffffffff, 0x00000100, + 0x9030, 0xffffffff, 0x00000100, + 0x9034, 0xffffffff, 0x00000100, + 0x9038, 0xffffffff, 0x00000100, + 0x903c, 0xffffffff, 0x00000100, + 0xad80, 0xffffffff, 0x00000100, + 0xac54, 0xffffffff, 0x00000100, + 0x897c, 0xffffffff, 0x06000100, + 0x9868, 0xffffffff, 0x00000100, + 0x9510, 0xffffffff, 0x00000100, + 0xaf04, 0xffffffff, 0x00000100, + 0xae04, 0xffffffff, 0x00000100, + 0x949c, 0xffffffff, 0x00000100, + 0x802c, 0xffffffff, 0xe0000000, + 0x9160, 0xffffffff, 0x00010000, + 0x9164, 0xffffffff, 0x00030002, + 0x9168, 0xffffffff, 0x00040007, + 0x916c, 0xffffffff, 0x00060005, + 0x9170, 0xffffffff, 0x00090008, + 0x9174, 0xffffffff, 0x00020001, + 0x9178, 0xffffffff, 0x00040003, + 0x917c, 0xffffffff, 0x00000007, + 0x9180, 0xffffffff, 0x00060005, + 0x9184, 0xffffffff, 0x00090008, + 0x9188, 0xffffffff, 0x00030002, + 0x918c, 0xffffffff, 0x00050004, + 0x9190, 0xffffffff, 0x00000008, + 0x9194, 0xffffffff, 0x00070006, + 0x9198, 0xffffffff, 0x000a0009, + 0x919c, 0xffffffff, 0x00040003, + 0x91a0, 0xffffffff, 0x00060005, + 0x91a4, 0xffffffff, 0x00000009, + 0x91a8, 0xffffffff, 0x00080007, + 0x91ac, 0xffffffff, 0x000b000a, + 0x91b0, 0xffffffff, 0x00050004, + 0x91b4, 0xffffffff, 0x00070006, + 0x91b8, 0xffffffff, 0x0008000b, + 0x91bc, 0xffffffff, 0x000a0009, + 0x91c0, 0xffffffff, 0x000d000c, + 0x91c4, 0xffffffff, 0x00060005, + 0x91c8, 0xffffffff, 0x00080007, + 0x91cc, 0xffffffff, 0x0000000b, + 0x91d0, 0xffffffff, 0x000a0009, + 0x91d4, 0xffffffff, 0x000d000c, + 0x9150, 0xffffffff, 0x96940200, + 0x8708, 0xffffffff, 0x00900100, + 0xc478, 0xffffffff, 0x00000080, + 0xc404, 0xffffffff, 0x0020003f, + 0x30, 0xffffffff, 0x0000001c, + 0x34, 0x000f0000, 0x000f0000, + 0x160c, 0xffffffff, 0x00000100, + 0x1024, 0xffffffff, 0x00000100, + 0x20a8, 0xffffffff, 0x00000104, + 0x264c, 0x000c0000, 0x000c0000, + 0x2648, 0x000c0000, 0x000c0000, + 0x2f50, 0x00000001, 0x00000001, + 0x30cc, 0xc0000fff, 0x00000104, + 0xc1e4, 0x00000001, 0x00000001, + 0xd0c0, 0xfffffff0, 0x00000100, + 0xd8c0, 0xfffffff0, 0x00000100 +}; + +static u32 verde_pg_init[] = +{ + 0x353c, 0xffffffff, 0x40000, + 0x3538, 0xffffffff, 0x200010ff, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x7007, + 0x3538, 0xffffffff, 0x300010ff, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x400000, + 0x3538, 0xffffffff, 0x100010ff, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x120200, + 0x3538, 0xffffffff, 0x500010ff, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x1e1e16, + 0x3538, 0xffffffff, 0x600010ff, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x171f1e, + 0x3538, 0xffffffff, 0x700010ff, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x353c, 0xffffffff, 0x0, + 0x3538, 0xffffffff, 0x9ff, + 0x3500, 0xffffffff, 0x0, + 0x3504, 0xffffffff, 0x10000800, + 0x3504, 0xffffffff, 0xf, + 0x3504, 0xffffffff, 0xf, + 0x3500, 0xffffffff, 0x4, + 0x3504, 0xffffffff, 0x1000051e, + 0x3504, 0xffffffff, 0xffff, + 0x3504, 0xffffffff, 0xffff, + 0x3500, 0xffffffff, 0x8, + 0x3504, 0xffffffff, 0x80500, + 0x3500, 0xffffffff, 0x12, + 0x3504, 0xffffffff, 0x9050c, + 0x3500, 0xffffffff, 0x1d, + 0x3504, 0xffffffff, 0xb052c, + 0x3500, 0xffffffff, 0x2a, + 0x3504, 0xffffffff, 0x1053e, + 0x3500, 0xffffffff, 0x2d, + 0x3504, 0xffffffff, 0x10546, + 0x3500, 0xffffffff, 0x30, + 0x3504, 0xffffffff, 0xa054e, + 0x3500, 0xffffffff, 0x3c, + 0x3504, 0xffffffff, 0x1055f, + 0x3500, 0xffffffff, 0x3f, + 0x3504, 0xffffffff, 0x10567, + 0x3500, 0xffffffff, 0x42, + 0x3504, 0xffffffff, 0x1056f, + 0x3500, 0xffffffff, 0x45, + 0x3504, 0xffffffff, 0x10572, + 0x3500, 0xffffffff, 0x48, + 0x3504, 0xffffffff, 0x20575, + 0x3500, 0xffffffff, 0x4c, + 0x3504, 0xffffffff, 0x190801, + 0x3500, 0xffffffff, 0x67, + 0x3504, 0xffffffff, 0x1082a, + 0x3500, 0xffffffff, 0x6a, + 0x3504, 0xffffffff, 0x1b082d, + 0x3500, 0xffffffff, 0x87, + 0x3504, 0xffffffff, 0x310851, + 0x3500, 0xffffffff, 0xba, + 0x3504, 0xffffffff, 0x891, + 0x3500, 0xffffffff, 0xbc, + 0x3504, 0xffffffff, 0x893, + 0x3500, 0xffffffff, 0xbe, + 0x3504, 0xffffffff, 0x20895, + 0x3500, 0xffffffff, 0xc2, + 0x3504, 0xffffffff, 0x20899, + 0x3500, 0xffffffff, 0xc6, + 0x3504, 0xffffffff, 0x2089d, + 0x3500, 0xffffffff, 0xca, + 0x3504, 0xffffffff, 0x8a1, + 0x3500, 0xffffffff, 0xcc, + 0x3504, 0xffffffff, 0x8a3, + 0x3500, 0xffffffff, 0xce, + 0x3504, 0xffffffff, 0x308a5, + 0x3500, 0xffffffff, 0xd3, + 0x3504, 0xffffffff, 0x6d08cd, + 0x3500, 0xffffffff, 0x142, + 0x3504, 0xffffffff, 0x2000095a, + 0x3504, 0xffffffff, 0x1, + 0x3500, 0xffffffff, 0x144, + 0x3504, 0xffffffff, 0x301f095b, + 0x3500, 0xffffffff, 0x165, + 0x3504, 0xffffffff, 0xc094d, + 0x3500, 0xffffffff, 0x173, + 0x3504, 0xffffffff, 0xf096d, + 0x3500, 0xffffffff, 0x184, + 0x3504, 0xffffffff, 0x15097f, + 0x3500, 0xffffffff, 0x19b, + 0x3504, 0xffffffff, 0xc0998, + 0x3500, 0xffffffff, 0x1a9, + 0x3504, 0xffffffff, 0x409a7, + 0x3500, 0xffffffff, 0x1af, + 0x3504, 0xffffffff, 0xcdc, + 0x3500, 0xffffffff, 0x1b1, + 0x3504, 0xffffffff, 0x800, + 0x3508, 0xffffffff, 0x6c9b2000, + 0x3510, 0xfc00, 0x2000, + 0x3544, 0xffffffff, 0xfc0, + 0x28d4, 0x00000100, 0x100 +}; + +static void si_init_golden_registers(struct radeon_device *rdev) +{ + switch (rdev->family) { + case CHIP_TAHITI: + radeon_program_register_sequence(rdev, + tahiti_golden_registers, + (const u32)ARRAY_SIZE(tahiti_golden_registers)); + radeon_program_register_sequence(rdev, + tahiti_golden_rlc_registers, + (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers)); + radeon_program_register_sequence(rdev, + tahiti_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init)); + radeon_program_register_sequence(rdev, + tahiti_golden_registers2, + (const u32)ARRAY_SIZE(tahiti_golden_registers2)); + break; + case CHIP_PITCAIRN: + radeon_program_register_sequence(rdev, + pitcairn_golden_registers, + (const u32)ARRAY_SIZE(pitcairn_golden_registers)); + radeon_program_register_sequence(rdev, + pitcairn_golden_rlc_registers, + (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers)); + radeon_program_register_sequence(rdev, + pitcairn_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init)); + break; + case CHIP_VERDE: + radeon_program_register_sequence(rdev, + verde_golden_registers, + (const u32)ARRAY_SIZE(verde_golden_registers)); + radeon_program_register_sequence(rdev, + verde_golden_rlc_registers, + (const u32)ARRAY_SIZE(verde_golden_rlc_registers)); + radeon_program_register_sequence(rdev, + verde_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init)); + radeon_program_register_sequence(rdev, + verde_pg_init, + (const u32)ARRAY_SIZE(verde_pg_init)); + break; + case CHIP_OLAND: + radeon_program_register_sequence(rdev, + oland_golden_registers, + (const u32)ARRAY_SIZE(oland_golden_registers)); + radeon_program_register_sequence(rdev, + oland_golden_rlc_registers, + (const u32)ARRAY_SIZE(oland_golden_rlc_registers)); + radeon_program_register_sequence(rdev, + oland_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init)); + break; + case CHIP_HAINAN: + radeon_program_register_sequence(rdev, + hainan_golden_registers, + (const u32)ARRAY_SIZE(hainan_golden_registers)); + radeon_program_register_sequence(rdev, + hainan_golden_registers2, + (const u32)ARRAY_SIZE(hainan_golden_registers2)); + radeon_program_register_sequence(rdev, + hainan_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init)); + break; + default: + break; + } +} + +/** + * si_get_allowed_info_register - fetch the register for the info ioctl + * + * @rdev: radeon_device pointer + * @reg: register offset in bytes + * @val: register value + * + * Returns 0 for success or -EINVAL for an invalid register + * + */ +int si_get_allowed_info_register(struct radeon_device *rdev, + u32 reg, u32 *val) +{ + switch (reg) { + case GRBM_STATUS: + case GRBM_STATUS2: + case GRBM_STATUS_SE0: + case GRBM_STATUS_SE1: + case SRBM_STATUS: + case SRBM_STATUS2: + case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET): + case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET): + case UVD_STATUS: + *val = RREG32(reg); + return 0; + default: + return -EINVAL; + } +} + +#define PCIE_BUS_CLK 10000 +#define TCLK (PCIE_BUS_CLK / 10) + +/** + * si_get_xclk - get the xclk + * + * @rdev: radeon_device pointer + * + * Returns the reference clock used by the gfx engine + * (SI). + */ +u32 si_get_xclk(struct radeon_device *rdev) +{ + u32 reference_clock = rdev->clock.spll.reference_freq; + u32 tmp; + + tmp = RREG32(CG_CLKPIN_CNTL_2); + if (tmp & MUX_TCLK_TO_XCLK) + return TCLK; + + tmp = RREG32(CG_CLKPIN_CNTL); + if (tmp & XTALIN_DIVIDE) + return reference_clock / 4; + + return reference_clock; +} /* get temperature in millidegrees */ int si_get_temp(struct radeon_device *rdev) @@ -197,36 +1456,135 @@ static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { {0x0000009f, 0x00a37400} }; +static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { + {0x0000006f, 0x03044000}, + {0x00000070, 0x0480c018}, + {0x00000071, 0x00000040}, + {0x00000072, 0x01000000}, + {0x00000074, 0x000000ff}, + {0x00000075, 0x00143400}, + {0x00000076, 0x08ec0800}, + {0x00000077, 0x040000cc}, + {0x00000079, 0x00000000}, + {0x0000007a, 0x21000409}, + {0x0000007c, 0x00000000}, + {0x0000007d, 0xe8000000}, + {0x0000007e, 0x044408a8}, + {0x0000007f, 0x00000003}, + {0x00000080, 0x00000000}, + {0x00000081, 0x01000000}, + {0x00000082, 0x02000000}, + {0x00000083, 0x00000000}, + {0x00000084, 0xe3f3e4f4}, + {0x00000085, 0x00052024}, + {0x00000087, 0x00000000}, + {0x00000088, 0x66036603}, + {0x00000089, 0x01000000}, + {0x0000008b, 0x1c0a0000}, + {0x0000008c, 0xff010000}, + {0x0000008e, 0xffffefff}, + {0x0000008f, 0xfff3efff}, + {0x00000090, 0xfff3efbf}, + {0x00000094, 0x00101101}, + {0x00000095, 0x00000fff}, + {0x00000096, 0x00116fff}, + {0x00000097, 0x60010000}, + {0x00000098, 0x10010000}, + {0x00000099, 0x00006000}, + {0x0000009a, 0x00001000}, + {0x0000009f, 0x00a17730} +}; + +static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { + {0x0000006f, 0x03044000}, + {0x00000070, 0x0480c018}, + {0x00000071, 0x00000040}, + {0x00000072, 0x01000000}, + {0x00000074, 0x000000ff}, + {0x00000075, 0x00143400}, + {0x00000076, 0x08ec0800}, + {0x00000077, 0x040000cc}, + {0x00000079, 0x00000000}, + {0x0000007a, 0x21000409}, + {0x0000007c, 0x00000000}, + {0x0000007d, 0xe8000000}, + {0x0000007e, 0x044408a8}, + {0x0000007f, 0x00000003}, + {0x00000080, 0x00000000}, + {0x00000081, 0x01000000}, + {0x00000082, 0x02000000}, + {0x00000083, 0x00000000}, + {0x00000084, 0xe3f3e4f4}, + {0x00000085, 0x00052024}, + {0x00000087, 0x00000000}, + {0x00000088, 0x66036603}, + {0x00000089, 0x01000000}, + {0x0000008b, 0x1c0a0000}, + {0x0000008c, 0xff010000}, + {0x0000008e, 0xffffefff}, + {0x0000008f, 0xfff3efff}, + {0x00000090, 0xfff3efbf}, + {0x00000094, 0x00101101}, + {0x00000095, 0x00000fff}, + {0x00000096, 0x00116fff}, + {0x00000097, 0x60010000}, + {0x00000098, 0x10010000}, + {0x00000099, 0x00006000}, + {0x0000009a, 0x00001000}, + {0x0000009f, 0x00a07730} +}; + /* ucode loading */ -static int si_mc_load_microcode(struct radeon_device *rdev) +int si_mc_load_microcode(struct radeon_device *rdev) { - const __be32 *fw_data; + const __be32 *fw_data = NULL; + const __le32 *new_fw_data = NULL; u32 running, blackout = 0; - u32 *io_mc_regs; + u32 *io_mc_regs = NULL; + const __le32 *new_io_mc_regs = NULL; int i, regs_size, ucode_size; if (!rdev->mc_fw) return -EINVAL; - ucode_size = rdev->mc_fw_size / 4; + if (rdev->new_fw) { + const struct mc_firmware_header_v1_0 *hdr = + (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; + + radeon_ucode_print_mc_hdr(&hdr->header); + regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); + new_io_mc_regs = (const __le32 *) + (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); + ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; + new_fw_data = (const __le32 *) + (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + } else { + ucode_size = rdev->mc_fw->size / 4; - switch (rdev->family) { - case CHIP_TAHITI: - io_mc_regs = (u32 *)&tahiti_io_mc_regs; - ucode_size = SI_MC_UCODE_SIZE; - regs_size = TAHITI_IO_MC_REGS_SIZE; - break; - case CHIP_PITCAIRN: - io_mc_regs = (u32 *)&pitcairn_io_mc_regs; - ucode_size = SI_MC_UCODE_SIZE; - regs_size = TAHITI_IO_MC_REGS_SIZE; - break; - case CHIP_VERDE: - default: - io_mc_regs = (u32 *)&verde_io_mc_regs; - ucode_size = SI_MC_UCODE_SIZE; - regs_size = TAHITI_IO_MC_REGS_SIZE; - break; + switch (rdev->family) { + case CHIP_TAHITI: + io_mc_regs = (u32 *)&tahiti_io_mc_regs; + regs_size = TAHITI_IO_MC_REGS_SIZE; + break; + case CHIP_PITCAIRN: + io_mc_regs = (u32 *)&pitcairn_io_mc_regs; + regs_size = TAHITI_IO_MC_REGS_SIZE; + break; + case CHIP_VERDE: + default: + io_mc_regs = (u32 *)&verde_io_mc_regs; + regs_size = TAHITI_IO_MC_REGS_SIZE; + break; + case CHIP_OLAND: + io_mc_regs = (u32 *)&oland_io_mc_regs; + regs_size = TAHITI_IO_MC_REGS_SIZE; + break; + case CHIP_HAINAN: + io_mc_regs = (u32 *)&hainan_io_mc_regs; + regs_size = TAHITI_IO_MC_REGS_SIZE; + break; + } + fw_data = (const __be32 *)rdev->mc_fw->data; } running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; @@ -243,13 +1601,21 @@ static int si_mc_load_microcode(struct radeon_device *rdev) /* load mc io regs */ for (i = 0; i < regs_size; i++) { - WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); - WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); + if (rdev->new_fw) { + WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++)); + WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++)); + } else { + WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); + WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); + } } /* load the MC ucode */ - fw_data = (const __be32 *)rdev->mc_fw; - for (i = 0; i < ucode_size; i++) - WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); + for (i = 0; i < ucode_size; i++) { + if (rdev->new_fw) + WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++)); + else + WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); + } /* put the engine back into the active state */ WREG32(MC_SEQ_SUP_CNTL, 0x00000008); @@ -278,134 +1644,268 @@ static int si_mc_load_microcode(struct radeon_device *rdev) static int si_init_microcode(struct radeon_device *rdev) { const char *chip_name; - const char *rlc_chip_name; + const char *new_chip_name; size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size; + size_t smc_req_size, mc2_req_size; char fw_name[30]; int err; + int new_fw = 0; DRM_DEBUG("\n"); switch (rdev->family) { case CHIP_TAHITI: - chip_name = "tahiti"; - rlc_chip_name = "tahiti"; + chip_name = "TAHITI"; + new_chip_name = "tahiti"; pfp_req_size = SI_PFP_UCODE_SIZE * 4; me_req_size = SI_PM4_UCODE_SIZE * 4; ce_req_size = SI_CE_UCODE_SIZE * 4; rlc_req_size = SI_RLC_UCODE_SIZE * 4; mc_req_size = SI_MC_UCODE_SIZE * 4; + mc2_req_size = TAHITI_MC_UCODE_SIZE * 4; + smc_req_size = roundup2(TAHITI_SMC_UCODE_SIZE, 4); break; case CHIP_PITCAIRN: - chip_name = "pitcairn"; - rlc_chip_name = "pitcairn"; + chip_name = "PITCAIRN"; + new_chip_name = "pitcairn"; pfp_req_size = SI_PFP_UCODE_SIZE * 4; me_req_size = SI_PM4_UCODE_SIZE * 4; ce_req_size = SI_CE_UCODE_SIZE * 4; rlc_req_size = SI_RLC_UCODE_SIZE * 4; mc_req_size = SI_MC_UCODE_SIZE * 4; + mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4; + smc_req_size = roundup2(PITCAIRN_SMC_UCODE_SIZE, 4); break; case CHIP_VERDE: - chip_name = "verde"; - rlc_chip_name = "verde"; + chip_name = "VERDE"; + new_chip_name = "verde"; pfp_req_size = SI_PFP_UCODE_SIZE * 4; me_req_size = SI_PM4_UCODE_SIZE * 4; ce_req_size = SI_CE_UCODE_SIZE * 4; rlc_req_size = SI_RLC_UCODE_SIZE * 4; mc_req_size = SI_MC_UCODE_SIZE * 4; + mc2_req_size = VERDE_MC_UCODE_SIZE * 4; + smc_req_size = roundup2(VERDE_SMC_UCODE_SIZE, 4); + break; + case CHIP_OLAND: + chip_name = "OLAND"; + new_chip_name = "oland"; + pfp_req_size = SI_PFP_UCODE_SIZE * 4; + me_req_size = SI_PM4_UCODE_SIZE * 4; + ce_req_size = SI_CE_UCODE_SIZE * 4; + rlc_req_size = SI_RLC_UCODE_SIZE * 4; + mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4; + smc_req_size = roundup2(OLAND_SMC_UCODE_SIZE, 4); + break; + case CHIP_HAINAN: + chip_name = "HAINAN"; + new_chip_name = "hainan"; + pfp_req_size = SI_PFP_UCODE_SIZE * 4; + me_req_size = SI_PM4_UCODE_SIZE * 4; + ce_req_size = SI_CE_UCODE_SIZE * 4; + rlc_req_size = SI_RLC_UCODE_SIZE * 4; + mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4; + smc_req_size = roundup2(HAINAN_SMC_UCODE_SIZE, 4); break; default: BUG(); } - DRM_INFO("Loading %s Microcode\n", chip_name); - - snprintf(fw_name, sizeof(fw_name), "radeon-%s_pfp", chip_name); - err = loadfirmware(fw_name, &rdev->pfp_fw, &rdev->pfp_fw_size); - if (err) - goto out; - if (rdev->pfp_fw_size != pfp_req_size) { - DRM_ERROR( - "si_cp: Bogus length %zu in firmware \"%s\"\n", - rdev->pfp_fw_size, fw_name); - err = -EINVAL; - goto out; - } - - snprintf(fw_name, sizeof(fw_name), "radeon-%s_me", chip_name); - err = loadfirmware(fw_name, &rdev->me_fw, &rdev->me_fw_size); - if (err) - goto out; - if (rdev->me_fw_size != me_req_size) { - DRM_ERROR( - "si_cp: Bogus length %zu in firmware \"%s\"\n", - rdev->me_fw_size, fw_name); - err = -EINVAL; - } + DRM_INFO("Loading %s Microcode\n", new_chip_name); - snprintf(fw_name, sizeof(fw_name), "radeon-%s_ce", chip_name); - err = loadfirmware(fw_name, &rdev->ce_fw, &rdev->ce_fw_size); - if (err) - goto out; - if (rdev->ce_fw_size != ce_req_size) { - DRM_ERROR( - "si_cp: Bogus length %zu in firmware \"%s\"\n", - rdev->ce_fw_size, fw_name); - err = -EINVAL; + snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name); + err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); + err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->pfp_fw->size != pfp_req_size) { + printk(KERN_ERR + "si_cp: Bogus length %zu in firmware \"%s\"\n", + rdev->pfp_fw->size, fw_name); + err = -EINVAL; + goto out; + } + } else { + err = radeon_ucode_validate(rdev->pfp_fw); + if (err) { + printk(KERN_ERR + "si_cp: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } } - snprintf(fw_name, sizeof(fw_name), "radeon-%s_rlc", rlc_chip_name); - err = loadfirmware(fw_name, &rdev->rlc_fw, &rdev->rlc_fw_size); - if (err) - goto out; - if (rdev->rlc_fw_size != rlc_req_size) { - DRM_ERROR( - "si_rlc: Bogus length %zu in firmware \"%s\"\n", - rdev->rlc_fw_size, fw_name); - err = -EINVAL; + snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name); + err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); + err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->me_fw->size != me_req_size) { + printk(KERN_ERR + "si_cp: Bogus length %zu in firmware \"%s\"\n", + rdev->me_fw->size, fw_name); + err = -EINVAL; + } + } else { + err = radeon_ucode_validate(rdev->me_fw); + if (err) { + printk(KERN_ERR + "si_cp: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } } - snprintf(fw_name, sizeof(fw_name), "radeon-%s_mc", chip_name); - err = loadfirmware(fw_name, &rdev->mc_fw, &rdev->mc_fw_size); - if (err) - goto out; - if (rdev->mc_fw_size != mc_req_size) { - DRM_ERROR( - "si_mc: Bogus length %zu in firmware \"%s\"\n", - rdev->mc_fw_size, fw_name); - err = -EINVAL; + snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name); + err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); + err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->ce_fw->size != ce_req_size) { + printk(KERN_ERR + "si_cp: Bogus length %zu in firmware \"%s\"\n", + rdev->ce_fw->size, fw_name); + err = -EINVAL; + } + } else { + err = radeon_ucode_validate(rdev->ce_fw); + if (err) { + printk(KERN_ERR + "si_cp: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } } -out: + snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name); + err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); if (err) { - if (err != -EINVAL) + snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name); + err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); + if (err) + goto out; + if (rdev->rlc_fw->size != rlc_req_size) { printk(KERN_ERR - "si_cp: Failed to load firmware \"%s\"\n", - fw_name); - if (rdev->pfp_fw) { - free(rdev->pfp_fw, M_DEVBUF, 0); - rdev->pfp_fw = NULL; + "si_rlc: Bogus length %zu in firmware \"%s\"\n", + rdev->rlc_fw->size, fw_name); + err = -EINVAL; } - if (rdev->me_fw) { - free(rdev->me_fw, M_DEVBUF, 0); - rdev->me_fw = NULL; + } else { + err = radeon_ucode_validate(rdev->rlc_fw); + if (err) { + printk(KERN_ERR + "si_cp: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; } - if (rdev->ce_fw) { - free(rdev->ce_fw, M_DEVBUF, 0); - rdev->ce_fw = NULL; + } + + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name); + err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name); + err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); + err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); + if (err) + goto out; } - if (rdev->rlc_fw) { - free(rdev->rlc_fw, M_DEVBUF, 0); - rdev->rlc_fw = NULL; + if ((rdev->mc_fw->size != mc_req_size) && + (rdev->mc_fw->size != mc2_req_size)) { + printk(KERN_ERR + "si_mc: Bogus length %zu in firmware \"%s\"\n", + rdev->mc_fw->size, fw_name); + err = -EINVAL; } - if (rdev->mc_fw) { - free(rdev->mc_fw, M_DEVBUF, 0); - rdev->mc_fw = NULL; + DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); + } else { + err = radeon_ucode_validate(rdev->mc_fw); + if (err) { + printk(KERN_ERR + "si_cp: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; } } - return err; -} -/* watermark setup */ -static u32 dce6_line_buffer_adjust(struct radeon_device *rdev, + snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name); + err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); + if (err) { + snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); + err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); + if (err) { + printk(KERN_ERR + "smc: error loading firmware \"%s\"\n", + fw_name); + release_firmware(rdev->smc_fw); + rdev->smc_fw = NULL; + err = 0; + } else if (rdev->smc_fw->size != smc_req_size) { + printk(KERN_ERR + "si_smc: Bogus length %zu in firmware \"%s\"\n", + rdev->smc_fw->size, fw_name); + err = -EINVAL; + } + } else { + err = radeon_ucode_validate(rdev->smc_fw); + if (err) { + printk(KERN_ERR + "si_cp: validation failed for firmware \"%s\"\n", + fw_name); + goto out; + } else { + new_fw++; + } + } + + if (new_fw == 0) { + rdev->new_fw = false; + } else if (new_fw < 6) { + printk(KERN_ERR "si_fw: mixing new and old firmware!\n"); + err = -EINVAL; + } else { + rdev->new_fw = true; + } +out: + if (err) { + if (err != -EINVAL) + printk(KERN_ERR + "si_cp: Failed to load firmware \"%s\"\n", + fw_name); + release_firmware(rdev->pfp_fw); + rdev->pfp_fw = NULL; + release_firmware(rdev->me_fw); + rdev->me_fw = NULL; + release_firmware(rdev->ce_fw); + rdev->ce_fw = NULL; + release_firmware(rdev->rlc_fw); + rdev->rlc_fw = NULL; + release_firmware(rdev->mc_fw); + rdev->mc_fw = NULL; + release_firmware(rdev->smc_fw); + rdev->smc_fw = NULL; + } + return err; +} + +/* watermark setup */ +static u32 dce6_line_buffer_adjust(struct radeon_device *rdev, struct radeon_crtc *radeon_crtc, struct drm_display_mode *mode, struct drm_display_mode *other_mode) @@ -750,7 +2250,8 @@ static void dce6_program_watermarks(struct radeon_device *rdev, u32 lb_size, u32 num_heads) { struct drm_display_mode *mode = &radeon_crtc->base.mode; - struct dce6_wm_params wm; + struct dce6_wm_params wm_low, wm_high; + u32 dram_channels; u32 pixel_period; u32 line_time = 0; u32 latency_watermark_a = 0, latency_watermark_b = 0; @@ -766,38 +2267,83 @@ static void dce6_program_watermarks(struct radeon_device *rdev, priority_a_cnt = 0; priority_b_cnt = 0; - wm.yclk = rdev->pm.current_mclk * 10; - wm.sclk = rdev->pm.current_sclk * 10; - wm.disp_clk = mode->clock; - wm.src_width = mode->crtc_hdisplay; - wm.active_time = mode->crtc_hdisplay * pixel_period; - wm.blank_time = line_time - wm.active_time; - wm.interlaced = false; - if (mode->flags & DRM_MODE_FLAG_INTERLACE) - wm.interlaced = true; - wm.vsc = radeon_crtc->vsc; - wm.vtaps = 1; - if (radeon_crtc->rmx_type != RMX_OFF) - wm.vtaps = 2; - wm.bytes_per_pixel = 4; /* XXX: get this from fb config */ - wm.lb_size = lb_size; if (rdev->family == CHIP_ARUBA) - wm.dram_channels = evergreen_get_number_of_dram_channels(rdev); + dram_channels = evergreen_get_number_of_dram_channels(rdev); else - wm.dram_channels = si_get_number_of_dram_channels(rdev); - wm.num_heads = num_heads; + dram_channels = si_get_number_of_dram_channels(rdev); + + /* watermark for high clocks */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + wm_high.yclk = + radeon_dpm_get_mclk(rdev, false) * 10; + wm_high.sclk = + radeon_dpm_get_sclk(rdev, false) * 10; + } else { + wm_high.yclk = rdev->pm.current_mclk * 10; + wm_high.sclk = rdev->pm.current_sclk * 10; + } + + wm_high.disp_clk = mode->clock; + wm_high.src_width = mode->crtc_hdisplay; + wm_high.active_time = mode->crtc_hdisplay * pixel_period; + wm_high.blank_time = line_time - wm_high.active_time; + wm_high.interlaced = false; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + wm_high.interlaced = true; + wm_high.vsc = radeon_crtc->vsc; + wm_high.vtaps = 1; + if (radeon_crtc->rmx_type != RMX_OFF) + wm_high.vtaps = 2; + wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ + wm_high.lb_size = lb_size; + wm_high.dram_channels = dram_channels; + wm_high.num_heads = num_heads; + + /* watermark for low clocks */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + wm_low.yclk = + radeon_dpm_get_mclk(rdev, true) * 10; + wm_low.sclk = + radeon_dpm_get_sclk(rdev, true) * 10; + } else { + wm_low.yclk = rdev->pm.current_mclk * 10; + wm_low.sclk = rdev->pm.current_sclk * 10; + } + + wm_low.disp_clk = mode->clock; + wm_low.src_width = mode->crtc_hdisplay; + wm_low.active_time = mode->crtc_hdisplay * pixel_period; + wm_low.blank_time = line_time - wm_low.active_time; + wm_low.interlaced = false; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + wm_low.interlaced = true; + wm_low.vsc = radeon_crtc->vsc; + wm_low.vtaps = 1; + if (radeon_crtc->rmx_type != RMX_OFF) + wm_low.vtaps = 2; + wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ + wm_low.lb_size = lb_size; + wm_low.dram_channels = dram_channels; + wm_low.num_heads = num_heads; /* set for high clocks */ - latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535); + latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535); /* set for low clocks */ - /* wm.yclk = low clk; wm.sclk = low clk */ - latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535); + latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535); /* possibly force display priority to high */ /* should really do this at mode validation time... */ - if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) || - !dce6_average_bandwidth_vs_available_bandwidth(&wm) || - !dce6_check_latency_hiding(&wm) || + if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || + !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) || + !dce6_check_latency_hiding(&wm_high) || + (rdev->disp_priority == 2)) { + DRM_DEBUG_KMS("force priority to high\n"); + priority_a_cnt |= PRIORITY_ALWAYS_ON; + priority_b_cnt |= PRIORITY_ALWAYS_ON; + } + if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || + !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) || + !dce6_check_latency_hiding(&wm_low) || (rdev->disp_priority == 2)) { DRM_DEBUG_KMS("force priority to high\n"); priority_a_cnt |= PRIORITY_ALWAYS_ON; @@ -827,6 +2373,9 @@ static void dce6_program_watermarks(struct radeon_device *rdev, c.full = dfixed_div(c, a); priority_b_mark = dfixed_trunc(c); priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; + + /* Save number of lines the linebuffer leads before the scanout */ + radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay); } /* select wm A */ @@ -853,6 +2402,10 @@ static void dce6_program_watermarks(struct radeon_device *rdev, WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); + /* save values for DPM */ + radeon_crtc->line_time = line_time; + radeon_crtc->wm_high = latency_watermark_a; + radeon_crtc->wm_low = latency_watermark_b; } void dce6_bandwidth_update(struct radeon_device *rdev) @@ -862,6 +2415,9 @@ void dce6_bandwidth_update(struct radeon_device *rdev) u32 num_heads = 0, lb_size; int i; + if (!rdev->mode_info.mode_config_initialized) + return; + radeon_update_display_priority(rdev); for (i = 0; i < rdev->num_crtc; i++) { @@ -1137,9 +2693,12 @@ static void si_tiling_mode_table_init(struct radeon_device *rdev) gb_tile_moden = 0; break; } + rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); } - } else if (rdev->family == CHIP_VERDE) { + } else if ((rdev->family == CHIP_VERDE) || + (rdev->family == CHIP_OLAND) || + (rdev->family == CHIP_HAINAN)) { for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { switch (reg_offset) { case 0: /* non-AA compressed depth or any compressed stencil */ @@ -1376,6 +2935,7 @@ static void si_tiling_mode_table_init(struct radeon_device *rdev) gb_tile_moden = 0; break; } + rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); } } else @@ -1586,6 +3146,40 @@ static void si_gpu_init(struct radeon_device *rdev) rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; break; + case CHIP_OLAND: + rdev->config.si.max_shader_engines = 1; + rdev->config.si.max_tile_pipes = 4; + rdev->config.si.max_cu_per_sh = 6; + rdev->config.si.max_sh_per_se = 1; + rdev->config.si.max_backends_per_se = 2; + rdev->config.si.max_texture_channel_caches = 4; + rdev->config.si.max_gprs = 256; + rdev->config.si.max_gs_threads = 16; + rdev->config.si.max_hw_contexts = 8; + + rdev->config.si.sc_prim_fifo_size_frontend = 0x20; + rdev->config.si.sc_prim_fifo_size_backend = 0x40; + rdev->config.si.sc_hiz_tile_fifo_size = 0x30; + rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; + gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; + break; + case CHIP_HAINAN: + rdev->config.si.max_shader_engines = 1; + rdev->config.si.max_tile_pipes = 4; + rdev->config.si.max_cu_per_sh = 5; + rdev->config.si.max_sh_per_se = 1; + rdev->config.si.max_backends_per_se = 1; + rdev->config.si.max_texture_channel_caches = 2; + rdev->config.si.max_gprs = 256; + rdev->config.si.max_gs_threads = 16; + rdev->config.si.max_hw_contexts = 8; + + rdev->config.si.sc_prim_fifo_size_frontend = 0x20; + rdev->config.si.sc_prim_fifo_size_backend = 0x40; + rdev->config.si.sc_hiz_tile_fifo_size = 0x30; + rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; + gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN; + break; } /* Initialize HDP */ @@ -1598,6 +3192,8 @@ static void si_gpu_init(struct radeon_device *rdev) } WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); + WREG32(SRBM_INT_CNTL, 1); + WREG32(SRBM_INT_ACK, 1); evergreen_fix_pci_max_read_req_size(rdev); @@ -1679,6 +3275,11 @@ static void si_gpu_init(struct radeon_device *rdev) WREG32(HDP_ADDR_CONFIG, gb_addr_config); WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config); WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config); + if (rdev->has_uvd) { + WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); + WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); + WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); + } si_tiling_mode_table_init(rdev); @@ -1690,6 +3291,13 @@ static void si_gpu_init(struct radeon_device *rdev) rdev->config.si.max_sh_per_se, rdev->config.si.max_cu_per_sh); + rdev->config.si.active_cus = 0; + for (i = 0; i < rdev->config.si.max_shader_engines; i++) { + for (j = 0; j < rdev->config.si.max_sh_per_se; j++) { + rdev->config.si.active_cus += + hweight32(si_get_cu_active_bitmap(rdev, i, j)); + } + } /* set HW defaults for 3D engine */ WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | @@ -1778,7 +3386,7 @@ void si_fence_ring_emit(struct radeon_device *rdev, /* EVENT_WRITE_EOP - flush caches, send int */ radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5)); - radeon_ring_write(ring, addr & 0xffffffff); + radeon_ring_write(ring, lower_32_bits(addr)); radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); radeon_ring_write(ring, fence->seq); radeon_ring_write(ring, 0); @@ -1790,6 +3398,7 @@ void si_fence_ring_emit(struct radeon_device *rdev, void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) { struct radeon_ring *ring = &rdev->ring[ib->ring]; + unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; u32 header; if (ib->is_const_ib) { @@ -1811,7 +3420,7 @@ void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); radeon_ring_write(ring, (1 << 8)); radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); + radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); radeon_ring_write(ring, next_rptr); } @@ -1825,14 +3434,13 @@ void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) #endif (ib->gpu_addr & 0xFFFFFFFC)); radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); - radeon_ring_write(ring, ib->length_dw | - (ib->vm ? (ib->vm->id << 24) : 0)); + radeon_ring_write(ring, ib->length_dw | (vm_id << 24)); if (!ib->is_const_ib) { /* flush read cache over gart for this vmid */ radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); - radeon_ring_write(ring, ib->vm ? ib->vm->id : 0); + radeon_ring_write(ring, vm_id); radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | PACKET3_TC_ACTION_ENA | @@ -1852,7 +3460,8 @@ static void si_cp_enable(struct radeon_device *rdev, bool enable) if (enable) WREG32(CP_ME_CNTL, 0); else { - radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); WREG32(SCRATCH_UMSK, 0); rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; @@ -1864,34 +3473,77 @@ static void si_cp_enable(struct radeon_device *rdev, bool enable) static int si_cp_load_microcode(struct radeon_device *rdev) { - const __be32 *fw_data; int i; - if (!rdev->me_fw || !rdev->pfp_fw) + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) return -EINVAL; si_cp_enable(rdev, false); - /* PFP */ - fw_data = (const __be32 *)rdev->pfp_fw; - WREG32(CP_PFP_UCODE_ADDR, 0); - for (i = 0; i < SI_PFP_UCODE_SIZE; i++) - WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); - WREG32(CP_PFP_UCODE_ADDR, 0); - - /* CE */ - fw_data = (const __be32 *)rdev->ce_fw; - WREG32(CP_CE_UCODE_ADDR, 0); - for (i = 0; i < SI_CE_UCODE_SIZE; i++) - WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); - WREG32(CP_CE_UCODE_ADDR, 0); - - /* ME */ - fw_data = (const __be32 *)rdev->me_fw; - WREG32(CP_ME_RAM_WADDR, 0); - for (i = 0; i < SI_PM4_UCODE_SIZE; i++) - WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); - WREG32(CP_ME_RAM_WADDR, 0); + if (rdev->new_fw) { + const struct gfx_firmware_header_v1_0 *pfp_hdr = + (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; + const struct gfx_firmware_header_v1_0 *ce_hdr = + (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; + const struct gfx_firmware_header_v1_0 *me_hdr = + (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; + const __le32 *fw_data; + u32 fw_size; + + radeon_ucode_print_gfx_hdr(&pfp_hdr->header); + radeon_ucode_print_gfx_hdr(&ce_hdr->header); + radeon_ucode_print_gfx_hdr(&me_hdr->header); + + /* PFP */ + fw_data = (const __le32 *) + (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; + WREG32(CP_PFP_UCODE_ADDR, 0); + for (i = 0; i < fw_size; i++) + WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); + WREG32(CP_PFP_UCODE_ADDR, 0); + + /* CE */ + fw_data = (const __le32 *) + (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; + WREG32(CP_CE_UCODE_ADDR, 0); + for (i = 0; i < fw_size; i++) + WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); + WREG32(CP_CE_UCODE_ADDR, 0); + + /* ME */ + fw_data = (const __be32 *) + (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); + fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; + WREG32(CP_ME_RAM_WADDR, 0); + for (i = 0; i < fw_size; i++) + WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++)); + WREG32(CP_ME_RAM_WADDR, 0); + } else { + const __be32 *fw_data; + + /* PFP */ + fw_data = (const __be32 *)rdev->pfp_fw->data; + WREG32(CP_PFP_UCODE_ADDR, 0); + for (i = 0; i < SI_PFP_UCODE_SIZE; i++) + WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); + WREG32(CP_PFP_UCODE_ADDR, 0); + + /* CE */ + fw_data = (const __be32 *)rdev->ce_fw->data; + WREG32(CP_CE_UCODE_ADDR, 0); + for (i = 0; i < SI_CE_UCODE_SIZE; i++) + WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); + WREG32(CP_CE_UCODE_ADDR, 0); + + /* ME */ + fw_data = (const __be32 *)rdev->me_fw->data; + WREG32(CP_ME_RAM_WADDR, 0); + for (i = 0; i < SI_PM4_UCODE_SIZE; i++) + WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); + WREG32(CP_ME_RAM_WADDR, 0); + } WREG32(CP_PFP_UCODE_ADDR, 0); WREG32(CP_CE_UCODE_ADDR, 0); @@ -1924,7 +3576,7 @@ static int si_cp_start(struct radeon_device *rdev) radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); radeon_ring_write(ring, 0xc000); radeon_ring_write(ring, 0xe000); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); si_cp_enable(rdev, true); @@ -1953,7 +3605,7 @@ static int si_cp_start(struct radeon_device *rdev) radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) { ring = &rdev->ring[i]; @@ -1963,7 +3615,7 @@ static int si_cp_start(struct radeon_device *rdev) radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0)); radeon_ring_write(ring, 0); - radeon_ring_unlock_commit(rdev, ring); + radeon_ring_unlock_commit(rdev, ring, false); } return 0; @@ -1994,16 +3646,7 @@ static int si_cp_resume(struct radeon_device *rdev) u32 rb_bufsz; int r; - /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ - WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | - SOFT_RESET_PA | - SOFT_RESET_VGT | - SOFT_RESET_SPI | - SOFT_RESET_SX)); - RREG32(GRBM_SOFT_RESET); - mdelay(15); - WREG32(GRBM_SOFT_RESET, 0); - RREG32(GRBM_SOFT_RESET); + si_enable_gui_idle_interrupt(rdev, false); WREG32(CP_SEM_WAIT_TIMER, 0x0); WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); @@ -2017,8 +3660,8 @@ static int si_cp_resume(struct radeon_device *rdev) /* ring 0 - compute and gfx */ /* Set ring buffer size */ ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; - rb_bufsz = drm_order(ring->ring_size / 8); - tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; + rb_bufsz = order_base_2(ring->ring_size / 8); + tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; #ifdef __BIG_ENDIAN tmp |= BUF_SWAP_32BIT; #endif @@ -2045,13 +3688,11 @@ static int si_cp_resume(struct radeon_device *rdev) WREG32(CP_RB0_BASE, ring->gpu_addr >> 8); - ring->rptr = RREG32(CP_RB0_RPTR); - /* ring1 - compute only */ /* Set ring buffer size */ ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; - rb_bufsz = drm_order(ring->ring_size / 8); - tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; + rb_bufsz = order_base_2(ring->ring_size / 8); + tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; #ifdef __BIG_ENDIAN tmp |= BUF_SWAP_32BIT; #endif @@ -2071,13 +3712,11 @@ static int si_cp_resume(struct radeon_device *rdev) WREG32(CP_RB1_BASE, ring->gpu_addr >> 8); - ring->rptr = RREG32(CP_RB1_RPTR); - /* ring2 - compute only */ /* Set ring buffer size */ ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; - rb_bufsz = drm_order(ring->ring_size / 8); - tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; + rb_bufsz = order_base_2(ring->ring_size / 8); + tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; #ifdef __BIG_ENDIAN tmp |= BUF_SWAP_32BIT; #endif @@ -2097,8 +3736,6 @@ static int si_cp_resume(struct radeon_device *rdev) WREG32(CP_RB2_BASE, ring->gpu_addr >> 8); - ring->rptr = RREG32(CP_RB2_RPTR); - /* start the rings */ si_cp_start(rdev); rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; @@ -2120,157 +3757,373 @@ static int si_cp_resume(struct radeon_device *rdev) rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; } + si_enable_gui_idle_interrupt(rdev, true); + + if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) + radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); + return 0; } -bool si_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +u32 si_gpu_check_soft_reset(struct radeon_device *rdev) { - u32 srbm_status; - u32 grbm_status, grbm_status2; - u32 grbm_status_se0, grbm_status_se1; + u32 reset_mask = 0; + u32 tmp; - srbm_status = RREG32(SRBM_STATUS); - grbm_status = RREG32(GRBM_STATUS); - grbm_status2 = RREG32(GRBM_STATUS2); - grbm_status_se0 = RREG32(GRBM_STATUS_SE0); - grbm_status_se1 = RREG32(GRBM_STATUS_SE1); - if (!(grbm_status & GUI_ACTIVE)) { - radeon_ring_lockup_update(ring); - return false; + /* GRBM_STATUS */ + tmp = RREG32(GRBM_STATUS); + if (tmp & (PA_BUSY | SC_BUSY | + BCI_BUSY | SX_BUSY | + TA_BUSY | VGT_BUSY | + DB_BUSY | CB_BUSY | + GDS_BUSY | SPI_BUSY | + IA_BUSY | IA_BUSY_NO_DMA)) + reset_mask |= RADEON_RESET_GFX; + + if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | + CP_BUSY | CP_COHERENCY_BUSY)) + reset_mask |= RADEON_RESET_CP; + + if (tmp & GRBM_EE_BUSY) + reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; + + /* GRBM_STATUS2 */ + tmp = RREG32(GRBM_STATUS2); + if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) + reset_mask |= RADEON_RESET_RLC; + + /* DMA_STATUS_REG 0 */ + tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET); + if (!(tmp & DMA_IDLE)) + reset_mask |= RADEON_RESET_DMA; + + /* DMA_STATUS_REG 1 */ + tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET); + if (!(tmp & DMA_IDLE)) + reset_mask |= RADEON_RESET_DMA1; + + /* SRBM_STATUS2 */ + tmp = RREG32(SRBM_STATUS2); + if (tmp & DMA_BUSY) + reset_mask |= RADEON_RESET_DMA; + + if (tmp & DMA1_BUSY) + reset_mask |= RADEON_RESET_DMA1; + + /* SRBM_STATUS */ + tmp = RREG32(SRBM_STATUS); + + if (tmp & IH_BUSY) + reset_mask |= RADEON_RESET_IH; + + if (tmp & SEM_BUSY) + reset_mask |= RADEON_RESET_SEM; + + if (tmp & GRBM_RQ_PENDING) + reset_mask |= RADEON_RESET_GRBM; + + if (tmp & VMC_BUSY) + reset_mask |= RADEON_RESET_VMC; + + if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | + MCC_BUSY | MCD_BUSY)) + reset_mask |= RADEON_RESET_MC; + + if (evergreen_is_display_hung(rdev)) + reset_mask |= RADEON_RESET_DISPLAY; + + /* VM_L2_STATUS */ + tmp = RREG32(VM_L2_STATUS); + if (tmp & L2_BUSY) + reset_mask |= RADEON_RESET_VMC; + + /* Skip MC reset as it's mostly likely not hung, just busy */ + if (reset_mask & RADEON_RESET_MC) { + DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); + reset_mask &= ~RADEON_RESET_MC; } - /* force CP activities */ - radeon_ring_force_activity(rdev, ring); - return radeon_ring_test_lockup(rdev, ring); + + return reset_mask; } -static void si_gpu_soft_reset_gfx(struct radeon_device *rdev) +static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) { - u32 grbm_reset = 0; + struct evergreen_mc_save save; + u32 grbm_soft_reset = 0, srbm_soft_reset = 0; + u32 tmp; - if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) + if (reset_mask == 0) return; - dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", - RREG32(GRBM_STATUS)); - dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", - RREG32(GRBM_STATUS2)); - dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", - RREG32(GRBM_STATUS_SE0)); - dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", - RREG32(GRBM_STATUS_SE1)); - dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", - RREG32(SRBM_STATUS)); + dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); + + evergreen_print_gpu_status_regs(rdev); + dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", + RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); + dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", + RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); + + /* disable PG/CG */ + si_fini_pg(rdev); + si_fini_cg(rdev); + + /* stop the rlc */ + si_rlc_stop(rdev); /* Disable CP parsing/prefetching */ WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); - /* reset all the gfx blocks */ - grbm_reset = (SOFT_RESET_CP | - SOFT_RESET_CB | - SOFT_RESET_DB | - SOFT_RESET_GDS | - SOFT_RESET_PA | - SOFT_RESET_SC | - SOFT_RESET_BCI | - SOFT_RESET_SPI | - SOFT_RESET_SX | - SOFT_RESET_TC | - SOFT_RESET_TA | - SOFT_RESET_VGT | - SOFT_RESET_IA); - - dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset); - WREG32(GRBM_SOFT_RESET, grbm_reset); - (void)RREG32(GRBM_SOFT_RESET); + if (reset_mask & RADEON_RESET_DMA) { + /* dma0 */ + tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); + tmp &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); + } + if (reset_mask & RADEON_RESET_DMA1) { + /* dma1 */ + tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); + tmp &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); + } + udelay(50); - WREG32(GRBM_SOFT_RESET, 0); - (void)RREG32(GRBM_SOFT_RESET); - - dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", - RREG32(GRBM_STATUS)); - dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", - RREG32(GRBM_STATUS2)); - dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", - RREG32(GRBM_STATUS_SE0)); - dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", - RREG32(GRBM_STATUS_SE1)); - dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", - RREG32(SRBM_STATUS)); + + evergreen_mc_stop(rdev, &save); + if (evergreen_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); + } + + if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) { + grbm_soft_reset = SOFT_RESET_CB | + SOFT_RESET_DB | + SOFT_RESET_GDS | + SOFT_RESET_PA | + SOFT_RESET_SC | + SOFT_RESET_BCI | + SOFT_RESET_SPI | + SOFT_RESET_SX | + SOFT_RESET_TC | + SOFT_RESET_TA | + SOFT_RESET_VGT | + SOFT_RESET_IA; + } + + if (reset_mask & RADEON_RESET_CP) { + grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT; + + srbm_soft_reset |= SOFT_RESET_GRBM; + } + + if (reset_mask & RADEON_RESET_DMA) + srbm_soft_reset |= SOFT_RESET_DMA; + + if (reset_mask & RADEON_RESET_DMA1) + srbm_soft_reset |= SOFT_RESET_DMA1; + + if (reset_mask & RADEON_RESET_DISPLAY) + srbm_soft_reset |= SOFT_RESET_DC; + + if (reset_mask & RADEON_RESET_RLC) + grbm_soft_reset |= SOFT_RESET_RLC; + + if (reset_mask & RADEON_RESET_SEM) + srbm_soft_reset |= SOFT_RESET_SEM; + + if (reset_mask & RADEON_RESET_IH) + srbm_soft_reset |= SOFT_RESET_IH; + + if (reset_mask & RADEON_RESET_GRBM) + srbm_soft_reset |= SOFT_RESET_GRBM; + + if (reset_mask & RADEON_RESET_VMC) + srbm_soft_reset |= SOFT_RESET_VMC; + + if (reset_mask & RADEON_RESET_MC) + srbm_soft_reset |= SOFT_RESET_MC; + + if (grbm_soft_reset) { + tmp = RREG32(GRBM_SOFT_RESET); + tmp |= grbm_soft_reset; + dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(GRBM_SOFT_RESET, tmp); + tmp = RREG32(GRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~grbm_soft_reset; + WREG32(GRBM_SOFT_RESET, tmp); + tmp = RREG32(GRBM_SOFT_RESET); + } + + if (srbm_soft_reset) { + tmp = RREG32(SRBM_SOFT_RESET); + tmp |= srbm_soft_reset; + dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~srbm_soft_reset; + WREG32(SRBM_SOFT_RESET, tmp); + tmp = RREG32(SRBM_SOFT_RESET); + } + + /* Wait a little for things to settle down */ + udelay(50); + + evergreen_mc_resume(rdev, &save); + udelay(50); + + evergreen_print_gpu_status_regs(rdev); +} + +static void si_set_clk_bypass_mode(struct radeon_device *rdev) +{ + u32 tmp, i; + + tmp = RREG32(CG_SPLL_FUNC_CNTL); + tmp |= SPLL_BYPASS_EN; + WREG32(CG_SPLL_FUNC_CNTL, tmp); + + tmp = RREG32(CG_SPLL_FUNC_CNTL_2); + tmp |= SPLL_CTLREQ_CHG; + WREG32(CG_SPLL_FUNC_CNTL_2, tmp); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS) + break; + udelay(1); + } + + tmp = RREG32(CG_SPLL_FUNC_CNTL_2); + tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE); + WREG32(CG_SPLL_FUNC_CNTL_2, tmp); + + tmp = RREG32(MPLL_CNTL_MODE); + tmp &= ~MPLL_MCLK_SEL; + WREG32(MPLL_CNTL_MODE, tmp); } -static void si_gpu_soft_reset_dma(struct radeon_device *rdev) +static void si_spll_powerdown(struct radeon_device *rdev) { u32 tmp; - if (RREG32(DMA_STATUS_REG) & DMA_IDLE) - return; + tmp = RREG32(SPLL_CNTL_MODE); + tmp |= SPLL_SW_DIR_CONTROL; + WREG32(SPLL_CNTL_MODE, tmp); - dev_info(rdev->dev, " DMA_STATUS_REG = 0x%08X\n", - RREG32(DMA_STATUS_REG)); + tmp = RREG32(CG_SPLL_FUNC_CNTL); + tmp |= SPLL_RESET; + WREG32(CG_SPLL_FUNC_CNTL, tmp); + tmp = RREG32(CG_SPLL_FUNC_CNTL); + tmp |= SPLL_SLEEP; + WREG32(CG_SPLL_FUNC_CNTL, tmp); + + tmp = RREG32(SPLL_CNTL_MODE); + tmp &= ~SPLL_SW_DIR_CONTROL; + WREG32(SPLL_CNTL_MODE, tmp); +} + +static void si_gpu_pci_config_reset(struct radeon_device *rdev) +{ + struct evergreen_mc_save save; + u32 tmp, i; + + dev_info(rdev->dev, "GPU pci config reset\n"); + + /* disable dpm? */ + + /* disable cg/pg */ + si_fini_pg(rdev); + si_fini_cg(rdev); + + /* Disable CP parsing/prefetching */ + WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); /* dma0 */ tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); tmp &= ~DMA_RB_ENABLE; WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); - /* dma1 */ tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); tmp &= ~DMA_RB_ENABLE; WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); + /* XXX other engines? */ + + /* halt the rlc, disable cp internal ints */ + si_rlc_stop(rdev); - /* Reset dma */ - WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1); - RREG32(SRBM_SOFT_RESET); udelay(50); - WREG32(SRBM_SOFT_RESET, 0); - dev_info(rdev->dev, " DMA_STATUS_REG = 0x%08X\n", - RREG32(DMA_STATUS_REG)); + /* disable mem access */ + evergreen_mc_stop(rdev, &save); + if (evergreen_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); + } + + /* set mclk/sclk to bypass */ + si_set_clk_bypass_mode(rdev); + /* powerdown spll */ + si_spll_powerdown(rdev); + /* disable BM */ + pci_clear_master(rdev->pdev); + /* reset */ + radeon_pci_config_reset(rdev); + /* wait for asic to come out of reset */ + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) + break; + udelay(1); + } } -static int si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) +int si_asic_reset(struct radeon_device *rdev) { - struct evergreen_mc_save save; + u32 reset_mask; - if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) - reset_mask &= ~(RADEON_RESET_GFX | RADEON_RESET_COMPUTE); + reset_mask = si_gpu_check_soft_reset(rdev); - if (RREG32(DMA_STATUS_REG) & DMA_IDLE) - reset_mask &= ~RADEON_RESET_DMA; + if (reset_mask) + r600_set_bios_scratch_engine_hung(rdev, true); - if (reset_mask == 0) - return 0; + /* try soft reset */ + si_gpu_soft_reset(rdev, reset_mask); - dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); + reset_mask = si_gpu_check_soft_reset(rdev); - dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", - RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); - dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", - RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); + /* try pci config reset */ + if (reset_mask && radeon_hard_reset) + si_gpu_pci_config_reset(rdev); - evergreen_mc_stop(rdev, &save); - if (radeon_mc_wait_for_idle(rdev)) { - dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); - } - - if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) - si_gpu_soft_reset_gfx(rdev); - - if (reset_mask & RADEON_RESET_DMA) - si_gpu_soft_reset_dma(rdev); + reset_mask = si_gpu_check_soft_reset(rdev); - /* Wait a little for things to settle down */ - udelay(50); + if (!reset_mask) + r600_set_bios_scratch_engine_hung(rdev, false); - evergreen_mc_resume(rdev, &save); return 0; } -int si_asic_reset(struct radeon_device *rdev) +/** + * si_gfx_is_lockup - Check if the GFX engine is locked up + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Check if the GFX engine is locked up. + * Returns true if the engine appears to be locked up, false if not. + */ +bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) { - return si_gpu_soft_reset(rdev, (RADEON_RESET_GFX | - RADEON_RESET_COMPUTE | - RADEON_RESET_DMA)); + u32 reset_mask = si_gpu_check_soft_reset(rdev); + + if (!(reset_mask & (RADEON_RESET_GFX | + RADEON_RESET_COMPUTE | + RADEON_RESET_CP))) { + radeon_ring_lockup_update(rdev, ring); + return false; + } + return radeon_ring_test_lockup(rdev, ring); } /* MC */ @@ -2294,8 +4147,9 @@ static void si_mc_program(struct radeon_device *rdev) if (radeon_mc_wait_for_idle(rdev)) { dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); } - /* Lockout access through VGA aperture*/ - WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); + if (!ASIC_IS_NODCE(rdev)) + /* Lockout access through VGA aperture*/ + WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); /* Update configuration */ WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12); @@ -2317,53 +4171,15 @@ static void si_mc_program(struct radeon_device *rdev) dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); } evergreen_mc_resume(rdev, &save); - /* we need to own VRAM, so turn off the VGA renderer here - * to stop it overwriting our objects */ - rv515_vga_render_disable(rdev); -} - -/* SI MC address space is 40 bits */ -static void si_vram_location(struct radeon_device *rdev, - struct radeon_mc *mc, u64 base) -{ - mc->vram_start = base; - if (mc->mc_vram_size > (0xFFFFFFFFFFULL - base + 1)) { - dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n"); - mc->real_vram_size = mc->aper_size; - mc->mc_vram_size = mc->aper_size; + if (!ASIC_IS_NODCE(rdev)) { + /* we need to own VRAM, so turn off the VGA renderer here + * to stop it overwriting our objects */ + rv515_vga_render_disable(rdev); } - mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; - dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", - mc->mc_vram_size >> 20, mc->vram_start, - mc->vram_end, mc->real_vram_size >> 20); } -static void si_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc) -{ - u64 size_af, size_bf; - - size_af = ((0xFFFFFFFFFFULL - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align; - size_bf = mc->vram_start & ~mc->gtt_base_align; - if (size_bf > size_af) { - if (mc->gtt_size > size_bf) { - dev_warn(rdev->dev, "limiting GTT\n"); - mc->gtt_size = size_bf; - } - mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size; - } else { - if (mc->gtt_size > size_af) { - dev_warn(rdev->dev, "limiting GTT\n"); - mc->gtt_size = size_af; - } - mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align; - } - mc->gtt_end = mc->gtt_start + mc->gtt_size - 1; - dev_info(rdev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n", - mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end); -} - -static void si_vram_gtt_location(struct radeon_device *rdev, - struct radeon_mc *mc) +void si_vram_gtt_location(struct radeon_device *rdev, + struct radeon_mc *mc) { if (mc->mc_vram_size > 0xFFC0000000ULL) { /* leave room for at least 1024M GTT */ @@ -2371,9 +4187,9 @@ static void si_vram_gtt_location(struct radeon_device *rdev, mc->real_vram_size = 0xFFC0000000ULL; mc->mc_vram_size = 0xFFC0000000ULL; } - si_vram_location(rdev, &rdev->mc, 0); + radeon_vram_location(rdev, &rdev->mc, 0); rdev->mc.gtt_base_align = 0; - si_gtt_location(rdev, mc); + radeon_gtt_location(rdev, mc); } static int si_mc_init(struct radeon_device *rdev) @@ -2466,23 +4282,25 @@ static int si_pcie_gart_enable(struct radeon_device *rdev) r = radeon_gart_table_vram_pin(rdev); if (r) return r; - radeon_gart_restore(rdev); /* Setup TLB control */ WREG32(MC_VM_MX_L1_TLB_CNTL, (0xA << 7) | ENABLE_L1_TLB | + ENABLE_L1_FRAGMENT_PROCESSING | SYSTEM_ACCESS_MODE_NOT_IN_SYS | ENABLE_ADVANCED_DRIVER_MODEL | SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); /* Setup L2 cache */ WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | + ENABLE_L2_FRAGMENT_PROCESSING | ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | EFFECTIVE_L2_QUEUE_SIZE(7) | CONTEXT1_IDENTITY_ACCESS_MODE(1)); WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | - L2_CACHE_BIGK_FRAGMENT_SIZE(0)); + BANK_SELECT(4) | + L2_CACHE_BIGK_FRAGMENT_SIZE(4)); /* setup context0 */ WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); @@ -2500,7 +4318,7 @@ static int si_pcie_gart_enable(struct radeon_device *rdev) /* empty context1-15 */ /* set vm size, must be a multiple of 4 */ WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); - WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); + WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1); /* Assign the pt base to something valid for now; the pts used for * the VMs are determined by the application and setup and assigned * on the fly in the vm part of radeon_gart.c @@ -2508,10 +4326,10 @@ static int si_pcie_gart_enable(struct radeon_device *rdev) for (i = 1; i < 16; i++) { if (i < 8) WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), - rdev->gart.table_addr >> 12); + rdev->vm_manager.saved_table_addr[i]); else WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), - rdev->gart.table_addr >> 12); + rdev->vm_manager.saved_table_addr[i]); } /* enable context1-15 */ @@ -2519,6 +4337,7 @@ static int si_pcie_gart_enable(struct radeon_device *rdev) (u32)(rdev->dummy_page.addr >> 12)); WREG32(VM_CONTEXT1_CNTL2, 4); WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | + PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | @@ -2542,6 +4361,17 @@ static int si_pcie_gart_enable(struct radeon_device *rdev) static void si_pcie_gart_disable(struct radeon_device *rdev) { + unsigned i; + + for (i = 1; i < 16; ++i) { + uint32_t reg; + if (i < 8) + reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2); + else + reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2); + rdev->vm_manager.saved_table_addr[i] = RREG32(reg); + } + /* Disable all tables */ WREG32(VM_CONTEXT0_CNTL, 0); WREG32(VM_CONTEXT1_CNTL, 0); @@ -2890,24 +4720,24 @@ static int si_vm_packet3_compute_check(struct radeon_device *rdev, int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) { int ret = 0; - u32 idx = 0; + u32 idx = 0, i; struct radeon_cs_packet pkt; do { pkt.idx = idx; - pkt.type = CP_PACKET_GET_TYPE(ib->ptr[idx]); - pkt.count = CP_PACKET_GET_COUNT(ib->ptr[idx]); + pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]); + pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]); pkt.one_reg_wr = 0; switch (pkt.type) { - case PACKET_TYPE0: + case RADEON_PACKET_TYPE0: dev_err(rdev->dev, "Packet0 not allowed!\n"); ret = -EINVAL; break; - case PACKET_TYPE2: + case RADEON_PACKET_TYPE2: idx += 1; break; - case PACKET_TYPE3: - pkt.opcode = CP_PACKET3_GET_OPCODE(ib->ptr[idx]); + case RADEON_PACKET_TYPE3: + pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]); if (ib->is_const_ib) ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt); else { @@ -2932,8 +4762,15 @@ int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) ret = -EINVAL; break; } - if (ret) + if (ret) { + for (i = 0; i < ib->length_dw; i++) { + if (i == idx) + printk("\t0x%08x <---\n", ib->ptr[i]); + else + printk("\t0x%08x\n", ib->ptr[i]); + } break; + } } while (idx < ib->length_dw); return ret; @@ -2957,304 +4794,1108 @@ void si_vm_fini(struct radeon_device *rdev) } /** - * si_vm_set_page - update the page tables using the CP + * si_vm_decode_fault - print human readable fault info * * @rdev: radeon_device pointer - * @pe: addr of the page entry - * @addr: dst addr to write into pe - * @count: number of page entries to update - * @incr: increase next addr by incr bytes - * @flags: access flags + * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value + * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value * - * Update the page tables using the CP (cayman-si). + * Print human readable fault information (SI). */ -void si_vm_set_page(struct radeon_device *rdev, uint64_t pe, - uint64_t addr, unsigned count, - uint32_t incr, uint32_t flags) -{ - struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index]; - uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); - uint64_t value; - unsigned ndw; - - if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) { - while (count) { - ndw = 2 + count * 2; - if (ndw > 0x3FFE) - ndw = 0x3FFE; - - radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, ndw)); - radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | - WRITE_DATA_DST_SEL(1))); - radeon_ring_write(ring, pe); - radeon_ring_write(ring, upper_32_bits(pe)); - for (; ndw > 2; ndw -= 2, --count, pe += 8) { - if (flags & RADEON_VM_PAGE_SYSTEM) { - value = radeon_vm_map_gart(rdev, addr); - value &= 0xFFFFFFFFFFFFF000ULL; - } else if (flags & RADEON_VM_PAGE_VALID) { - value = addr; - } else { - value = 0; - } - addr += incr; - value |= r600_flags; - radeon_ring_write(ring, value); - radeon_ring_write(ring, upper_32_bits(value)); - } +static void si_vm_decode_fault(struct radeon_device *rdev, + u32 status, u32 addr) +{ + u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; + u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; + u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; + char *block; + + if (rdev->family == CHIP_TAHITI) { + switch (mc_id) { + case 160: + case 144: + case 96: + case 80: + case 224: + case 208: + case 32: + case 16: + block = "CB"; + break; + case 161: + case 145: + case 97: + case 81: + case 225: + case 209: + case 33: + case 17: + block = "CB_FMASK"; + break; + case 162: + case 146: + case 98: + case 82: + case 226: + case 210: + case 34: + case 18: + block = "CB_CMASK"; + break; + case 163: + case 147: + case 99: + case 83: + case 227: + case 211: + case 35: + case 19: + block = "CB_IMMED"; + break; + case 164: + case 148: + case 100: + case 84: + case 228: + case 212: + case 36: + case 20: + block = "DB"; + break; + case 165: + case 149: + case 101: + case 85: + case 229: + case 213: + case 37: + case 21: + block = "DB_HTILE"; + break; + case 167: + case 151: + case 103: + case 87: + case 231: + case 215: + case 39: + case 23: + block = "DB_STEN"; + break; + case 72: + case 68: + case 64: + case 8: + case 4: + case 0: + case 136: + case 132: + case 128: + case 200: + case 196: + case 192: + block = "TC"; + break; + case 112: + case 48: + block = "CP"; + break; + case 49: + case 177: + case 50: + case 178: + block = "SH"; + break; + case 53: + case 190: + block = "VGT"; + break; + case 117: + block = "IH"; + break; + case 51: + case 115: + block = "RLC"; + break; + case 119: + case 183: + block = "DMA0"; + break; + case 61: + block = "DMA1"; + break; + case 248: + case 120: + block = "HDP"; + break; + default: + block = "unknown"; + break; } } else { - /* DMA */ - if (flags & RADEON_VM_PAGE_SYSTEM) { - while (count) { - ndw = count * 2; - if (ndw > 0xFFFFE) - ndw = 0xFFFFE; - - /* for non-physically contiguous pages (system) */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw)); - radeon_ring_write(ring, pe); - radeon_ring_write(ring, upper_32_bits(pe) & 0xff); - for (; ndw > 0; ndw -= 2, --count, pe += 8) { - if (flags & RADEON_VM_PAGE_SYSTEM) { - value = radeon_vm_map_gart(rdev, addr); - value &= 0xFFFFFFFFFFFFF000ULL; - } else if (flags & RADEON_VM_PAGE_VALID) { - value = addr; - } else { - value = 0; - } - addr += incr; - value |= r600_flags; - radeon_ring_write(ring, value); - radeon_ring_write(ring, upper_32_bits(value)); + switch (mc_id) { + case 32: + case 16: + case 96: + case 80: + case 160: + case 144: + case 224: + case 208: + block = "CB"; + break; + case 33: + case 17: + case 97: + case 81: + case 161: + case 145: + case 225: + case 209: + block = "CB_FMASK"; + break; + case 34: + case 18: + case 98: + case 82: + case 162: + case 146: + case 226: + case 210: + block = "CB_CMASK"; + break; + case 35: + case 19: + case 99: + case 83: + case 163: + case 147: + case 227: + case 211: + block = "CB_IMMED"; + break; + case 36: + case 20: + case 100: + case 84: + case 164: + case 148: + case 228: + case 212: + block = "DB"; + break; + case 37: + case 21: + case 101: + case 85: + case 165: + case 149: + case 229: + case 213: + block = "DB_HTILE"; + break; + case 39: + case 23: + case 103: + case 87: + case 167: + case 151: + case 231: + case 215: + block = "DB_STEN"; + break; + case 72: + case 68: + case 8: + case 4: + case 136: + case 132: + case 200: + case 196: + block = "TC"; + break; + case 112: + case 48: + block = "CP"; + break; + case 49: + case 177: + case 50: + case 178: + block = "SH"; + break; + case 53: + block = "VGT"; + break; + case 117: + block = "IH"; + break; + case 51: + case 115: + block = "RLC"; + break; + case 119: + case 183: + block = "DMA0"; + break; + case 61: + block = "DMA1"; + break; + case 248: + case 120: + block = "HDP"; + break; + default: + block = "unknown"; + break; + } + } + + printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n", + protections, vmid, addr, + (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", + block, mc_id); +} + +void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) +{ + /* write new base address */ + radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | + WRITE_DATA_DST_SEL(0))); + + if (vm_id < 8) { + radeon_ring_write(ring, + (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); + } else { + radeon_ring_write(ring, + (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); + } + radeon_ring_write(ring, 0); + radeon_ring_write(ring, pd_addr >> 12); + + /* flush hdp cache */ + radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | + WRITE_DATA_DST_SEL(0))); + radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, 0x1); + + /* bits 0-15 are the VM contexts0-15 */ + radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | + WRITE_DATA_DST_SEL(0))); + radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, 1 << vm_id); + + /* wait for the invalidate to complete */ + radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); + radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) | /* always */ + WAIT_REG_MEM_ENGINE(0))); /* me */ + radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, 0); /* ref */ + radeon_ring_write(ring, 0); /* mask */ + radeon_ring_write(ring, 0x20); /* poll interval */ + + /* sync PFP to ME, otherwise we might get invalid PFP reads */ + radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); + radeon_ring_write(ring, 0x0); +} + +/* + * Power and clock gating + */ +static void si_wait_for_rlc_serdes(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0) + break; + udelay(1); + } + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0) + break; + udelay(1); + } +} + +static void si_enable_gui_idle_interrupt(struct radeon_device *rdev, + bool enable) +{ + u32 tmp = RREG32(CP_INT_CNTL_RING0); + u32 mask; + int i; + + if (enable) + tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + else + tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + WREG32(CP_INT_CNTL_RING0, tmp); + + if (!enable) { + /* read a gfx register */ + tmp = RREG32(DB_DEPTH_INFO); + + mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS; + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS)) + break; + udelay(1); + } + } +} + +static void si_set_uvd_dcm(struct radeon_device *rdev, + bool sw_mode) +{ + u32 tmp, tmp2; + + tmp = RREG32(UVD_CGC_CTRL); + tmp &= ~(CLK_OD_MASK | CG_DT_MASK); + tmp |= DCM | CG_DT(1) | CLK_OD(4); + + if (sw_mode) { + tmp &= ~0x7ffff800; + tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7); + } else { + tmp |= 0x7ffff800; + tmp2 = 0; + } + + WREG32(UVD_CGC_CTRL, tmp); + WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2); +} + +void si_init_uvd_internal_cg(struct radeon_device *rdev) +{ + bool hw_mode = true; + + if (hw_mode) { + si_set_uvd_dcm(rdev, false); + } else { + u32 tmp = RREG32(UVD_CGC_CTRL); + tmp &= ~DCM; + WREG32(UVD_CGC_CTRL, tmp); + } +} + +static u32 si_halt_rlc(struct radeon_device *rdev) +{ + u32 data, orig; + + orig = data = RREG32(RLC_CNTL); + + if (data & RLC_ENABLE) { + data &= ~RLC_ENABLE; + WREG32(RLC_CNTL, data); + + si_wait_for_rlc_serdes(rdev); + } + + return orig; +} + +static void si_update_rlc(struct radeon_device *rdev, u32 rlc) +{ + u32 tmp; + + tmp = RREG32(RLC_CNTL); + if (tmp != rlc) + WREG32(RLC_CNTL, rlc); +} + +static void si_enable_dma_pg(struct radeon_device *rdev, bool enable) +{ + u32 data, orig; + + orig = data = RREG32(DMA_PG); + if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA)) + data |= PG_CNTL_ENABLE; + else + data &= ~PG_CNTL_ENABLE; + if (orig != data) + WREG32(DMA_PG, data); +} + +static void si_init_dma_pg(struct radeon_device *rdev) +{ + u32 tmp; + + WREG32(DMA_PGFSM_WRITE, 0x00002000); + WREG32(DMA_PGFSM_CONFIG, 0x100010ff); + + for (tmp = 0; tmp < 5; tmp++) + WREG32(DMA_PGFSM_WRITE, 0); +} + +static void si_enable_gfx_cgpg(struct radeon_device *rdev, + bool enable) +{ + u32 tmp; + + if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { + tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10); + WREG32(RLC_TTOP_D, tmp); + + tmp = RREG32(RLC_PG_CNTL); + tmp |= GFX_PG_ENABLE; + WREG32(RLC_PG_CNTL, tmp); + + tmp = RREG32(RLC_AUTO_PG_CTRL); + tmp |= AUTO_PG_EN; + WREG32(RLC_AUTO_PG_CTRL, tmp); + } else { + tmp = RREG32(RLC_AUTO_PG_CTRL); + tmp &= ~AUTO_PG_EN; + WREG32(RLC_AUTO_PG_CTRL, tmp); + + tmp = RREG32(DB_RENDER_CONTROL); + } +} + +static void si_init_gfx_cgpg(struct radeon_device *rdev) +{ + u32 tmp; + + WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); + + tmp = RREG32(RLC_PG_CNTL); + tmp |= GFX_PG_SRC; + WREG32(RLC_PG_CNTL, tmp); + + WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); + + tmp = RREG32(RLC_AUTO_PG_CTRL); + + tmp &= ~GRBM_REG_SGIT_MASK; + tmp |= GRBM_REG_SGIT(0x700); + tmp &= ~PG_AFTER_GRBM_REG_ST_MASK; + WREG32(RLC_AUTO_PG_CTRL, tmp); +} + +static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) +{ + u32 mask = 0, tmp, tmp1; + int i; + + si_select_se_sh(rdev, se, sh); + tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); + tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); + si_select_se_sh(rdev, 0xffffffff, 0xffffffff); + + tmp &= 0xffff0000; + + tmp |= tmp1; + tmp >>= 16; + + for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) { + mask <<= 1; + mask |= 1; + } + + return (~tmp) & mask; +} + +static void si_init_ao_cu_mask(struct radeon_device *rdev) +{ + u32 i, j, k, active_cu_number = 0; + u32 mask, counter, cu_bitmap; + u32 tmp = 0; + + for (i = 0; i < rdev->config.si.max_shader_engines; i++) { + for (j = 0; j < rdev->config.si.max_sh_per_se; j++) { + mask = 1; + cu_bitmap = 0; + counter = 0; + for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) { + if (si_get_cu_active_bitmap(rdev, i, j) & mask) { + if (counter < 2) + cu_bitmap |= mask; + counter++; } + mask <<= 1; } - } else { - while (count) { - ndw = count * 2; - if (ndw > 0xFFFFE) - ndw = 0xFFFFE; - if (flags & RADEON_VM_PAGE_VALID) - value = addr; - else - value = 0; - /* for physically contiguous pages (vram) */ - radeon_ring_write(ring, DMA_PTE_PDE_PACKET(ndw)); - radeon_ring_write(ring, pe); /* dst addr */ - radeon_ring_write(ring, upper_32_bits(pe) & 0xff); - radeon_ring_write(ring, r600_flags); /* mask */ - radeon_ring_write(ring, 0); - radeon_ring_write(ring, value); /* value */ - radeon_ring_write(ring, upper_32_bits(value)); - radeon_ring_write(ring, incr); /* increment size */ - radeon_ring_write(ring, 0); - pe += ndw * 4; - addr += (ndw / 2) * incr; - count -= ndw / 2; - } + active_cu_number += counter; + tmp |= (cu_bitmap << (i * 16 + j * 8)); + } + } + + WREG32(RLC_PG_AO_CU_MASK, tmp); + + tmp = RREG32(RLC_MAX_PG_CU); + tmp &= ~MAX_PU_CU_MASK; + tmp |= MAX_PU_CU(active_cu_number); + WREG32(RLC_MAX_PG_CU, tmp); +} + +static void si_enable_cgcg(struct radeon_device *rdev, + bool enable) +{ + u32 data, orig, tmp; + + orig = data = RREG32(RLC_CGCG_CGLS_CTRL); + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { + si_enable_gui_idle_interrupt(rdev, true); + + WREG32(RLC_GCPM_GENERAL_3, 0x00000080); + + tmp = si_halt_rlc(rdev); + + WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); + WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); + WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff); + + si_wait_for_rlc_serdes(rdev); + + si_update_rlc(rdev, tmp); + + WREG32(RLC_SERDES_WR_CTRL, 0x007000ff); + + data |= CGCG_EN | CGLS_EN; + } else { + si_enable_gui_idle_interrupt(rdev, false); + + RREG32(CB_CGTT_SCLK_CTRL); + RREG32(CB_CGTT_SCLK_CTRL); + RREG32(CB_CGTT_SCLK_CTRL); + RREG32(CB_CGTT_SCLK_CTRL); + + data &= ~(CGCG_EN | CGLS_EN); + } + + if (orig != data) + WREG32(RLC_CGCG_CGLS_CTRL, data); +} + +static void si_enable_mgcg(struct radeon_device *rdev, + bool enable) +{ + u32 data, orig, tmp = 0; + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { + orig = data = RREG32(CGTS_SM_CTRL_REG); + data = 0x96940200; + if (orig != data) + WREG32(CGTS_SM_CTRL_REG, data); + + if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { + orig = data = RREG32(CP_MEM_SLP_CNTL); + data |= CP_MEM_LS_EN; + if (orig != data) + WREG32(CP_MEM_SLP_CNTL, data); + } + + orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); + data &= 0xffffffc0; + if (orig != data) + WREG32(RLC_CGTT_MGCG_OVERRIDE, data); + + tmp = si_halt_rlc(rdev); + + WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); + WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); + WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff); + + si_update_rlc(rdev, tmp); + } else { + orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); + data |= 0x00000003; + if (orig != data) + WREG32(RLC_CGTT_MGCG_OVERRIDE, data); + + data = RREG32(CP_MEM_SLP_CNTL); + if (data & CP_MEM_LS_EN) { + data &= ~CP_MEM_LS_EN; + WREG32(CP_MEM_SLP_CNTL, data); + } + orig = data = RREG32(CGTS_SM_CTRL_REG); + data |= LS_OVERRIDE | OVERRIDE; + if (orig != data) + WREG32(CGTS_SM_CTRL_REG, data); + + tmp = si_halt_rlc(rdev); + + WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); + WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); + WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff); + + si_update_rlc(rdev, tmp); + } +} + +static void si_enable_uvd_mgcg(struct radeon_device *rdev, + bool enable) +{ + u32 orig, data, tmp; + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { + tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); + tmp |= 0x3fff; + WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); + + orig = data = RREG32(UVD_CGC_CTRL); + data |= DCM; + if (orig != data) + WREG32(UVD_CGC_CTRL, data); + + WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0); + WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0); + } else { + tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); + tmp &= ~0x3fff; + WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); + + orig = data = RREG32(UVD_CGC_CTRL); + data &= ~DCM; + if (orig != data) + WREG32(UVD_CGC_CTRL, data); + + WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff); + WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff); + } +} + +static const u32 mc_cg_registers[] = +{ + MC_HUB_MISC_HUB_CG, + MC_HUB_MISC_SIP_CG, + MC_HUB_MISC_VM_CG, + MC_XPB_CLK_GAT, + ATC_MISC_CG, + MC_CITF_MISC_WR_CG, + MC_CITF_MISC_RD_CG, + MC_CITF_MISC_VM_CG, + VM_L2_CG, +}; + +static void si_enable_mc_ls(struct radeon_device *rdev, + bool enable) +{ + int i; + u32 orig, data; + + for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { + orig = data = RREG32(mc_cg_registers[i]); + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) + data |= MC_LS_ENABLE; + else + data &= ~MC_LS_ENABLE; + if (data != orig) + WREG32(mc_cg_registers[i], data); + } +} + +static void si_enable_mc_mgcg(struct radeon_device *rdev, + bool enable) +{ + int i; + u32 orig, data; + + for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { + orig = data = RREG32(mc_cg_registers[i]); + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) + data |= MC_CG_ENABLE; + else + data &= ~MC_CG_ENABLE; + if (data != orig) + WREG32(mc_cg_registers[i], data); + } +} + +static void si_enable_dma_mgcg(struct radeon_device *rdev, + bool enable) +{ + u32 orig, data, offset; + int i; + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { + for (i = 0; i < 2; i++) { + if (i == 0) + offset = DMA0_REGISTER_OFFSET; + else + offset = DMA1_REGISTER_OFFSET; + orig = data = RREG32(DMA_POWER_CNTL + offset); + data &= ~MEM_POWER_OVERRIDE; + if (data != orig) + WREG32(DMA_POWER_CNTL + offset, data); + WREG32(DMA_CLK_CTRL + offset, 0x00000100); + } + } else { + for (i = 0; i < 2; i++) { + if (i == 0) + offset = DMA0_REGISTER_OFFSET; + else + offset = DMA1_REGISTER_OFFSET; + orig = data = RREG32(DMA_POWER_CNTL + offset); + data |= MEM_POWER_OVERRIDE; + if (data != orig) + WREG32(DMA_POWER_CNTL + offset, data); + + orig = data = RREG32(DMA_CLK_CTRL + offset); + data = 0xff000000; + if (data != orig) + WREG32(DMA_CLK_CTRL + offset, data); } } } -void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +static void si_enable_bif_mgls(struct radeon_device *rdev, + bool enable) { - struct radeon_ring *ring = &rdev->ring[ridx]; + u32 orig, data; - if (vm == NULL) - return; + orig = data = RREG32_PCIE(PCIE_CNTL2); - /* write new base address */ - radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); - radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | - WRITE_DATA_DST_SEL(0))); + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) + data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | + REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; + else + data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | + REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); - if (vm->id < 8) { - radeon_ring_write(ring, - (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); - } else { - radeon_ring_write(ring, - (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); - } - radeon_ring_write(ring, 0); - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + if (orig != data) + WREG32_PCIE(PCIE_CNTL2, data); +} - /* flush hdp cache */ - radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); - radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | - WRITE_DATA_DST_SEL(0))); - radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 0x1); +static void si_enable_hdp_mgcg(struct radeon_device *rdev, + bool enable) +{ + u32 orig, data; - /* bits 0-15 are the VM contexts0-15 */ - radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); - radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | - WRITE_DATA_DST_SEL(0))); - radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); - radeon_ring_write(ring, 0); - radeon_ring_write(ring, 1 << vm->id); + orig = data = RREG32(HDP_HOST_PATH_CNTL); - /* sync PFP to ME, otherwise we might get invalid PFP reads */ - radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); - radeon_ring_write(ring, 0x0); + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) + data &= ~CLOCK_GATING_DIS; + else + data |= CLOCK_GATING_DIS; + + if (orig != data) + WREG32(HDP_HOST_PATH_CNTL, data); } -void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +static void si_enable_hdp_ls(struct radeon_device *rdev, + bool enable) { - struct radeon_ring *ring = &rdev->ring[ridx]; + u32 orig, data; - if (vm == NULL) - return; + orig = data = RREG32(HDP_MEM_POWER_LS); - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); - if (vm->id < 8) { - radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2)); - } else { - radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2)); - } - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) + data |= HDP_LS_ENABLE; + else + data &= ~HDP_LS_ENABLE; - /* flush hdp cache */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); - radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); - radeon_ring_write(ring, 1); - - /* bits 0-7 are the VM contexts0-7 */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); - radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2)); - radeon_ring_write(ring, 1 << vm->id); + if (orig != data) + WREG32(HDP_MEM_POWER_LS, data); } -/* - * RLC - */ -void si_rlc_fini(struct radeon_device *rdev) +static void si_update_cg(struct radeon_device *rdev, + u32 block, bool enable) { - int r; + if (block & RADEON_CG_BLOCK_GFX) { + si_enable_gui_idle_interrupt(rdev, false); + /* order matters! */ + if (enable) { + si_enable_mgcg(rdev, true); + si_enable_cgcg(rdev, true); + } else { + si_enable_cgcg(rdev, false); + si_enable_mgcg(rdev, false); + } + si_enable_gui_idle_interrupt(rdev, true); + } + + if (block & RADEON_CG_BLOCK_MC) { + si_enable_mc_mgcg(rdev, enable); + si_enable_mc_ls(rdev, enable); + } + + if (block & RADEON_CG_BLOCK_SDMA) { + si_enable_dma_mgcg(rdev, enable); + } - /* save restore block */ - if (rdev->rlc.save_restore_obj) { - r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); - if (unlikely(r != 0)) - dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); - radeon_bo_unpin(rdev->rlc.save_restore_obj); - radeon_bo_unreserve(rdev->rlc.save_restore_obj); + if (block & RADEON_CG_BLOCK_BIF) { + si_enable_bif_mgls(rdev, enable); + } - radeon_bo_unref(&rdev->rlc.save_restore_obj); - rdev->rlc.save_restore_obj = NULL; + if (block & RADEON_CG_BLOCK_UVD) { + if (rdev->has_uvd) { + si_enable_uvd_mgcg(rdev, enable); + } } - /* clear state block */ - if (rdev->rlc.clear_state_obj) { - r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); - if (unlikely(r != 0)) - dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); - radeon_bo_unpin(rdev->rlc.clear_state_obj); - radeon_bo_unreserve(rdev->rlc.clear_state_obj); + if (block & RADEON_CG_BLOCK_HDP) { + si_enable_hdp_mgcg(rdev, enable); + si_enable_hdp_ls(rdev, enable); + } +} - radeon_bo_unref(&rdev->rlc.clear_state_obj); - rdev->rlc.clear_state_obj = NULL; +static void si_init_cg(struct radeon_device *rdev) +{ + si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | + RADEON_CG_BLOCK_MC | + RADEON_CG_BLOCK_SDMA | + RADEON_CG_BLOCK_BIF | + RADEON_CG_BLOCK_HDP), true); + if (rdev->has_uvd) { + si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true); + si_init_uvd_internal_cg(rdev); } } -int si_rlc_init(struct radeon_device *rdev) +static void si_fini_cg(struct radeon_device *rdev) { - int r; + if (rdev->has_uvd) { + si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false); + } + si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | + RADEON_CG_BLOCK_MC | + RADEON_CG_BLOCK_SDMA | + RADEON_CG_BLOCK_BIF | + RADEON_CG_BLOCK_HDP), false); +} - /* save restore block */ - if (rdev->rlc.save_restore_obj == NULL) { - r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_VRAM, NULL, - &rdev->rlc.save_restore_obj); - if (r) { - dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); - return r; +u32 si_get_csb_size(struct radeon_device *rdev) +{ + u32 count = 0; + const struct cs_section_def *sect = NULL; + const struct cs_extent_def *ext = NULL; + + if (rdev->rlc.cs_data == NULL) + return 0; + + /* begin clear state */ + count += 2; + /* context control state */ + count += 3; + + for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { + for (ext = sect->section; ext->extent != NULL; ++ext) { + if (sect->id == SECT_CONTEXT) + count += 2 + ext->reg_count; + else + return 0; } } + /* pa_sc_raster_config */ + count += 3; + /* end clear state */ + count += 2; + /* clear state */ + count += 2; - r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); - if (unlikely(r != 0)) { - si_rlc_fini(rdev); - return r; + return count; +} + +void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) +{ + u32 count = 0, i; + const struct cs_section_def *sect = NULL; + const struct cs_extent_def *ext = NULL; + + if (rdev->rlc.cs_data == NULL) + return; + if (buffer == NULL) + return; + + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); + buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); + + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); + buffer[count++] = cpu_to_le32(0x80000000); + buffer[count++] = cpu_to_le32(0x80000000); + + for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { + for (ext = sect->section; ext->extent != NULL; ++ext) { + if (sect->id == SECT_CONTEXT) { + buffer[count++] = + cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); + buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); + for (i = 0; i < ext->reg_count; i++) + buffer[count++] = cpu_to_le32(ext->extent[i]); + } else { + return; + } + } } - r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, - &rdev->rlc.save_restore_gpu_addr); - radeon_bo_unreserve(rdev->rlc.save_restore_obj); - if (r) { - dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); - si_rlc_fini(rdev); - return r; + + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1)); + buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); + switch (rdev->family) { + case CHIP_TAHITI: + case CHIP_PITCAIRN: + buffer[count++] = cpu_to_le32(0x2a00126a); + break; + case CHIP_VERDE: + buffer[count++] = cpu_to_le32(0x0000124a); + break; + case CHIP_OLAND: + buffer[count++] = cpu_to_le32(0x00000082); + break; + case CHIP_HAINAN: + buffer[count++] = cpu_to_le32(0x00000000); + break; + default: + buffer[count++] = cpu_to_le32(0x00000000); + break; } - /* clear state block */ - if (rdev->rlc.clear_state_obj == NULL) { - r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_VRAM, NULL, - &rdev->rlc.clear_state_obj); - if (r) { - dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); - si_rlc_fini(rdev); - return r; + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); + buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); + + buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); + buffer[count++] = cpu_to_le32(0); +} + +static void si_init_pg(struct radeon_device *rdev) +{ + if (rdev->pg_flags) { + if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) { + si_init_dma_pg(rdev); } + si_init_ao_cu_mask(rdev); + if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { + si_init_gfx_cgpg(rdev); + } else { + WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); + WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); + } + si_enable_dma_pg(rdev, true); + si_enable_gfx_cgpg(rdev, true); + } else { + WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); + WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); } - r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); - if (unlikely(r != 0)) { - si_rlc_fini(rdev); - return r; - } - r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, - &rdev->rlc.clear_state_gpu_addr); - radeon_bo_unreserve(rdev->rlc.clear_state_obj); - if (r) { - dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); - si_rlc_fini(rdev); - return r; +} + +static void si_fini_pg(struct radeon_device *rdev) +{ + if (rdev->pg_flags) { + si_enable_dma_pg(rdev, false); + si_enable_gfx_cgpg(rdev, false); } +} - return 0; +/* + * RLC + */ +void si_rlc_reset(struct radeon_device *rdev) +{ + u32 tmp = RREG32(GRBM_SOFT_RESET); + + tmp |= SOFT_RESET_RLC; + WREG32(GRBM_SOFT_RESET, tmp); + udelay(50); + tmp &= ~SOFT_RESET_RLC; + WREG32(GRBM_SOFT_RESET, tmp); + udelay(50); } static void si_rlc_stop(struct radeon_device *rdev) { WREG32(RLC_CNTL, 0); + + si_enable_gui_idle_interrupt(rdev, false); + + si_wait_for_rlc_serdes(rdev); } static void si_rlc_start(struct radeon_device *rdev) { WREG32(RLC_CNTL, RLC_ENABLE); + + si_enable_gui_idle_interrupt(rdev, true); + + udelay(50); +} + +static bool si_lbpw_supported(struct radeon_device *rdev) +{ + u32 tmp; + + /* Enable LBPW only for DDR3 */ + tmp = RREG32(MC_SEQ_MISC0); + if ((tmp & 0xF0000000) == 0xB0000000) + return true; + return false; +} + +static void si_enable_lbpw(struct radeon_device *rdev, bool enable) +{ + u32 tmp; + + tmp = RREG32(RLC_LB_CNTL); + if (enable) + tmp |= LOAD_BALANCE_ENABLE; + else + tmp &= ~LOAD_BALANCE_ENABLE; + WREG32(RLC_LB_CNTL, tmp); + + if (!enable) { + si_select_se_sh(rdev, 0xffffffff, 0xffffffff); + WREG32(SPI_LB_CU_MASK, 0x00ff); + } } static int si_rlc_resume(struct radeon_device *rdev) { u32 i; - const __be32 *fw_data; if (!rdev->rlc_fw) return -EINVAL; si_rlc_stop(rdev); + si_rlc_reset(rdev); + + si_init_pg(rdev); + + si_init_cg(rdev); + WREG32(RLC_RL_BASE, 0); WREG32(RLC_RL_SIZE, 0); WREG32(RLC_LB_CNTL, 0); WREG32(RLC_LB_CNTR_MAX, 0xffffffff); WREG32(RLC_LB_CNTR_INIT, 0); - - WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); - WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); + WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); WREG32(RLC_MC_CNTL, 0); WREG32(RLC_UCODE_CNTL, 0); - fw_data = (const __be32 *)rdev->rlc_fw; - for (i = 0; i < SI_RLC_UCODE_SIZE; i++) { - WREG32(RLC_UCODE_ADDR, i); - WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); + if (rdev->new_fw) { + const struct rlc_firmware_header_v1_0 *hdr = + (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; + u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; + const __le32 *fw_data = (const __le32 *) + (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + + radeon_ucode_print_rlc_hdr(&hdr->header); + + for (i = 0; i < fw_size; i++) { + WREG32(RLC_UCODE_ADDR, i); + WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++)); + } + } else { + const __be32 *fw_data = + (const __be32 *)rdev->rlc_fw->data; + for (i = 0; i < SI_RLC_UCODE_SIZE; i++) { + WREG32(RLC_UCODE_ADDR, i); + WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); + } } WREG32(RLC_UCODE_ADDR, 0); + si_enable_lbpw(rdev, si_lbpw_supported(rdev)); + si_rlc_start(rdev); return 0; @@ -3292,7 +5933,9 @@ static void si_disable_interrupt_state(struct radeon_device *rdev) { u32 tmp; - WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + tmp = RREG32(CP_INT_CNTL_RING0) & + (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + WREG32(CP_INT_CNTL_RING0, tmp); WREG32(CP_INT_CNTL_RING1, 0); WREG32(CP_INT_CNTL_RING2, 0); tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; @@ -3300,8 +5943,11 @@ static void si_disable_interrupt_state(struct radeon_device *rdev) tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp); WREG32(GRBM_INT_CNTL, 0); - WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); - WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); + WREG32(SRBM_INT_CNTL, 0); + if (rdev->num_crtc >= 2) { + WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); + WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); + } if (rdev->num_crtc >= 4) { WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); @@ -3311,8 +5957,10 @@ static void si_disable_interrupt_state(struct radeon_device *rdev) WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); } - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); + if (rdev->num_crtc >= 2) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); + } if (rdev->num_crtc >= 4) { WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); @@ -3322,21 +5970,22 @@ static void si_disable_interrupt_state(struct radeon_device *rdev) WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); } - WREG32(DACA_AUTODETECT_INT_CONTROL, 0); - - tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD1_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD2_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD3_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD4_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD5_INT_CONTROL, tmp); - tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; - WREG32(DC_HPD6_INT_CONTROL, tmp); + if (!ASIC_IS_NODCE(rdev)) { + WREG32(DAC_AUTODETECT_INT_CONTROL, 0); + tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD1_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD2_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD3_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD4_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD5_INT_CONTROL, tmp); + tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; + WREG32(DC_HPD6_INT_CONTROL, tmp); + } } static int si_irq_init(struct radeon_device *rdev) @@ -3373,7 +6022,7 @@ static int si_irq_init(struct radeon_device *rdev) WREG32(INTERRUPT_CNTL, interrupt_cntl); WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); - rb_bufsz = drm_order(rdev->ih.ring_size / 4); + rb_bufsz = order_base_2(rdev->ih.ring_size / 4); ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | IH_WPTR_OVERFLOW_CLEAR | @@ -3402,9 +6051,7 @@ static int si_irq_init(struct radeon_device *rdev) /* force the active interrupt state to all disabled */ si_disable_interrupt_state(rdev); -#ifdef notyet pci_set_master(rdev->pdev); -#endif /* enable irqs */ si_enable_interrupts(rdev); @@ -3414,13 +6061,13 @@ static int si_irq_init(struct radeon_device *rdev) int si_irq_set(struct radeon_device *rdev) { - u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; + u32 cp_int_cntl; u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; - u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; + u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0; u32 grbm_int_cntl = 0; - u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; u32 dma_cntl, dma_cntl1; + u32 thermal_int = 0; if (!rdev->irq.installed) { WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); @@ -3434,16 +6081,24 @@ int si_irq_set(struct radeon_device *rdev) return 0; } - hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; - hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; + cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & + (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); + + if (!ASIC_IS_NODCE(rdev)) { + hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); + } dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; + thermal_int = RREG32(CG_THERMAL_INT) & + ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + /* enable CP interrupts on all rings */ if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { DRM_DEBUG("si_irq_set: sw int gfx\n"); @@ -3498,27 +6153,27 @@ int si_irq_set(struct radeon_device *rdev) } if (rdev->irq.hpd[0]) { DRM_DEBUG("si_irq_set: hpd 1\n"); - hpd1 |= DC_HPDx_INT_EN; + hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[1]) { DRM_DEBUG("si_irq_set: hpd 2\n"); - hpd2 |= DC_HPDx_INT_EN; + hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[2]) { DRM_DEBUG("si_irq_set: hpd 3\n"); - hpd3 |= DC_HPDx_INT_EN; + hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[3]) { DRM_DEBUG("si_irq_set: hpd 4\n"); - hpd4 |= DC_HPDx_INT_EN; + hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[4]) { DRM_DEBUG("si_irq_set: hpd 5\n"); - hpd5 |= DC_HPDx_INT_EN; + hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } if (rdev->irq.hpd[5]) { DRM_DEBUG("si_irq_set: hpd 6\n"); - hpd6 |= DC_HPDx_INT_EN; + hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; } WREG32(CP_INT_CNTL_RING0, cp_int_cntl); @@ -3530,8 +6185,15 @@ int si_irq_set(struct radeon_device *rdev) WREG32(GRBM_INT_CNTL, grbm_int_cntl); - WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); - WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); + if (rdev->irq.dpm_thermal) { + DRM_DEBUG("dpm thermal\n"); + thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; + } + + if (rdev->num_crtc >= 2) { + WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); + WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); + } if (rdev->num_crtc >= 4) { WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); @@ -3541,23 +6203,38 @@ int si_irq_set(struct radeon_device *rdev) WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); } - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2); + if (rdev->num_crtc >= 2) { + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + } if (rdev->num_crtc >= 4) { - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); } if (rdev->num_crtc >= 6) { - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); } - WREG32(DC_HPD1_INT_CONTROL, hpd1); - WREG32(DC_HPD2_INT_CONTROL, hpd2); - WREG32(DC_HPD3_INT_CONTROL, hpd3); - WREG32(DC_HPD4_INT_CONTROL, hpd4); - WREG32(DC_HPD5_INT_CONTROL, hpd5); - WREG32(DC_HPD6_INT_CONTROL, hpd6); + if (!ASIC_IS_NODCE(rdev)) { + WREG32(DC_HPD1_INT_CONTROL, hpd1); + WREG32(DC_HPD2_INT_CONTROL, hpd2); + WREG32(DC_HPD3_INT_CONTROL, hpd3); + WREG32(DC_HPD4_INT_CONTROL, hpd4); + WREG32(DC_HPD5_INT_CONTROL, hpd5); + WREG32(DC_HPD6_INT_CONTROL, hpd6); + } + + WREG32(CG_THERMAL_INT, thermal_int); + + /* posting read */ + RREG32(SRBM_STATUS); return 0; } @@ -3566,6 +6243,9 @@ static inline void si_irq_ack(struct radeon_device *rdev) { u32 tmp; + if (ASIC_IS_NODCE(rdev)) + return; + rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); @@ -3652,10 +6332,41 @@ static inline void si_irq_ack(struct radeon_device *rdev) WREG32(DC_HPD5_INT_CONTROL, tmp); } if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { - tmp = RREG32(DC_HPD5_INT_CONTROL); + tmp = RREG32(DC_HPD6_INT_CONTROL); tmp |= DC_HPDx_INT_ACK; WREG32(DC_HPD6_INT_CONTROL, tmp); } + + if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) { + tmp = RREG32(DC_HPD1_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD1_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) { + tmp = RREG32(DC_HPD2_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD2_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) { + tmp = RREG32(DC_HPD3_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD3_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) { + tmp = RREG32(DC_HPD4_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD4_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) { + tmp = RREG32(DC_HPD5_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD5_INT_CONTROL, tmp); + } + if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) { + tmp = RREG32(DC_HPD6_INT_CONTROL); + tmp |= DC_HPDx_RX_INT_ACK; + WREG32(DC_HPD6_INT_CONTROL, tmp); + } } static void si_irq_disable(struct radeon_device *rdev) @@ -3689,12 +6400,13 @@ static inline u32 si_get_ih_wptr(struct radeon_device *rdev) wptr = RREG32(IH_RB_WPTR); if (wptr & RB_OVERFLOW) { + wptr &= ~RB_OVERFLOW; /* When a ring buffer overflow happen start parsing interrupt * from the last not overwritten vector (wptr + 16). Hopefully * this should allow us to catchup. */ - dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", - wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); + dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; tmp = RREG32(IH_RB_CNTL); tmp |= IH_WPTR_OVERFLOW_CLEAR; @@ -3720,14 +6432,15 @@ int si_irq_process(struct radeon_device *rdev) u32 src_id, src_data, ring_id; u32 ring_index; bool queue_hotplug = false; + bool queue_dp = false; + bool queue_thermal = false; + u32 status, addr; if (!rdev->ih.enabled || rdev->shutdown) return IRQ_NONE; wptr = si_get_ih_wptr(rdev); - if (wptr == rdev->ih.rptr) - return IRQ_NONE; restart_ih: /* is somebody else already processing irqs? */ if (atomic_xchg(&rdev->ih.lock, 1)) @@ -3753,23 +6466,27 @@ restart_ih: case 1: /* D1 vblank/vline */ switch (src_data) { case 0: /* D1 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[0]) { - drm_handle_vblank(rdev->ddev, 0); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[0])) - radeon_crtc_handle_flip(rdev, 0); - rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D1 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[0]) { + drm_handle_vblank(rdev->ddev, 0); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[0])) + radeon_crtc_handle_vblank(rdev, 0); + rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D1 vblank\n"); + break; case 1: /* D1 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; - DRM_DEBUG("IH: D1 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; + DRM_DEBUG("IH: D1 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3779,23 +6496,27 @@ restart_ih: case 2: /* D2 vblank/vline */ switch (src_data) { case 0: /* D2 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[1]) { - drm_handle_vblank(rdev->ddev, 1); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[1])) - radeon_crtc_handle_flip(rdev, 1); - rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D2 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[1]) { + drm_handle_vblank(rdev->ddev, 1); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[1])) + radeon_crtc_handle_vblank(rdev, 1); + rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D2 vblank\n"); + break; case 1: /* D2 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; - DRM_DEBUG("IH: D2 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; + DRM_DEBUG("IH: D2 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3805,23 +6526,27 @@ restart_ih: case 3: /* D3 vblank/vline */ switch (src_data) { case 0: /* D3 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[2]) { - drm_handle_vblank(rdev->ddev, 2); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[2])) - radeon_crtc_handle_flip(rdev, 2); - rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D3 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[2]) { + drm_handle_vblank(rdev->ddev, 2); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[2])) + radeon_crtc_handle_vblank(rdev, 2); + rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D3 vblank\n"); + break; case 1: /* D3 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; - DRM_DEBUG("IH: D3 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; + DRM_DEBUG("IH: D3 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3831,23 +6556,27 @@ restart_ih: case 4: /* D4 vblank/vline */ switch (src_data) { case 0: /* D4 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[3]) { - drm_handle_vblank(rdev->ddev, 3); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[3])) - radeon_crtc_handle_flip(rdev, 3); - rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D4 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[3]) { + drm_handle_vblank(rdev->ddev, 3); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[3])) + radeon_crtc_handle_vblank(rdev, 3); + rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D4 vblank\n"); + break; case 1: /* D4 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; - DRM_DEBUG("IH: D4 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; + DRM_DEBUG("IH: D4 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3857,23 +6586,27 @@ restart_ih: case 5: /* D5 vblank/vline */ switch (src_data) { case 0: /* D5 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[4]) { - drm_handle_vblank(rdev->ddev, 4); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[4])) - radeon_crtc_handle_flip(rdev, 4); - rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D5 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[4]) { + drm_handle_vblank(rdev->ddev, 4); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[4])) + radeon_crtc_handle_vblank(rdev, 4); + rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D5 vblank\n"); + break; case 1: /* D5 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; - DRM_DEBUG("IH: D5 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; + DRM_DEBUG("IH: D5 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); @@ -3883,87 +6616,180 @@ restart_ih: case 6: /* D6 vblank/vline */ switch (src_data) { case 0: /* D6 vblank */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { - if (rdev->irq.crtc_vblank_int[5]) { - drm_handle_vblank(rdev->ddev, 5); - rdev->pm.vblank_sync = true; - wake_up(&rdev->irq.vblank_queue); - } - if (atomic_read(&rdev->irq.pflip[5])) - radeon_crtc_handle_flip(rdev, 5); - rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; - DRM_DEBUG("IH: D6 vblank\n"); + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + if (rdev->irq.crtc_vblank_int[5]) { + drm_handle_vblank(rdev->ddev, 5); + rdev->pm.vblank_sync = true; + wake_up(&rdev->irq.vblank_queue); } + if (atomic_read(&rdev->irq.pflip[5])) + radeon_crtc_handle_vblank(rdev, 5); + rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; + DRM_DEBUG("IH: D6 vblank\n"); + break; case 1: /* D6 vline */ - if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; - DRM_DEBUG("IH: D6 vline\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; + DRM_DEBUG("IH: D6 vline\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); break; } break; + case 8: /* D1 page flip */ + case 10: /* D2 page flip */ + case 12: /* D3 page flip */ + case 14: /* D4 page flip */ + case 16: /* D5 page flip */ + case 18: /* D6 page flip */ + DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); + if (radeon_use_pflipirq > 0) + radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); + break; case 42: /* HPD hotplug */ switch (src_data) { case 0: - if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD1\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD1\n"); + break; case 1: - if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD2\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD2\n"); + break; case 2: - if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD3\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD3\n"); + break; case 3: - if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD4\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD4\n"); + break; case 4: - if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD5\n"); - } + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD5\n"); + + break; + case 5: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; + queue_hotplug = true; + DRM_DEBUG("IH: HPD6\n"); + + break; + case 6: + if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 1\n"); + + break; + case 7: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 2\n"); + + break; + case 8: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 3\n"); + + break; + case 9: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 4\n"); + + break; + case 10: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 5\n"); + break; - case 5: - if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { - rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; - queue_hotplug = true; - DRM_DEBUG("IH: HPD6\n"); - } + case 11: + if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT)) + DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); + + rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT; + queue_dp = true; + DRM_DEBUG("IH: HPD_RX 6\n"); + break; default: DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); break; } break; + case 96: + DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); + WREG32(SRBM_INT_ACK, 0x1); + break; + case 124: /* UVD */ + DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); + radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); + break; case 146: case 147: + addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); + status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); + /* reset addr and status */ + WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); + if (addr == 0x0 && status == 0x0) + break; dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", - RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); + addr); dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", - RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); - /* reset addr and status */ - WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); + status); + si_vm_decode_fault(rdev, status, addr); break; case 176: /* RINGID0 CP_INT */ radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); @@ -3992,6 +6818,16 @@ restart_ih: DRM_DEBUG("IH: DMA trap\n"); radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); break; + case 230: /* thermal low to high */ + DRM_DEBUG("IH: thermal low to high\n"); + rdev->pm.dpm.thermal.high_to_low = false; + queue_thermal = true; + break; + case 231: /* thermal high to low */ + DRM_DEBUG("IH: thermal high to low\n"); + rdev->pm.dpm.thermal.high_to_low = true; + queue_thermal = true; + break; case 233: /* GUI IDLE */ DRM_DEBUG("IH: GUI idle\n"); break; @@ -4007,11 +6843,15 @@ restart_ih: /* wptr/rptr are in bytes! */ rptr += 16; rptr &= rdev->ih.ptr_mask; + WREG32(IH_RB_RPTR, rptr); } + if (queue_dp) + schedule_work(&rdev->dp_work); if (queue_hotplug) - task_add(systq, &rdev->hotplug_task); + schedule_delayed_work(&rdev->hotplug_work, 0); + if (queue_thermal && rdev->pm.dpm_enabled) + schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; - WREG32(IH_RB_RPTR, rdev->ih.rptr); atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ @@ -4022,80 +6862,6 @@ restart_ih: return IRQ_HANDLED; } -/** - * si_copy_dma - copy pages using the DMA engine - * - * @rdev: radeon_device pointer - * @src_offset: src GPU address - * @dst_offset: dst GPU address - * @num_gpu_pages: number of GPU pages to xfer - * @fence: radeon fence object - * - * Copy GPU paging using the DMA engine (SI). - * Used by the radeon ttm implementation to move pages if - * registered as the asic copy callback. - */ -int si_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence) -{ - struct radeon_semaphore *sem = NULL; - int ring_index = rdev->asic->copy.dma_ring_index; - struct radeon_ring *ring = &rdev->ring[ring_index]; - u32 size_in_bytes, cur_size_in_bytes; - int i, num_loops; - int r = 0; - - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return r; - } - - size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); - num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff); - r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); - return r; - } - - if (radeon_fence_need_sync(*fence, ring->idx)) { - radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring, - ring->idx); - radeon_fence_note_sync(*fence, ring->idx); - } else { - radeon_semaphore_free(rdev, &sem, NULL); - } - - for (i = 0; i < num_loops; i++) { - cur_size_in_bytes = size_in_bytes; - if (cur_size_in_bytes > 0xFFFFF) - cur_size_in_bytes = 0xFFFFF; - size_in_bytes -= cur_size_in_bytes; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes)); - radeon_ring_write(ring, dst_offset & 0xffffffff); - radeon_ring_write(ring, src_offset & 0xffffffff); - radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); - radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); - src_offset += cur_size_in_bytes; - dst_offset += cur_size_in_bytes; - } - - r = radeon_fence_emit(rdev, fence, ring->idx); - if (r) { - radeon_ring_unlock_undo(rdev, ring); - return r; - } - - radeon_ring_unlock_commit(rdev, ring); - radeon_semaphore_free(rdev, &sem, *fence); - - return r; -} - /* * startup/shutdown callbacks */ @@ -4104,42 +6870,39 @@ static int si_startup(struct radeon_device *rdev) struct radeon_ring *ring; int r; + /* enable pcie gen2/3 link */ + si_pcie_gen3_enable(rdev); + /* enable aspm */ + si_program_aspm(rdev); + + /* scratch needs to be initialized before MC */ + r = r600_vram_scratch_init(rdev); + if (r) + return r; + si_mc_program(rdev); - if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || - !rdev->rlc_fw || !rdev->mc_fw) { - r = si_init_microcode(rdev); + if (!rdev->pm.dpm_enabled) { + r = si_mc_load_microcode(rdev); if (r) { - DRM_ERROR("Failed to load firmware!\n"); + DRM_ERROR("Failed to load MC firmware!\n"); return r; } } - r = si_mc_load_microcode(rdev); - if (r) { - DRM_ERROR("Failed to load MC firmware!\n"); - return r; - } - - r = r600_vram_scratch_init(rdev); - if (r) - return r; - r = si_pcie_gart_enable(rdev); if (r) return r; si_gpu_init(rdev); -#if 0 - r = evergreen_blit_init(rdev); - if (r) { - r600_blit_fini(rdev); - rdev->asic->copy = NULL; - dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); - } -#endif /* allocate rlc buffers */ - r = si_rlc_init(rdev); + if (rdev->family == CHIP_VERDE) { + rdev->rlc.reg_list = verde_rlc_save_restore_register_list; + rdev->rlc.reg_list_size = + (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list); + } + rdev->rlc.cs_data = si_cs_data; + r = sumo_rlc_init(rdev); if (r) { DRM_ERROR("Failed to init rlc BOs!\n"); return r; @@ -4180,6 +6943,34 @@ static int si_startup(struct radeon_device *rdev) return r; } + if (rdev->has_uvd) { + r = uvd_v2_2_resume(rdev); + if (!r) { + r = radeon_fence_driver_start_ring(rdev, + R600_RING_TYPE_UVD_INDEX); + if (r) + dev_err(rdev->dev, "UVD fences init error (%d).\n", r); + } + if (r) + rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; + } + + r = radeon_vce_resume(rdev); + if (!r) { + r = vce_v1_0_resume(rdev); + if (!r) + r = radeon_fence_driver_start_ring(rdev, + TN_RING_TYPE_VCE1_INDEX); + if (!r) + r = radeon_fence_driver_start_ring(rdev, + TN_RING_TYPE_VCE2_INDEX); + } + if (r) { + dev_err(rdev->dev, "VCE init error (%d).\n", r); + rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; + rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; + } + /* Enable IRQ */ if (!rdev->irq.installed) { r = radeon_irq_kms_init(rdev); @@ -4197,38 +6988,31 @@ static int si_startup(struct radeon_device *rdev) ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, - CP_RB0_RPTR, CP_RB0_WPTR, - 0, 0xfffff, RADEON_CP_PACKET2); + RADEON_CP_PACKET2); if (r) return r; ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, - CP_RB1_RPTR, CP_RB1_WPTR, - 0, 0xfffff, RADEON_CP_PACKET2); + RADEON_CP_PACKET2); if (r) return r; ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, - CP_RB2_RPTR, CP_RB2_WPTR, - 0, 0xfffff, RADEON_CP_PACKET2); + RADEON_CP_PACKET2); if (r) return r; ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, - DMA_RB_RPTR + DMA0_REGISTER_OFFSET, - DMA_RB_WPTR + DMA0_REGISTER_OFFSET, - 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); + DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); if (r) return r; ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, - DMA_RB_RPTR + DMA1_REGISTER_OFFSET, - DMA_RB_WPTR + DMA1_REGISTER_OFFSET, - 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); + DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); if (r) return r; @@ -4243,6 +7027,35 @@ static int si_startup(struct radeon_device *rdev) if (r) return r; + if (rdev->has_uvd) { + ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + if (ring->ring_size) { + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + RADEON_CP_PACKET2); + if (!r) + r = uvd_v1_0_init(rdev); + if (r) + DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); + } + } + + r = -ENOENT; + + ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; + if (ring->ring_size) + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + VCE_CMD_NO_OP); + + ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; + if (ring->ring_size) + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + VCE_CMD_NO_OP); + + if (!r) + r = vce_v1_0_init(rdev); + else if (r != -ENOENT) + DRM_ERROR("radeon: failed initializing VCE (%d).\n", r); + r = radeon_ib_pool_init(rdev); if (r) { dev_err(rdev->dev, "IB initialization failed (%d).\n", r); @@ -4255,6 +7068,10 @@ static int si_startup(struct radeon_device *rdev) return r; } + r = radeon_audio_init(rdev); + if (r) + return r; + return 0; } @@ -4269,6 +7086,12 @@ int si_resume(struct radeon_device *rdev) /* post card */ atom_asic_init(rdev->mode_info.atom_context); + /* init golden registers */ + si_init_golden_registers(rdev); + + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_resume(rdev); + rdev->accel_working = true; r = si_startup(rdev); if (r) { @@ -4283,9 +7106,18 @@ int si_resume(struct radeon_device *rdev) int si_suspend(struct radeon_device *rdev) { + radeon_pm_suspend(rdev); + radeon_audio_fini(rdev); radeon_vm_manager_fini(rdev); si_cp_enable(rdev, false); cayman_dma_stop(rdev); + if (rdev->has_uvd) { + uvd_v1_0_fini(rdev); + radeon_uvd_suspend(rdev); + radeon_vce_suspend(rdev); + } + si_fini_pg(rdev); + si_fini_cg(rdev); si_irq_suspend(rdev); radeon_wb_disable(rdev); si_pcie_gart_disable(rdev); @@ -4326,6 +7158,8 @@ int si_init(struct radeon_device *rdev) DRM_INFO("GPU not posted. posting now...\n"); atom_asic_init(rdev->mode_info.atom_context); } + /* init golden registers */ + si_init_golden_registers(rdev); /* Initialize scratch registers */ si_scratch_init(rdev); /* Initialize surface registers */ @@ -4347,6 +7181,18 @@ int si_init(struct radeon_device *rdev) if (r) return r; + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || + !rdev->rlc_fw || !rdev->mc_fw) { + r = si_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + + /* Initialize power management */ + radeon_pm_init(rdev); + ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; ring->ring_obj = NULL; r600_ring_init(rdev, ring, 1024 * 1024); @@ -4367,6 +7213,26 @@ int si_init(struct radeon_device *rdev) ring->ring_obj = NULL; r600_ring_init(rdev, ring, 64 * 1024); + if (rdev->has_uvd) { + r = radeon_uvd_init(rdev); + if (!r) { + ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 4096); + } + } + + r = radeon_vce_init(rdev); + if (!r) { + ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 4096); + + ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; + ring->ring_obj = NULL; + r600_ring_init(rdev, ring, 4096); + } + rdev->ih.ring_obj = NULL; r600_ih_ring_init(rdev, 64 * 1024); @@ -4381,7 +7247,7 @@ int si_init(struct radeon_device *rdev) si_cp_fini(rdev); cayman_dma_fini(rdev); si_irq_fini(rdev); - si_rlc_fini(rdev); + sumo_rlc_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); radeon_vm_manager_fini(rdev); @@ -4404,17 +7270,22 @@ int si_init(struct radeon_device *rdev) void si_fini(struct radeon_device *rdev) { -#if 0 - r600_blit_fini(rdev); -#endif + radeon_pm_fini(rdev); si_cp_fini(rdev); cayman_dma_fini(rdev); + si_fini_pg(rdev); + si_fini_cg(rdev); si_irq_fini(rdev); - si_rlc_fini(rdev); + sumo_rlc_fini(rdev); radeon_wb_fini(rdev); radeon_vm_manager_fini(rdev); radeon_ib_pool_fini(rdev); radeon_irq_kms_fini(rdev); + if (rdev->has_uvd) { + uvd_v1_0_fini(rdev); + radeon_uvd_fini(rdev); + radeon_vce_fini(rdev); + } si_pcie_gart_fini(rdev); r600_vram_scratch_fini(rdev); radeon_gem_fini(rdev); @@ -4426,14 +7297,14 @@ void si_fini(struct radeon_device *rdev) } /** - * si_get_gpu_clock - return GPU clock counter snapshot + * si_get_gpu_clock_counter - return GPU clock counter snapshot * * @rdev: radeon_device pointer * * Fetches a GPU clock counter snapshot (SI). * Returns the 64 bit clock counter snapshot. */ -uint64_t si_get_gpu_clock(struct radeon_device *rdev) +uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev) { uint64_t clock; @@ -4444,3 +7315,588 @@ uint64_t si_get_gpu_clock(struct radeon_device *rdev) mutex_unlock(&rdev->gpu_clock_mutex); return clock; } + +int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) +{ + unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; + int r; + + /* bypass vclk and dclk with bclk */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), + ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); + + /* put PLL in bypass mode */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); + + if (!vclk || !dclk) { + /* keep the Bypass mode */ + return 0; + } + + r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, + 16384, 0x03FFFFFF, 0, 128, 5, + &fb_div, &vclk_div, &dclk_div); + if (r) + return r; + + /* set RESET_ANTI_MUX to 0 */ + WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK); + + /* set VCO_MODE to 1 */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); + + /* disable sleep mode */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); + + /* deassert UPLL_RESET */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); + + mdelay(1); + + r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); + if (r) + return r; + + /* assert UPLL_RESET again */ + WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); + + /* disable spread spectrum. */ + WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); + + /* set feedback divider */ + WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); + + /* set ref divider to 0 */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); + + if (fb_div < 307200) + WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); + else + WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); + + /* set PDIV_A and PDIV_B */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), + ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); + + /* give the PLL some time to settle */ + mdelay(15); + + /* deassert PLL_RESET */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); + + mdelay(15); + + /* switch from bypass mode to normal mode */ + WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); + + r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); + if (r) + return r; + + /* switch VCLK and DCLK selection */ + WREG32_P(CG_UPLL_FUNC_CNTL_2, + VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), + ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); + + mdelay(100); + + return 0; +} + +static void si_pcie_gen3_enable(struct radeon_device *rdev) +{ + struct pci_dev _root; + struct pci_dev *root; + int bridge_pos, gpu_pos; + u32 speed_cntl, mask, current_data_rate; + int ret, i; + u16 tmp16; + + root = &_root; + root->pc = rdev->pdev->pc; + root->tag = *rdev->ddev->bridgetag; + + if (pci_is_root_bus(rdev->pdev->bus)) + return; + + if (radeon_pcie_gen2 == 0) + return; + + if (rdev->flags & RADEON_IS_IGP) + return; + + if (!(rdev->flags & RADEON_IS_PCIE)) + return; + + ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); + if (ret != 0) + return; + + if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) + return; + + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> + LC_CURRENT_DATA_RATE_SHIFT; + if (mask & DRM_PCIE_SPEED_80) { + if (current_data_rate == 2) { + DRM_INFO("PCIE gen 3 link speeds already enabled\n"); + return; + } + DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); + } else if (mask & DRM_PCIE_SPEED_50) { + if (current_data_rate == 1) { + DRM_INFO("PCIE gen 2 link speeds already enabled\n"); + return; + } + DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); + } + + bridge_pos = pci_pcie_cap(root); + if (!bridge_pos) + return; + + gpu_pos = pci_pcie_cap(rdev->pdev); + if (!gpu_pos) + return; + + if (mask & DRM_PCIE_SPEED_80) { + /* re-try equalization if gen3 is not already enabled */ + if (current_data_rate != 2) { + u16 bridge_cfg, gpu_cfg; + u16 bridge_cfg2, gpu_cfg2; + u32 max_lw, current_lw, tmp; + + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); + + tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; + pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); + + tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; + pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); + + tmp = RREG32_PCIE(PCIE_LC_STATUS1); + max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; + current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; + + if (current_lw < max_lw) { + tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); + if (tmp & LC_RENEGOTIATION_SUPPORT) { + tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); + tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); + tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); + } + } + + for (i = 0; i < 10; i++) { + /* check status */ + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); + if (tmp16 & PCI_EXP_DEVSTA_TRPND) + break; + + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); + + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); + tmp |= LC_SET_QUIESCE; + WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); + tmp |= LC_REDO_EQ; + WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); + + mdelay(100); + + /* linkctl */ + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); + tmp16 &= ~PCI_EXP_LNKCTL_HAWD; + tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); + pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); + + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); + tmp16 &= ~PCI_EXP_LNKCTL_HAWD; + tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); + pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); + + /* linkctl2 */ + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); + tmp16 &= ~((1 << 4) | (7 << 9)); + tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); + pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); + + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); + tmp16 &= ~((1 << 4) | (7 << 9)); + tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); + pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); + + tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); + tmp &= ~LC_SET_QUIESCE; + WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); + } + } + } + + /* set the link speed */ + speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; + speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); + + pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); + tmp16 &= ~0xf; + if (mask & DRM_PCIE_SPEED_80) + tmp16 |= 3; /* gen3 */ + else if (mask & DRM_PCIE_SPEED_50) + tmp16 |= 2; /* gen2 */ + else + tmp16 |= 1; /* gen1 */ + pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); + + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; + WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); + + for (i = 0; i < rdev->usec_timeout; i++) { + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); + if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) + break; + udelay(1); + } +} + +static void si_program_aspm(struct radeon_device *rdev) +{ + u32 data, orig; + bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; + bool disable_clkreq = false; + + if (radeon_aspm == 0) + return; + + if (!(rdev->flags & RADEON_IS_PCIE)) + return; + + orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); + data &= ~LC_XMIT_N_FTS_MASK; + data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); + + orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); + data |= LC_GO_TO_RECOVERY; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); + + orig = data = RREG32_PCIE(PCIE_P_CNTL); + data |= P_IGNORE_EDB_ERR; + if (orig != data) + WREG32_PCIE(PCIE_P_CNTL, data); + + orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); + data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); + data |= LC_PMI_TO_L1_DIS; + if (!disable_l0s) + data |= LC_L0S_INACTIVITY(7); + + if (!disable_l1) { + data |= LC_L1_INACTIVITY(7); + data &= ~LC_PMI_TO_L1_DIS; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL, data); + + if (!disable_plloff_in_l1) { + bool clk_req_support; + + orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); + data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); + if (orig != data) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); + + orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); + data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); + if (orig != data) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); + + orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); + data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); + if (orig != data) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); + + orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); + data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); + if (orig != data) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); + + if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) { + orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); + data &= ~PLL_RAMP_UP_TIME_0_MASK; + if (orig != data) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); + + orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); + data &= ~PLL_RAMP_UP_TIME_1_MASK; + if (orig != data) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); + + orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2); + data &= ~PLL_RAMP_UP_TIME_2_MASK; + if (orig != data) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data); + + orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3); + data &= ~PLL_RAMP_UP_TIME_3_MASK; + if (orig != data) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data); + + orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); + data &= ~PLL_RAMP_UP_TIME_0_MASK; + if (orig != data) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); + + orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); + data &= ~PLL_RAMP_UP_TIME_1_MASK; + if (orig != data) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); + + orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2); + data &= ~PLL_RAMP_UP_TIME_2_MASK; + if (orig != data) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data); + + orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3); + data &= ~PLL_RAMP_UP_TIME_3_MASK; + if (orig != data) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data); + } + orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); + data &= ~LC_DYN_LANES_PWR_STATE_MASK; + data |= LC_DYN_LANES_PWR_STATE(3); + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); + + orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL); + data &= ~LS2_EXIT_TIME_MASK; + if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) + data |= LS2_EXIT_TIME(5); + if (orig != data) + WREG32_PIF_PHY0(PB0_PIF_CNTL, data); + + orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL); + data &= ~LS2_EXIT_TIME_MASK; + if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) + data |= LS2_EXIT_TIME(5); + if (orig != data) + WREG32_PIF_PHY1(PB1_PIF_CNTL, data); + + if (!disable_clkreq && + !pci_is_root_bus(rdev->pdev->bus)) { + u32 lnkcap; + struct pci_dev _root; + struct pci_dev *root; + + root = &_root; + root->pc = rdev->pdev->pc; + root->tag = *rdev->ddev->bridgetag; + + clk_req_support = false; + pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); + if (lnkcap & PCI_EXP_LNKCAP_CLKPM) + clk_req_support = true; + } else { + clk_req_support = false; + } + + if (clk_req_support) { + orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); + data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); + + orig = data = RREG32(THM_CLK_CNTL); + data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); + data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); + if (orig != data) + WREG32(THM_CLK_CNTL, data); + + orig = data = RREG32(MISC_CLK_CNTL); + data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); + data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); + if (orig != data) + WREG32(MISC_CLK_CNTL, data); + + orig = data = RREG32(CG_CLKPIN_CNTL); + data &= ~BCLK_AS_XCLK; + if (orig != data) + WREG32(CG_CLKPIN_CNTL, data); + + orig = data = RREG32(CG_CLKPIN_CNTL_2); + data &= ~FORCE_BIF_REFCLK_EN; + if (orig != data) + WREG32(CG_CLKPIN_CNTL_2, data); + + orig = data = RREG32(MPLL_BYPASSCLK_SEL); + data &= ~MPLL_CLKOUT_SEL_MASK; + data |= MPLL_CLKOUT_SEL(4); + if (orig != data) + WREG32(MPLL_BYPASSCLK_SEL, data); + + orig = data = RREG32(SPLL_CNTL_MODE); + data &= ~SPLL_REFCLK_SEL_MASK; + if (orig != data) + WREG32(SPLL_CNTL_MODE, data); + } + } + } else { + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL, data); + } + + orig = data = RREG32_PCIE(PCIE_CNTL2); + data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; + if (orig != data) + WREG32_PCIE(PCIE_CNTL2, data); + + if (!disable_l0s) { + data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); + if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { + data = RREG32_PCIE(PCIE_LC_STATUS1); + if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { + orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); + data &= ~LC_L0S_INACTIVITY_MASK; + if (orig != data) + WREG32_PCIE_PORT(PCIE_LC_CNTL, data); + } + } + } +} + +int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev) +{ + unsigned i; + + /* make sure VCEPLL_CTLREQ is deasserted */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK); + + mdelay(10); + + /* assert UPLL_CTLREQ */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK); + + /* wait for CTLACK and CTLACK2 to get asserted */ + for (i = 0; i < 100; ++i) { + uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK; + if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask) + break; + mdelay(10); + } + + /* deassert UPLL_CTLREQ */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK); + + if (i == 100) { + DRM_ERROR("Timeout setting UVD clocks!\n"); + return -ETIMEDOUT; + } + + return 0; +} + +int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) +{ + unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0; + int r; + + /* bypass evclk and ecclk with bclk */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2, + EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1), + ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK)); + + /* put PLL in bypass mode */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK, + ~VCEPLL_BYPASS_EN_MASK); + + if (!evclk || !ecclk) { + /* keep the Bypass mode, put PLL to sleep */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK, + ~VCEPLL_SLEEP_MASK); + return 0; + } + + r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000, + 16384, 0x03FFFFFF, 0, 128, 5, + &fb_div, &evclk_div, &ecclk_div); + if (r) + return r; + + /* set RESET_ANTI_MUX to 0 */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK); + + /* set VCO_MODE to 1 */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK, + ~VCEPLL_VCO_MODE_MASK); + + /* toggle VCEPLL_SLEEP to 1 then back to 0 */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK, + ~VCEPLL_SLEEP_MASK); + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK); + + /* deassert VCEPLL_RESET */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK); + + mdelay(1); + + r = si_vce_send_vcepll_ctlreq(rdev); + if (r) + return r; + + /* assert VCEPLL_RESET again */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK); + + /* disable spread spectrum. */ + WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); + + /* set feedback divider */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK); + + /* set ref divider to 0 */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK); + + /* set PDIV_A and PDIV_B */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2, + VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div), + ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK)); + + /* give the PLL some time to settle */ + mdelay(15); + + /* deassert PLL_RESET */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK); + + mdelay(15); + + /* switch from bypass mode to normal mode */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK); + + r = si_vce_send_vcepll_ctlreq(rdev); + if (r) + return r; + + /* switch VCLK and DCLK selection */ + WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2, + EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16), + ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK)); + + mdelay(100); + + return 0; +} diff --git a/sys/dev/pci/drm/radeon/si_blit_shaders.c b/sys/dev/pci/drm/radeon/si_blit_shaders.c index 45f87fdb676..7e1ff00997d 100644 --- a/sys/dev/pci/drm/radeon/si_blit_shaders.c +++ b/sys/dev/pci/drm/radeon/si_blit_shaders.c @@ -1,4 +1,3 @@ -/* $OpenBSD: si_blit_shaders.c,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2011 Advanced Micro Devices, Inc. * @@ -25,9 +24,7 @@ * Alex Deucher */ -#include - -#include +#include const u32 si_default_state[] = { diff --git a/sys/dev/pci/drm/radeon/si_blit_shaders.h b/sys/dev/pci/drm/radeon/si_blit_shaders.h index ac2abf9bf50..c739e51e396 100644 --- a/sys/dev/pci/drm/radeon/si_blit_shaders.h +++ b/sys/dev/pci/drm/radeon/si_blit_shaders.h @@ -1,4 +1,3 @@ -/* $OpenBSD: si_blit_shaders.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2011 Advanced Micro Devices, Inc. * diff --git a/sys/dev/pci/drm/radeon/si_dma.c b/sys/dev/pci/drm/radeon/si_dma.c new file mode 100644 index 00000000000..9b2ab3044fa --- /dev/null +++ b/sys/dev/pci/drm/radeon/si_dma.c @@ -0,0 +1,284 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "radeon_trace.h" +#include "sid.h" + +u32 si_gpu_check_soft_reset(struct radeon_device *rdev); + +/** + * si_dma_is_lockup - Check if the DMA engine is locked up + * + * @rdev: radeon_device pointer + * @ring: radeon_ring structure holding ring information + * + * Check if the async DMA engine is locked up. + * Returns true if the engine appears to be locked up, false if not. + */ +bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +{ + u32 reset_mask = si_gpu_check_soft_reset(rdev); + u32 mask; + + if (ring->idx == R600_RING_TYPE_DMA_INDEX) + mask = RADEON_RESET_DMA; + else + mask = RADEON_RESET_DMA1; + + if (!(reset_mask & mask)) { + radeon_ring_lockup_update(rdev, ring); + return false; + } + return radeon_ring_test_lockup(rdev, ring); +} + +/** + * si_dma_vm_copy_pages - update PTEs by copying them from the GART + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @src: src addr where to copy from + * @count: number of page entries to update + * + * Update PTEs by copying them from the GART using the DMA (SI). + */ +void si_dma_vm_copy_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, uint64_t src, + unsigned count) +{ + while (count) { + unsigned bytes = count * 8; + if (bytes > 0xFFFF8) + bytes = 0xFFFF8; + + ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY, + 1, 0, 0, bytes); + ib->ptr[ib->length_dw++] = lower_32_bits(pe); + ib->ptr[ib->length_dw++] = lower_32_bits(src); + ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; + ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff; + + pe += bytes; + src += bytes; + count -= bytes / 8; + } +} + +/** + * si_dma_vm_write_pages - update PTEs by writing them manually + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: access flags + * + * Update PTEs by writing them manually using the DMA (SI). + */ +void si_dma_vm_write_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags) +{ + uint64_t value; + unsigned ndw; + + while (count) { + ndw = count * 2; + if (ndw > 0xFFFFE) + ndw = 0xFFFFE; + + /* for non-physically contiguous pages (system) */ + ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw); + ib->ptr[ib->length_dw++] = pe; + ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; + for (; ndw > 0; ndw -= 2, --count, pe += 8) { + if (flags & R600_PTE_SYSTEM) { + value = radeon_vm_map_gart(rdev, addr); + } else if (flags & R600_PTE_VALID) { + value = addr; + } else { + value = 0; + } + addr += incr; + value |= flags; + ib->ptr[ib->length_dw++] = value; + ib->ptr[ib->length_dw++] = upper_32_bits(value); + } + } +} + +/** + * si_dma_vm_set_pages - update the page tables using the DMA + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: access flags + * + * Update the page tables using the DMA (SI). + */ +void si_dma_vm_set_pages(struct radeon_device *rdev, + struct radeon_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint32_t flags) +{ + uint64_t value; + unsigned ndw; + + while (count) { + ndw = count * 2; + if (ndw > 0xFFFFE) + ndw = 0xFFFFE; + + if (flags & R600_PTE_VALID) + value = addr; + else + value = 0; + + /* for physically contiguous pages (vram) */ + ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw); + ib->ptr[ib->length_dw++] = pe; /* dst addr */ + ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; + ib->ptr[ib->length_dw++] = flags; /* mask */ + ib->ptr[ib->length_dw++] = 0; + ib->ptr[ib->length_dw++] = value; /* value */ + ib->ptr[ib->length_dw++] = upper_32_bits(value); + ib->ptr[ib->length_dw++] = incr; /* increment size */ + ib->ptr[ib->length_dw++] = 0; + pe += ndw * 4; + addr += (ndw / 2) * incr; + count -= ndw / 2; + } +} + +void si_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) + +{ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); + if (vm_id < 8) { + radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2)); + } else { + radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2)); + } + radeon_ring_write(ring, pd_addr >> 12); + + /* flush hdp cache */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); + radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); + radeon_ring_write(ring, 1); + + /* bits 0-7 are the VM contexts0-7 */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); + radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2)); + radeon_ring_write(ring, 1 << vm_id); + + /* wait for invalidate to complete */ + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0)); + radeon_ring_write(ring, VM_INVALIDATE_REQUEST); + radeon_ring_write(ring, 0xff << 16); /* retry */ + radeon_ring_write(ring, 1 << vm_id); /* mask */ + radeon_ring_write(ring, 0); /* value */ + radeon_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */ +} + +/** + * si_copy_dma - copy pages using the DMA engine + * + * @rdev: radeon_device pointer + * @src_offset: src GPU address + * @dst_offset: dst GPU address + * @num_gpu_pages: number of GPU pages to xfer + * @resv: reservation object to sync to + * + * Copy GPU paging using the DMA engine (SI). + * Used by the radeon ttm implementation to move pages if + * registered as the asic copy callback. + */ +struct radeon_fence *si_copy_dma(struct radeon_device *rdev, + uint64_t src_offset, uint64_t dst_offset, + unsigned num_gpu_pages, + struct reservation_object *resv) +{ + struct radeon_fence *fence; + struct radeon_sync sync; + int ring_index = rdev->asic->copy.dma_ring_index; + struct radeon_ring *ring = &rdev->ring[ring_index]; + u32 size_in_bytes, cur_size_in_bytes; + int i, num_loops; + int r = 0; + + radeon_sync_create(&sync); + + size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); + num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff); + r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); + if (r) { + DRM_ERROR("radeon: moving bo (%d).\n", r); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); + + for (i = 0; i < num_loops; i++) { + cur_size_in_bytes = size_in_bytes; + if (cur_size_in_bytes > 0xFFFFF) + cur_size_in_bytes = 0xFFFFF; + size_in_bytes -= cur_size_in_bytes; + radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes)); + radeon_ring_write(ring, lower_32_bits(dst_offset)); + radeon_ring_write(ring, lower_32_bits(src_offset)); + radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); + radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); + src_offset += cur_size_in_bytes; + dst_offset += cur_size_in_bytes; + } + + r = radeon_fence_emit(rdev, &fence, ring->idx); + if (r) { + radeon_ring_unlock_undo(rdev, ring); + radeon_sync_free(rdev, &sync, NULL); + return ERR_PTR(r); + } + + radeon_ring_unlock_commit(rdev, ring, false); + radeon_sync_free(rdev, &sync, fence); + + return fence; +} + diff --git a/sys/dev/pci/drm/radeon/si_dpm.c b/sys/dev/pci/drm/radeon/si_dpm.c new file mode 100644 index 00000000000..688f90bde23 --- /dev/null +++ b/sys/dev/pci/drm/radeon/si_dpm.c @@ -0,0 +1,7178 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "sid.h" +#include "r600_dpm.h" +#include "si_dpm.h" +#include "atom.h" + +#define MC_CG_ARB_FREQ_F0 0x0a +#define MC_CG_ARB_FREQ_F1 0x0b +#define MC_CG_ARB_FREQ_F2 0x0c +#define MC_CG_ARB_FREQ_F3 0x0d + +#define SMC_RAM_END 0x20000 + +#define SCLK_MIN_DEEPSLEEP_FREQ 1350 + +static const struct si_cac_config_reg cac_weights_tahiti[] = +{ + { 0x0, 0x0000ffff, 0, 0xc, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0x101, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0xc, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x8fc, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x95, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x34e, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x1a1, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0xda, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x46, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x208, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0xe7, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x948, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x167, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x31, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0x18e, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg lcac_tahiti[] = +{ + { 0x143, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x146, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0x146, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x149, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0x149, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x14c, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0x14c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x9b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x9b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x9e, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x9e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x101, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x101, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x107, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x107, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10a, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x10a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x10d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x8f, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x8f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x92, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x92, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x95, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x95, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x152, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x152, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x155, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x155, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x158, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x158, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x113, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x113, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x116, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x116, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x119, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND }, + { 0x119, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x122, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x122, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x125, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x125, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x128, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x128, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x12b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x12b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } + +}; + +static const struct si_cac_config_reg cac_override_tahiti[] = +{ + { 0xFFFFFFFF } +}; + +static const struct si_powertune_data powertune_data_tahiti = +{ + ((1 << 16) | 27027), + 6, + 0, + 4, + 95, + { + 0UL, + 0UL, + 4521550UL, + 309631529UL, + -1270850L, + 4513710L, + 40 + }, + 595000000UL, + 12, + { + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }, + true +}; + +static const struct si_dte_data dte_data_tahiti = +{ + { 1159409, 0, 0, 0, 0 }, + { 777, 0, 0, 0, 0 }, + 2, + 54000, + 127000, + 25, + 2, + 10, + 13, + { 27, 31, 35, 39, 43, 47, 54, 61, 67, 74, 81, 88, 95, 0, 0, 0 }, + { 240888759, 221057860, 235370597, 162287531, 158510299, 131423027, 116673180, 103067515, 87941937, 76209048, 68209175, 64090048, 58301890, 0, 0, 0 }, + { 12024, 11189, 11451, 8411, 7939, 6666, 5681, 4905, 4241, 3720, 3354, 3122, 2890, 0, 0, 0 }, + 85, + false +}; + +static const struct si_dte_data dte_data_tahiti_le = +{ + { 0x1E8480, 0x7A1200, 0x2160EC0, 0x3938700, 0 }, + { 0x7D, 0x7D, 0x4E4, 0xB00, 0 }, + 0x5, + 0xAFC8, + 0x64, + 0x32, + 1, + 0, + 0x10, + { 0x78, 0x7C, 0x82, 0x88, 0x8E, 0x94, 0x9A, 0xA0, 0xA6, 0xAC, 0xB0, 0xB4, 0xB8, 0xBC, 0xC0, 0xC4 }, + { 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700 }, + { 0x2AF8, 0x2AF8, 0x29BB, 0x27F9, 0x2637, 0x2475, 0x22B3, 0x20F1, 0x1F2F, 0x1D6D, 0x1734, 0x1414, 0x10F4, 0xDD4, 0xAB4, 0x794 }, + 85, + true +}; + +static const struct si_dte_data dte_data_tahiti_pro = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x0, 0x0, 0x0, 0x0, 0x0 }, + 5, + 45000, + 100, + 0xA, + 1, + 0, + 0x10, + { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }, + { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 }, + { 0x7D0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +static const struct si_dte_data dte_data_new_zealand = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0 }, + { 0x29B, 0x3E9, 0x537, 0x7D2, 0 }, + 0x5, + 0xAFC8, + 0x69, + 0x32, + 1, + 0, + 0x10, + { 0x82, 0xA0, 0xB4, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE }, + { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 }, + { 0xDAC, 0x1388, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685 }, + 85, + true +}; + +static const struct si_dte_data dte_data_aruba_pro = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x0, 0x0, 0x0, 0x0, 0x0 }, + 5, + 45000, + 100, + 0xA, + 1, + 0, + 0x10, + { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }, + { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 }, + { 0x1000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +static const struct si_dte_data dte_data_malta = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x0, 0x0, 0x0, 0x0, 0x0 }, + 5, + 45000, + 100, + 0xA, + 1, + 0, + 0x10, + { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }, + { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 }, + { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +struct si_cac_config_reg cac_weights_pitcairn[] = +{ + { 0x0, 0x0000ffff, 0, 0x8a, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x24d, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x19, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0xc11, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0x7f3, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x403, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x367, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x4c9, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x45d, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0x36d, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x534, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x5da, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x880, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0x201, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0x9f, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x1f, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x5de, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x7b, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x13, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0xf9, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x66, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x13, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0x186, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg lcac_pitcairn[] = +{ + { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x143, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x9b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x9b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x107, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x107, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x113, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x113, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x152, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x152, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x8f, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x8f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x146, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x146, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x9e, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x9e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10a, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x10a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x116, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x116, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x155, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x155, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x92, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x92, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x149, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x149, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x101, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x101, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x10d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x119, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x119, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x158, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x158, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x95, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x95, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x14c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x14c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x122, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x122, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x125, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x125, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x128, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x128, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x12b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x12b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_override_pitcairn[] = +{ + { 0xFFFFFFFF } +}; + +static const struct si_powertune_data powertune_data_pitcairn = +{ + ((1 << 16) | 27027), + 5, + 0, + 6, + 100, + { + 51600000UL, + 1800000UL, + 7194395UL, + 309631529UL, + -1270850L, + 4513710L, + 100 + }, + 117830498UL, + 12, + { + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }, + true +}; + +static const struct si_dte_data dte_data_pitcairn = +{ + { 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0 }, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + 0, + false +}; + +static const struct si_dte_data dte_data_curacao_xt = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x0, 0x0, 0x0, 0x0, 0x0 }, + 5, + 45000, + 100, + 0xA, + 1, + 0, + 0x10, + { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }, + { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 }, + { 0x1D17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +static const struct si_dte_data dte_data_curacao_pro = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x0, 0x0, 0x0, 0x0, 0x0 }, + 5, + 45000, + 100, + 0xA, + 1, + 0, + 0x10, + { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }, + { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 }, + { 0x1D17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +static const struct si_dte_data dte_data_neptune_xt = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x0, 0x0, 0x0, 0x0, 0x0 }, + 5, + 45000, + 100, + 0xA, + 1, + 0, + 0x10, + { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }, + { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 }, + { 0x3A2F, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +static const struct si_cac_config_reg cac_weights_chelsea_pro[] = +{ + { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x2BD, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_weights_chelsea_xt[] = +{ + { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x30A, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_weights_heathrow[] = +{ + { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x362, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_weights_cape_verde_pro[] = +{ + { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x315, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_weights_cape_verde[] = +{ + { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x3BA, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg lcac_cape_verde[] = +{ + { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x143, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x9b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x9b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x107, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x107, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x113, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x113, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x152, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x152, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x8f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x8f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x146, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x146, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_override_cape_verde[] = +{ + { 0xFFFFFFFF } +}; + +static const struct si_powertune_data powertune_data_cape_verde = +{ + ((1 << 16) | 0x6993), + 5, + 0, + 7, + 105, + { + 0UL, + 0UL, + 7194395UL, + 309631529UL, + -1270850L, + 4513710L, + 100 + }, + 117830498UL, + 12, + { + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }, + true +}; + +static const struct si_dte_data dte_data_cape_verde = +{ + { 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0 }, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + 0, + false +}; + +static const struct si_dte_data dte_data_venus_xtx = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x71C, 0xAAB, 0xE39, 0x11C7, 0x0 }, + 5, + 55000, + 0x69, + 0xA, + 1, + 0, + 0x3, + { 0x96, 0xB4, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + { 0x895440, 0x3D0900, 0x989680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + { 0xD6D8, 0x88B8, 0x1555, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +static const struct si_dte_data dte_data_venus_xt = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0xBDA, 0x11C7, 0x17B4, 0x1DA1, 0x0 }, + 5, + 55000, + 0x69, + 0xA, + 1, + 0, + 0x3, + { 0x96, 0xB4, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + { 0x895440, 0x3D0900, 0x989680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + { 0xAFC8, 0x88B8, 0x238E, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +static const struct si_dte_data dte_data_venus_pro = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x11C7, 0x1AAB, 0x238E, 0x2C72, 0x0 }, + 5, + 55000, + 0x69, + 0xA, + 1, + 0, + 0x3, + { 0x96, 0xB4, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + { 0x895440, 0x3D0900, 0x989680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + { 0x88B8, 0x88B8, 0x3555, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +struct si_cac_config_reg cac_weights_oland[] = +{ + { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x3BA, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_weights_mars_pro[] = +{ + { 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_weights_mars_xt[] = +{ + { 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x60, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_weights_oland_pro[] = +{ + { 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x90, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_weights_oland_xt[] = +{ + { 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x120, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg lcac_oland[] = +{ + { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x143, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND }, + { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg lcac_mars_pro[] = +{ + { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND }, + { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x143, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND }, + { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_cac_config_reg cac_override_oland[] = +{ + { 0xFFFFFFFF } +}; + +static const struct si_powertune_data powertune_data_oland = +{ + ((1 << 16) | 0x6993), + 5, + 0, + 7, + 105, + { + 0UL, + 0UL, + 7194395UL, + 309631529UL, + -1270850L, + 4513710L, + 100 + }, + 117830498UL, + 12, + { + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }, + true +}; + +static const struct si_powertune_data powertune_data_mars_pro = +{ + ((1 << 16) | 0x6993), + 5, + 0, + 7, + 105, + { + 0UL, + 0UL, + 7194395UL, + 309631529UL, + -1270850L, + 4513710L, + 100 + }, + 117830498UL, + 12, + { + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }, + true +}; + +static const struct si_dte_data dte_data_oland = +{ + { 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0 }, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + 0, + false +}; + +static const struct si_dte_data dte_data_mars_pro = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x0, 0x0, 0x0, 0x0, 0x0 }, + 5, + 55000, + 105, + 0xA, + 1, + 0, + 0x10, + { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }, + { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 }, + { 0xF627, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + +static const struct si_dte_data dte_data_sun_xt = +{ + { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 }, + { 0x0, 0x0, 0x0, 0x0, 0x0 }, + 5, + 55000, + 105, + 0xA, + 1, + 0, + 0x10, + { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }, + { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 }, + { 0xD555, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, + 90, + true +}; + + +static const struct si_cac_config_reg cac_weights_hainan[] = +{ + { 0x0, 0x0000ffff, 0, 0x2d9, SISLANDS_CACCONFIG_CGIND }, + { 0x0, 0xffff0000, 16, 0x22b, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0x0000ffff, 0, 0x21c, SISLANDS_CACCONFIG_CGIND }, + { 0x1, 0xffff0000, 16, 0x1dc, SISLANDS_CACCONFIG_CGIND }, + { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0x0000ffff, 0, 0x24e, SISLANDS_CACCONFIG_CGIND }, + { 0x3, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x4, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0x0000ffff, 0, 0x35e, SISLANDS_CACCONFIG_CGIND }, + { 0x5, 0xffff0000, 16, 0x1143, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0x0000ffff, 0, 0xe17, SISLANDS_CACCONFIG_CGIND }, + { 0x6, 0xffff0000, 16, 0x441, SISLANDS_CACCONFIG_CGIND }, + { 0x18f, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0x0000ffff, 0, 0x28b, SISLANDS_CACCONFIG_CGIND }, + { 0x7, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x8, 0xffff0000, 16, 0xabe, SISLANDS_CACCONFIG_CGIND }, + { 0x9, 0x0000ffff, 0, 0xf11, SISLANDS_CACCONFIG_CGIND }, + { 0xa, 0x0000ffff, 0, 0x907, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0x0000ffff, 0, 0xb45, SISLANDS_CACCONFIG_CGIND }, + { 0xb, 0xffff0000, 16, 0xd1e, SISLANDS_CACCONFIG_CGIND }, + { 0xc, 0x0000ffff, 0, 0xa2c, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0x0000ffff, 0, 0x62, SISLANDS_CACCONFIG_CGIND }, + { 0xd, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0xe, 0x0000ffff, 0, 0x1f3, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0x0000ffff, 0, 0x42, SISLANDS_CACCONFIG_CGIND }, + { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x10, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0x0000ffff, 0, 0x709, SISLANDS_CACCONFIG_CGIND }, + { 0x11, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x12, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x13, 0xffff0000, 16, 0x3a, SISLANDS_CACCONFIG_CGIND }, + { 0x14, 0x0000ffff, 0, 0x357, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0x0000ffff, 0, 0x9f, SISLANDS_CACCONFIG_CGIND }, + { 0x15, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0x0000ffff, 0, 0x314, SISLANDS_CACCONFIG_CGIND }, + { 0x16, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x17, 0x0000ffff, 0, 0x6d, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND }, + { 0x6d, 0x0000ffff, 0, 0x1b9, SISLANDS_CACCONFIG_CGIND }, + { 0xFFFFFFFF } +}; + +static const struct si_powertune_data powertune_data_hainan = +{ + ((1 << 16) | 0x6993), + 5, + 0, + 9, + 105, + { + 0UL, + 0UL, + 7194395UL, + 309631529UL, + -1270850L, + 4513710L, + 100 + }, + 117830498UL, + 12, + { + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + }, + true +}; + +struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev); +struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev); +struct ni_power_info *ni_get_pi(struct radeon_device *rdev); +struct ni_ps *ni_get_ps(struct radeon_ps *rps); + +extern int si_mc_load_microcode(struct radeon_device *rdev); +extern void vce_v1_0_enable_mgcg(struct radeon_device *rdev, bool enable); + +static int si_populate_voltage_value(struct radeon_device *rdev, + const struct atom_voltage_table *table, + u16 value, SISLANDS_SMC_VOLTAGE_VALUE *voltage); +static int si_get_std_voltage_value(struct radeon_device *rdev, + SISLANDS_SMC_VOLTAGE_VALUE *voltage, + u16 *std_voltage); +static int si_write_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 value); +static int si_convert_power_level_to_smc(struct radeon_device *rdev, + struct rv7xx_pl *pl, + SISLANDS_SMC_HW_PERFORMANCE_LEVEL *level); +static int si_calculate_sclk_params(struct radeon_device *rdev, + u32 engine_clock, + SISLANDS_SMC_SCLK_VALUE *sclk); + +static void si_thermal_start_smc_fan_control(struct radeon_device *rdev); +static void si_fan_ctrl_set_default_mode(struct radeon_device *rdev); + +static struct si_power_info *si_get_pi(struct radeon_device *rdev) +{ + struct si_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +static void si_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff, + u16 v, s32 t, u32 ileakage, u32 *leakage) +{ + s64 kt, kv, leakage_w, i_leakage, vddc; + s64 temperature, t_slope, t_intercept, av, bv, t_ref; + s64 tmp; + + i_leakage = div64_s64(drm_int2fixp(ileakage), 100); + vddc = div64_s64(drm_int2fixp(v), 1000); + temperature = div64_s64(drm_int2fixp(t), 1000); + + t_slope = div64_s64(drm_int2fixp(coeff->t_slope), 100000000); + t_intercept = div64_s64(drm_int2fixp(coeff->t_intercept), 100000000); + av = div64_s64(drm_int2fixp(coeff->av), 100000000); + bv = div64_s64(drm_int2fixp(coeff->bv), 100000000); + t_ref = drm_int2fixp(coeff->t_ref); + + tmp = drm_fixp_mul(t_slope, vddc) + t_intercept; + kt = drm_fixp_exp(drm_fixp_mul(tmp, temperature)); + kt = drm_fixp_div(kt, drm_fixp_exp(drm_fixp_mul(tmp, t_ref))); + kv = drm_fixp_mul(av, drm_fixp_exp(drm_fixp_mul(bv, vddc))); + + leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc); + + *leakage = drm_fixp2int(leakage_w * 1000); +} + +static void si_calculate_leakage_for_v_and_t(struct radeon_device *rdev, + const struct ni_leakage_coeffients *coeff, + u16 v, + s32 t, + u32 i_leakage, + u32 *leakage) +{ + si_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage); +} + +static void si_calculate_leakage_for_v_formula(const struct ni_leakage_coeffients *coeff, + const u32 fixed_kt, u16 v, + u32 ileakage, u32 *leakage) +{ + s64 kt, kv, leakage_w, i_leakage, vddc; + + i_leakage = div64_s64(drm_int2fixp(ileakage), 100); + vddc = div64_s64(drm_int2fixp(v), 1000); + + kt = div64_s64(drm_int2fixp(fixed_kt), 100000000); + kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 100000000), + drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 100000000), vddc))); + + leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc); + + *leakage = drm_fixp2int(leakage_w * 1000); +} + +static void si_calculate_leakage_for_v(struct radeon_device *rdev, + const struct ni_leakage_coeffients *coeff, + const u32 fixed_kt, + u16 v, + u32 i_leakage, + u32 *leakage) +{ + si_calculate_leakage_for_v_formula(coeff, fixed_kt, v, i_leakage, leakage); +} + + +static void si_update_dte_from_pl2(struct radeon_device *rdev, + struct si_dte_data *dte_data) +{ + u32 p_limit1 = rdev->pm.dpm.tdp_limit; + u32 p_limit2 = rdev->pm.dpm.near_tdp_limit; + u32 k = dte_data->k; + u32 t_max = dte_data->max_t; + u32 t_split[5] = { 10, 15, 20, 25, 30 }; + u32 t_0 = dte_data->t0; + u32 i; + + if (p_limit2 != 0 && p_limit2 <= p_limit1) { + dte_data->tdep_count = 3; + + for (i = 0; i < k; i++) { + dte_data->r[i] = + (t_split[i] * (t_max - t_0/(u32)1000) * (1 << 14)) / + (p_limit2 * (u32)100); + } + + dte_data->tdep_r[1] = dte_data->r[4] * 2; + + for (i = 2; i < SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE; i++) { + dte_data->tdep_r[i] = dte_data->r[4]; + } + } else { + DRM_ERROR("Invalid PL2! DTE will not be updated.\n"); + } +} + +static void si_initialize_powertune_defaults(struct radeon_device *rdev) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + bool update_dte_from_pl2 = false; + + if (rdev->family == CHIP_TAHITI) { + si_pi->cac_weights = cac_weights_tahiti; + si_pi->lcac_config = lcac_tahiti; + si_pi->cac_override = cac_override_tahiti; + si_pi->powertune_data = &powertune_data_tahiti; + si_pi->dte_data = dte_data_tahiti; + + switch (rdev->pdev->device) { + case 0x6798: + si_pi->dte_data.enable_dte_by_default = true; + break; + case 0x6799: + si_pi->dte_data = dte_data_new_zealand; + break; + case 0x6790: + case 0x6791: + case 0x6792: + case 0x679E: + si_pi->dte_data = dte_data_aruba_pro; + update_dte_from_pl2 = true; + break; + case 0x679B: + si_pi->dte_data = dte_data_malta; + update_dte_from_pl2 = true; + break; + case 0x679A: + si_pi->dte_data = dte_data_tahiti_pro; + update_dte_from_pl2 = true; + break; + default: + if (si_pi->dte_data.enable_dte_by_default == true) + DRM_ERROR("DTE is not enabled!\n"); + break; + } + } else if (rdev->family == CHIP_PITCAIRN) { + switch (rdev->pdev->device) { + case 0x6810: + case 0x6818: + si_pi->cac_weights = cac_weights_pitcairn; + si_pi->lcac_config = lcac_pitcairn; + si_pi->cac_override = cac_override_pitcairn; + si_pi->powertune_data = &powertune_data_pitcairn; + si_pi->dte_data = dte_data_curacao_xt; + update_dte_from_pl2 = true; + break; + case 0x6819: + case 0x6811: + si_pi->cac_weights = cac_weights_pitcairn; + si_pi->lcac_config = lcac_pitcairn; + si_pi->cac_override = cac_override_pitcairn; + si_pi->powertune_data = &powertune_data_pitcairn; + si_pi->dte_data = dte_data_curacao_pro; + update_dte_from_pl2 = true; + break; + case 0x6800: + case 0x6806: + si_pi->cac_weights = cac_weights_pitcairn; + si_pi->lcac_config = lcac_pitcairn; + si_pi->cac_override = cac_override_pitcairn; + si_pi->powertune_data = &powertune_data_pitcairn; + si_pi->dte_data = dte_data_neptune_xt; + update_dte_from_pl2 = true; + break; + default: + si_pi->cac_weights = cac_weights_pitcairn; + si_pi->lcac_config = lcac_pitcairn; + si_pi->cac_override = cac_override_pitcairn; + si_pi->powertune_data = &powertune_data_pitcairn; + si_pi->dte_data = dte_data_pitcairn; + break; + } + } else if (rdev->family == CHIP_VERDE) { + si_pi->lcac_config = lcac_cape_verde; + si_pi->cac_override = cac_override_cape_verde; + si_pi->powertune_data = &powertune_data_cape_verde; + + switch (rdev->pdev->device) { + case 0x683B: + case 0x683F: + case 0x6829: + case 0x6835: + si_pi->cac_weights = cac_weights_cape_verde_pro; + si_pi->dte_data = dte_data_cape_verde; + break; + case 0x682C: + si_pi->cac_weights = cac_weights_cape_verde_pro; + si_pi->dte_data = dte_data_sun_xt; + break; + case 0x6825: + case 0x6827: + si_pi->cac_weights = cac_weights_heathrow; + si_pi->dte_data = dte_data_cape_verde; + break; + case 0x6824: + case 0x682D: + si_pi->cac_weights = cac_weights_chelsea_xt; + si_pi->dte_data = dte_data_cape_verde; + break; + case 0x682F: + si_pi->cac_weights = cac_weights_chelsea_pro; + si_pi->dte_data = dte_data_cape_verde; + break; + case 0x6820: + si_pi->cac_weights = cac_weights_heathrow; + si_pi->dte_data = dte_data_venus_xtx; + break; + case 0x6821: + si_pi->cac_weights = cac_weights_heathrow; + si_pi->dte_data = dte_data_venus_xt; + break; + case 0x6823: + case 0x682B: + case 0x6822: + case 0x682A: + si_pi->cac_weights = cac_weights_chelsea_pro; + si_pi->dte_data = dte_data_venus_pro; + break; + default: + si_pi->cac_weights = cac_weights_cape_verde; + si_pi->dte_data = dte_data_cape_verde; + break; + } + } else if (rdev->family == CHIP_OLAND) { + switch (rdev->pdev->device) { + case 0x6601: + case 0x6621: + case 0x6603: + case 0x6605: + si_pi->cac_weights = cac_weights_mars_pro; + si_pi->lcac_config = lcac_mars_pro; + si_pi->cac_override = cac_override_oland; + si_pi->powertune_data = &powertune_data_mars_pro; + si_pi->dte_data = dte_data_mars_pro; + update_dte_from_pl2 = true; + break; + case 0x6600: + case 0x6606: + case 0x6620: + case 0x6604: + si_pi->cac_weights = cac_weights_mars_xt; + si_pi->lcac_config = lcac_mars_pro; + si_pi->cac_override = cac_override_oland; + si_pi->powertune_data = &powertune_data_mars_pro; + si_pi->dte_data = dte_data_mars_pro; + update_dte_from_pl2 = true; + break; + case 0x6611: + case 0x6613: + case 0x6608: + si_pi->cac_weights = cac_weights_oland_pro; + si_pi->lcac_config = lcac_mars_pro; + si_pi->cac_override = cac_override_oland; + si_pi->powertune_data = &powertune_data_mars_pro; + si_pi->dte_data = dte_data_mars_pro; + update_dte_from_pl2 = true; + break; + case 0x6610: + si_pi->cac_weights = cac_weights_oland_xt; + si_pi->lcac_config = lcac_mars_pro; + si_pi->cac_override = cac_override_oland; + si_pi->powertune_data = &powertune_data_mars_pro; + si_pi->dte_data = dte_data_mars_pro; + update_dte_from_pl2 = true; + break; + default: + si_pi->cac_weights = cac_weights_oland; + si_pi->lcac_config = lcac_oland; + si_pi->cac_override = cac_override_oland; + si_pi->powertune_data = &powertune_data_oland; + si_pi->dte_data = dte_data_oland; + break; + } + } else if (rdev->family == CHIP_HAINAN) { + si_pi->cac_weights = cac_weights_hainan; + si_pi->lcac_config = lcac_oland; + si_pi->cac_override = cac_override_oland; + si_pi->powertune_data = &powertune_data_hainan; + si_pi->dte_data = dte_data_sun_xt; + update_dte_from_pl2 = true; + } else { + DRM_ERROR("Unknown SI asic revision, failed to initialize PowerTune!\n"); + return; + } + + ni_pi->enable_power_containment = false; + ni_pi->enable_cac = false; + ni_pi->enable_sq_ramping = false; + si_pi->enable_dte = false; + + if (si_pi->powertune_data->enable_powertune_by_default) { + ni_pi->enable_power_containment= true; + ni_pi->enable_cac = true; + if (si_pi->dte_data.enable_dte_by_default) { + si_pi->enable_dte = true; + if (update_dte_from_pl2) + si_update_dte_from_pl2(rdev, &si_pi->dte_data); + + } + ni_pi->enable_sq_ramping = true; + } + + ni_pi->driver_calculate_cac_leakage = true; + ni_pi->cac_configuration_required = true; + + if (ni_pi->cac_configuration_required) { + ni_pi->support_cac_long_term_average = true; + si_pi->dyn_powertune_data.l2_lta_window_size = + si_pi->powertune_data->l2_lta_window_size_default; + si_pi->dyn_powertune_data.lts_truncate = + si_pi->powertune_data->lts_truncate_default; + } else { + ni_pi->support_cac_long_term_average = false; + si_pi->dyn_powertune_data.l2_lta_window_size = 0; + si_pi->dyn_powertune_data.lts_truncate = 0; + } + + si_pi->dyn_powertune_data.disable_uvd_powertune = false; +} + +static u32 si_get_smc_power_scaling_factor(struct radeon_device *rdev) +{ + return 1; +} + +static u32 si_calculate_cac_wintime(struct radeon_device *rdev) +{ + u32 xclk; + u32 wintime; + u32 cac_window; + u32 cac_window_size; + + xclk = radeon_get_xclk(rdev); + + if (xclk == 0) + return 0; + + cac_window = RREG32(CG_CAC_CTRL) & CAC_WINDOW_MASK; + cac_window_size = ((cac_window & 0xFFFF0000) >> 16) * (cac_window & 0x0000FFFF); + + wintime = (cac_window_size * 100) / xclk; + + return wintime; +} + +static u32 si_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor) +{ + return power_in_watts; +} + +static int si_calculate_adjusted_tdp_limits(struct radeon_device *rdev, + bool adjust_polarity, + u32 tdp_adjustment, + u32 *tdp_limit, + u32 *near_tdp_limit) +{ + u32 adjustment_delta, max_tdp_limit; + + if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit) + return -EINVAL; + + max_tdp_limit = ((100 + 100) * rdev->pm.dpm.tdp_limit) / 100; + + if (adjust_polarity) { + *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100; + *near_tdp_limit = rdev->pm.dpm.near_tdp_limit_adjusted + (*tdp_limit - rdev->pm.dpm.tdp_limit); + } else { + *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100; + adjustment_delta = rdev->pm.dpm.tdp_limit - *tdp_limit; + if (adjustment_delta < rdev->pm.dpm.near_tdp_limit_adjusted) + *near_tdp_limit = rdev->pm.dpm.near_tdp_limit_adjusted - adjustment_delta; + else + *near_tdp_limit = 0; + } + + if ((*tdp_limit <= 0) || (*tdp_limit > max_tdp_limit)) + return -EINVAL; + if ((*near_tdp_limit <= 0) || (*near_tdp_limit > *tdp_limit)) + return -EINVAL; + + return 0; +} + +static int si_populate_smc_tdp_limits(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + + if (ni_pi->enable_power_containment) { + SISLANDS_SMC_STATETABLE *smc_table = &si_pi->smc_statetable; + PP_SIslands_PAPMParameters *papm_parm; + struct radeon_ppm_table *ppm = rdev->pm.dpm.dyn_state.ppm_table; + u32 scaling_factor = si_get_smc_power_scaling_factor(rdev); + u32 tdp_limit; + u32 near_tdp_limit; + int ret; + + if (scaling_factor == 0) + return -EINVAL; + + memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE)); + + ret = si_calculate_adjusted_tdp_limits(rdev, + false, /* ??? */ + rdev->pm.dpm.tdp_adjustment, + &tdp_limit, + &near_tdp_limit); + if (ret) + return ret; + + smc_table->dpm2Params.TDPLimit = + cpu_to_be32(si_scale_power_for_smc(tdp_limit, scaling_factor) * 1000); + smc_table->dpm2Params.NearTDPLimit = + cpu_to_be32(si_scale_power_for_smc(near_tdp_limit, scaling_factor) * 1000); + smc_table->dpm2Params.SafePowerLimit = + cpu_to_be32(si_scale_power_for_smc((near_tdp_limit * SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100, scaling_factor) * 1000); + + ret = si_copy_bytes_to_smc(rdev, + (si_pi->state_table_start + offsetof(SISLANDS_SMC_STATETABLE, dpm2Params) + + offsetof(PP_SIslands_DPM2Parameters, TDPLimit)), + (u8 *)(&(smc_table->dpm2Params.TDPLimit)), + sizeof(u32) * 3, + si_pi->sram_end); + if (ret) + return ret; + + if (si_pi->enable_ppm) { + papm_parm = &si_pi->papm_parm; + memset(papm_parm, 0, sizeof(PP_SIslands_PAPMParameters)); + papm_parm->NearTDPLimitTherm = cpu_to_be32(ppm->dgpu_tdp); + papm_parm->dGPU_T_Limit = cpu_to_be32(ppm->tj_max); + papm_parm->dGPU_T_Warning = cpu_to_be32(95); + papm_parm->dGPU_T_Hysteresis = cpu_to_be32(5); + papm_parm->PlatformPowerLimit = 0xffffffff; + papm_parm->NearTDPLimitPAPM = 0xffffffff; + + ret = si_copy_bytes_to_smc(rdev, si_pi->papm_cfg_table_start, + (u8 *)papm_parm, + sizeof(PP_SIslands_PAPMParameters), + si_pi->sram_end); + if (ret) + return ret; + } + } + return 0; +} + +static int si_populate_smc_tdp_limits_2(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + + if (ni_pi->enable_power_containment) { + SISLANDS_SMC_STATETABLE *smc_table = &si_pi->smc_statetable; + u32 scaling_factor = si_get_smc_power_scaling_factor(rdev); + int ret; + + memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE)); + + smc_table->dpm2Params.NearTDPLimit = + cpu_to_be32(si_scale_power_for_smc(rdev->pm.dpm.near_tdp_limit_adjusted, scaling_factor) * 1000); + smc_table->dpm2Params.SafePowerLimit = + cpu_to_be32(si_scale_power_for_smc((rdev->pm.dpm.near_tdp_limit_adjusted * SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100, scaling_factor) * 1000); + + ret = si_copy_bytes_to_smc(rdev, + (si_pi->state_table_start + + offsetof(SISLANDS_SMC_STATETABLE, dpm2Params) + + offsetof(PP_SIslands_DPM2Parameters, NearTDPLimit)), + (u8 *)(&(smc_table->dpm2Params.NearTDPLimit)), + sizeof(u32) * 2, + si_pi->sram_end); + if (ret) + return ret; + } + + return 0; +} + +static u16 si_calculate_power_efficiency_ratio(struct radeon_device *rdev, + const u16 prev_std_vddc, + const u16 curr_std_vddc) +{ + u64 margin = (u64)SISLANDS_DPM2_PWREFFICIENCYRATIO_MARGIN; + u64 prev_vddc = (u64)prev_std_vddc; + u64 curr_vddc = (u64)curr_std_vddc; + u64 pwr_efficiency_ratio, n, d; + + if ((prev_vddc == 0) || (curr_vddc == 0)) + return 0; + + n = div64_u64((u64)1024 * curr_vddc * curr_vddc * ((u64)1000 + margin), (u64)1000); + d = prev_vddc * prev_vddc; + pwr_efficiency_ratio = div64_u64(n, d); + + if (pwr_efficiency_ratio > (u64)0xFFFF) + return 0; + + return (u16)pwr_efficiency_ratio; +} + +static bool si_should_disable_uvd_powertune(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + + if (si_pi->dyn_powertune_data.disable_uvd_powertune && + radeon_state->vclk && radeon_state->dclk) + return true; + + return false; +} + +static int si_populate_power_containment_values(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + SISLANDS_SMC_SWSTATE *smc_state) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + SISLANDS_SMC_VOLTAGE_VALUE vddc; + u32 prev_sclk; + u32 max_sclk; + u32 min_sclk; + u16 prev_std_vddc; + u16 curr_std_vddc; + int i; + u16 pwr_efficiency_ratio; + u8 max_ps_percent; + bool disable_uvd_power_tune; + int ret; + + if (ni_pi->enable_power_containment == false) + return 0; + + if (state->performance_level_count == 0) + return -EINVAL; + + if (smc_state->levelCount != state->performance_level_count) + return -EINVAL; + + disable_uvd_power_tune = si_should_disable_uvd_powertune(rdev, radeon_state); + + smc_state->levels[0].dpm2.MaxPS = 0; + smc_state->levels[0].dpm2.NearTDPDec = 0; + smc_state->levels[0].dpm2.AboveSafeInc = 0; + smc_state->levels[0].dpm2.BelowSafeInc = 0; + smc_state->levels[0].dpm2.PwrEfficiencyRatio = 0; + + for (i = 1; i < state->performance_level_count; i++) { + prev_sclk = state->performance_levels[i-1].sclk; + max_sclk = state->performance_levels[i].sclk; + if (i == 1) + max_ps_percent = SISLANDS_DPM2_MAXPS_PERCENT_M; + else + max_ps_percent = SISLANDS_DPM2_MAXPS_PERCENT_H; + + if (prev_sclk > max_sclk) + return -EINVAL; + + if ((max_ps_percent == 0) || + (prev_sclk == max_sclk) || + disable_uvd_power_tune) { + min_sclk = max_sclk; + } else if (i == 1) { + min_sclk = prev_sclk; + } else { + min_sclk = (prev_sclk * (u32)max_ps_percent) / 100; + } + + if (min_sclk < state->performance_levels[0].sclk) + min_sclk = state->performance_levels[0].sclk; + + if (min_sclk == 0) + return -EINVAL; + + ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table, + state->performance_levels[i-1].vddc, &vddc); + if (ret) + return ret; + + ret = si_get_std_voltage_value(rdev, &vddc, &prev_std_vddc); + if (ret) + return ret; + + ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table, + state->performance_levels[i].vddc, &vddc); + if (ret) + return ret; + + ret = si_get_std_voltage_value(rdev, &vddc, &curr_std_vddc); + if (ret) + return ret; + + pwr_efficiency_ratio = si_calculate_power_efficiency_ratio(rdev, + prev_std_vddc, curr_std_vddc); + + smc_state->levels[i].dpm2.MaxPS = (u8)((SISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk); + smc_state->levels[i].dpm2.NearTDPDec = SISLANDS_DPM2_NEAR_TDP_DEC; + smc_state->levels[i].dpm2.AboveSafeInc = SISLANDS_DPM2_ABOVE_SAFE_INC; + smc_state->levels[i].dpm2.BelowSafeInc = SISLANDS_DPM2_BELOW_SAFE_INC; + smc_state->levels[i].dpm2.PwrEfficiencyRatio = cpu_to_be16(pwr_efficiency_ratio); + } + + return 0; +} + +static int si_populate_sq_ramping_values(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + SISLANDS_SMC_SWSTATE *smc_state) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + u32 sq_power_throttle, sq_power_throttle2; + bool enable_sq_ramping = ni_pi->enable_sq_ramping; + int i; + + if (state->performance_level_count == 0) + return -EINVAL; + + if (smc_state->levelCount != state->performance_level_count) + return -EINVAL; + + if (rdev->pm.dpm.sq_ramping_threshold == 0) + return -EINVAL; + + if (SISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT)) + enable_sq_ramping = false; + + if (SISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT)) + enable_sq_ramping = false; + + if (SISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT)) + enable_sq_ramping = false; + + if (SISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT)) + enable_sq_ramping = false; + + if (SISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT)) + enable_sq_ramping = false; + + for (i = 0; i < state->performance_level_count; i++) { + sq_power_throttle = 0; + sq_power_throttle2 = 0; + + if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) && + enable_sq_ramping) { + sq_power_throttle |= MAX_POWER(SISLANDS_DPM2_SQ_RAMP_MAX_POWER); + sq_power_throttle |= MIN_POWER(SISLANDS_DPM2_SQ_RAMP_MIN_POWER); + sq_power_throttle2 |= MAX_POWER_DELTA(SISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA); + sq_power_throttle2 |= STI_SIZE(SISLANDS_DPM2_SQ_RAMP_STI_SIZE); + sq_power_throttle2 |= LTI_RATIO(SISLANDS_DPM2_SQ_RAMP_LTI_RATIO); + } else { + sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK; + sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK; + } + + smc_state->levels[i].SQPowerThrottle = cpu_to_be32(sq_power_throttle); + smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2); + } + + return 0; +} + +static int si_enable_power_containment(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + bool enable) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + PPSMC_Result smc_result; + int ret = 0; + + if (ni_pi->enable_power_containment) { + if (enable) { + if (!si_should_disable_uvd_powertune(rdev, radeon_new_state)) { + smc_result = si_send_msg_to_smc(rdev, PPSMC_TDPClampingActive); + if (smc_result != PPSMC_Result_OK) { + ret = -EINVAL; + ni_pi->pc_enabled = false; + } else { + ni_pi->pc_enabled = true; + } + } + } else { + smc_result = si_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive); + if (smc_result != PPSMC_Result_OK) + ret = -EINVAL; + ni_pi->pc_enabled = false; + } + } + + return ret; +} + +static int si_initialize_smc_dte_tables(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + int ret = 0; + struct si_dte_data *dte_data = &si_pi->dte_data; + Smc_SIslands_DTE_Configuration *dte_tables = NULL; + u32 table_size; + u8 tdep_count; + u32 i; + + if (dte_data == NULL) + si_pi->enable_dte = false; + + if (si_pi->enable_dte == false) + return 0; + + if (dte_data->k <= 0) + return -EINVAL; + + dte_tables = kzalloc(sizeof(Smc_SIslands_DTE_Configuration), GFP_KERNEL); + if (dte_tables == NULL) { + si_pi->enable_dte = false; + return -ENOMEM; + } + + table_size = dte_data->k; + + if (table_size > SMC_SISLANDS_DTE_MAX_FILTER_STAGES) + table_size = SMC_SISLANDS_DTE_MAX_FILTER_STAGES; + + tdep_count = dte_data->tdep_count; + if (tdep_count > SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE) + tdep_count = SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE; + + dte_tables->K = cpu_to_be32(table_size); + dte_tables->T0 = cpu_to_be32(dte_data->t0); + dte_tables->MaxT = cpu_to_be32(dte_data->max_t); + dte_tables->WindowSize = dte_data->window_size; + dte_tables->temp_select = dte_data->temp_select; + dte_tables->DTE_mode = dte_data->dte_mode; + dte_tables->Tthreshold = cpu_to_be32(dte_data->t_threshold); + + if (tdep_count > 0) + table_size--; + + for (i = 0; i < table_size; i++) { + dte_tables->tau[i] = cpu_to_be32(dte_data->tau[i]); + dte_tables->R[i] = cpu_to_be32(dte_data->r[i]); + } + + dte_tables->Tdep_count = tdep_count; + + for (i = 0; i < (u32)tdep_count; i++) { + dte_tables->T_limits[i] = dte_data->t_limits[i]; + dte_tables->Tdep_tau[i] = cpu_to_be32(dte_data->tdep_tau[i]); + dte_tables->Tdep_R[i] = cpu_to_be32(dte_data->tdep_r[i]); + } + + ret = si_copy_bytes_to_smc(rdev, si_pi->dte_table_start, (u8 *)dte_tables, + sizeof(Smc_SIslands_DTE_Configuration), si_pi->sram_end); + kfree(dte_tables); + + return ret; +} + +static int si_get_cac_std_voltage_max_min(struct radeon_device *rdev, + u16 *max, u16 *min) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + struct radeon_cac_leakage_table *table = + &rdev->pm.dpm.dyn_state.cac_leakage_table; + u32 i; + u32 v0_loadline; + + + if (table == NULL) + return -EINVAL; + + *max = 0; + *min = 0xFFFF; + + for (i = 0; i < table->count; i++) { + if (table->entries[i].vddc > *max) + *max = table->entries[i].vddc; + if (table->entries[i].vddc < *min) + *min = table->entries[i].vddc; + } + + if (si_pi->powertune_data->lkge_lut_v0_percent > 100) + return -EINVAL; + + v0_loadline = (*min) * (100 - si_pi->powertune_data->lkge_lut_v0_percent) / 100; + + if (v0_loadline > 0xFFFFUL) + return -EINVAL; + + *min = (u16)v0_loadline; + + if ((*min > *max) || (*max == 0) || (*min == 0)) + return -EINVAL; + + return 0; +} + +static u16 si_get_cac_std_voltage_step(u16 max, u16 min) +{ + return ((max - min) + (SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES - 1)) / + SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; +} + +static int si_init_dte_leakage_table(struct radeon_device *rdev, + PP_SIslands_CacConfig *cac_tables, + u16 vddc_max, u16 vddc_min, u16 vddc_step, + u16 t0, u16 t_step) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 leakage; + unsigned int i, j; + s32 t; + u32 smc_leakage; + u32 scaling_factor; + u16 voltage; + + scaling_factor = si_get_smc_power_scaling_factor(rdev); + + for (i = 0; i < SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES ; i++) { + t = (1000 * (i * t_step + t0)); + + for (j = 0; j < SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) { + voltage = vddc_max - (vddc_step * j); + + si_calculate_leakage_for_v_and_t(rdev, + &si_pi->powertune_data->leakage_coefficients, + voltage, + t, + si_pi->dyn_powertune_data.cac_leakage, + &leakage); + + smc_leakage = si_scale_power_for_smc(leakage, scaling_factor) / 4; + + if (smc_leakage > 0xFFFF) + smc_leakage = 0xFFFF; + + cac_tables->cac_lkge_lut[i][SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES-1-j] = + cpu_to_be16((u16)smc_leakage); + } + } + return 0; +} + +static int si_init_simplified_leakage_table(struct radeon_device *rdev, + PP_SIslands_CacConfig *cac_tables, + u16 vddc_max, u16 vddc_min, u16 vddc_step) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 leakage; + unsigned int i, j; + u32 smc_leakage; + u32 scaling_factor; + u16 voltage; + + scaling_factor = si_get_smc_power_scaling_factor(rdev); + + for (j = 0; j < SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) { + voltage = vddc_max - (vddc_step * j); + + si_calculate_leakage_for_v(rdev, + &si_pi->powertune_data->leakage_coefficients, + si_pi->powertune_data->fixed_kt, + voltage, + si_pi->dyn_powertune_data.cac_leakage, + &leakage); + + smc_leakage = si_scale_power_for_smc(leakage, scaling_factor) / 4; + + if (smc_leakage > 0xFFFF) + smc_leakage = 0xFFFF; + + for (i = 0; i < SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES ; i++) + cac_tables->cac_lkge_lut[i][SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES-1-j] = + cpu_to_be16((u16)smc_leakage); + } + return 0; +} + +static int si_initialize_smc_cac_tables(struct radeon_device *rdev) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + PP_SIslands_CacConfig *cac_tables = NULL; + u16 vddc_max, vddc_min, vddc_step; + u16 t0, t_step; + u32 load_line_slope, reg; + int ret = 0; + u32 ticks_per_us = radeon_get_xclk(rdev) / 100; + + if (ni_pi->enable_cac == false) + return 0; + + cac_tables = kzalloc(sizeof(PP_SIslands_CacConfig), GFP_KERNEL); + if (!cac_tables) + return -ENOMEM; + + reg = RREG32(CG_CAC_CTRL) & ~CAC_WINDOW_MASK; + reg |= CAC_WINDOW(si_pi->powertune_data->cac_window); + WREG32(CG_CAC_CTRL, reg); + + si_pi->dyn_powertune_data.cac_leakage = rdev->pm.dpm.cac_leakage; + si_pi->dyn_powertune_data.dc_pwr_value = + si_pi->powertune_data->dc_cac[NISLANDS_DCCAC_LEVEL_0]; + si_pi->dyn_powertune_data.wintime = si_calculate_cac_wintime(rdev); + si_pi->dyn_powertune_data.shift_n = si_pi->powertune_data->shift_n_default; + + si_pi->dyn_powertune_data.leakage_minimum_temperature = 80 * 1000; + + ret = si_get_cac_std_voltage_max_min(rdev, &vddc_max, &vddc_min); + if (ret) + goto done_free; + + vddc_step = si_get_cac_std_voltage_step(vddc_max, vddc_min); + vddc_min = vddc_max - (vddc_step * (SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES - 1)); + t_step = 4; + t0 = 60; + + if (si_pi->enable_dte || ni_pi->driver_calculate_cac_leakage) + ret = si_init_dte_leakage_table(rdev, cac_tables, + vddc_max, vddc_min, vddc_step, + t0, t_step); + else + ret = si_init_simplified_leakage_table(rdev, cac_tables, + vddc_max, vddc_min, vddc_step); + if (ret) + goto done_free; + + load_line_slope = ((u32)rdev->pm.dpm.load_line_slope << SMC_SISLANDS_SCALE_R) / 100; + + cac_tables->l2numWin_TDP = cpu_to_be32(si_pi->dyn_powertune_data.l2_lta_window_size); + cac_tables->lts_truncate_n = si_pi->dyn_powertune_data.lts_truncate; + cac_tables->SHIFT_N = si_pi->dyn_powertune_data.shift_n; + cac_tables->lkge_lut_V0 = cpu_to_be32((u32)vddc_min); + cac_tables->lkge_lut_Vstep = cpu_to_be32((u32)vddc_step); + cac_tables->R_LL = cpu_to_be32(load_line_slope); + cac_tables->WinTime = cpu_to_be32(si_pi->dyn_powertune_data.wintime); + cac_tables->calculation_repeats = cpu_to_be32(2); + cac_tables->dc_cac = cpu_to_be32(0); + cac_tables->log2_PG_LKG_SCALE = 12; + cac_tables->cac_temp = si_pi->powertune_data->operating_temp; + cac_tables->lkge_lut_T0 = cpu_to_be32((u32)t0); + cac_tables->lkge_lut_Tstep = cpu_to_be32((u32)t_step); + + ret = si_copy_bytes_to_smc(rdev, si_pi->cac_table_start, (u8 *)cac_tables, + sizeof(PP_SIslands_CacConfig), si_pi->sram_end); + + if (ret) + goto done_free; + + ret = si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_ticks_per_us, ticks_per_us); + +done_free: + if (ret) { + ni_pi->enable_cac = false; + ni_pi->enable_power_containment = false; + } + + kfree(cac_tables); + + return 0; +} + +static int si_program_cac_config_registers(struct radeon_device *rdev, + const struct si_cac_config_reg *cac_config_regs) +{ + const struct si_cac_config_reg *config_regs = cac_config_regs; + u32 data = 0, offset; + + if (!config_regs) + return -EINVAL; + + while (config_regs->offset != 0xFFFFFFFF) { + switch (config_regs->type) { + case SISLANDS_CACCONFIG_CGIND: + offset = SMC_CG_IND_START + config_regs->offset; + if (offset < SMC_CG_IND_END) + data = RREG32_SMC(offset); + break; + default: + data = RREG32(config_regs->offset << 2); + break; + } + + data &= ~config_regs->mask; + data |= ((config_regs->value << config_regs->shift) & config_regs->mask); + + switch (config_regs->type) { + case SISLANDS_CACCONFIG_CGIND: + offset = SMC_CG_IND_START + config_regs->offset; + if (offset < SMC_CG_IND_END) + WREG32_SMC(offset, data); + break; + default: + WREG32(config_regs->offset << 2, data); + break; + } + config_regs++; + } + return 0; +} + +static int si_initialize_hardware_cac_manager(struct radeon_device *rdev) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + int ret; + + if ((ni_pi->enable_cac == false) || + (ni_pi->cac_configuration_required == false)) + return 0; + + ret = si_program_cac_config_registers(rdev, si_pi->lcac_config); + if (ret) + return ret; + ret = si_program_cac_config_registers(rdev, si_pi->cac_override); + if (ret) + return ret; + ret = si_program_cac_config_registers(rdev, si_pi->cac_weights); + if (ret) + return ret; + + return 0; +} + +static int si_enable_smc_cac(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + bool enable) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + PPSMC_Result smc_result; + int ret = 0; + + if (ni_pi->enable_cac) { + if (enable) { + if (!si_should_disable_uvd_powertune(rdev, radeon_new_state)) { + if (ni_pi->support_cac_long_term_average) { + smc_result = si_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable); + if (smc_result != PPSMC_Result_OK) + ni_pi->support_cac_long_term_average = false; + } + + smc_result = si_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac); + if (smc_result != PPSMC_Result_OK) { + ret = -EINVAL; + ni_pi->cac_enabled = false; + } else { + ni_pi->cac_enabled = true; + } + + if (si_pi->enable_dte) { + smc_result = si_send_msg_to_smc(rdev, PPSMC_MSG_EnableDTE); + if (smc_result != PPSMC_Result_OK) + ret = -EINVAL; + } + } + } else if (ni_pi->cac_enabled) { + if (si_pi->enable_dte) + smc_result = si_send_msg_to_smc(rdev, PPSMC_MSG_DisableDTE); + + smc_result = si_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac); + + ni_pi->cac_enabled = false; + + if (ni_pi->support_cac_long_term_average) + smc_result = si_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable); + } + } + return ret; +} + +static int si_init_smc_spll_table(struct radeon_device *rdev) +{ + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + SMC_SISLANDS_SPLL_DIV_TABLE *spll_table; + SISLANDS_SMC_SCLK_VALUE sclk_params; + u32 fb_div, p_div; + u32 clk_s, clk_v; + u32 sclk = 0; + int ret = 0; + u32 tmp; + int i; + + if (si_pi->spll_table_start == 0) + return -EINVAL; + + spll_table = kzalloc(sizeof(SMC_SISLANDS_SPLL_DIV_TABLE), GFP_KERNEL); + if (spll_table == NULL) + return -ENOMEM; + + for (i = 0; i < 256; i++) { + ret = si_calculate_sclk_params(rdev, sclk, &sclk_params); + if (ret) + break; + + p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT; + fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT; + clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT; + clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT; + + fb_div &= ~0x00001FFF; + fb_div >>= 1; + clk_v >>= 6; + + if (p_div & ~(SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT)) + ret = -EINVAL; + if (fb_div & ~(SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT)) + ret = -EINVAL; + if (clk_s & ~(SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT)) + ret = -EINVAL; + if (clk_v & ~(SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT)) + ret = -EINVAL; + + if (ret) + break; + + tmp = ((fb_div << SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) | + ((p_div << SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_MASK); + spll_table->freq[i] = cpu_to_be32(tmp); + + tmp = ((clk_v << SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_MASK) | + ((clk_s << SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_MASK); + spll_table->ss[i] = cpu_to_be32(tmp); + + sclk += 512; + } + + + if (!ret) + ret = si_copy_bytes_to_smc(rdev, si_pi->spll_table_start, + (u8 *)spll_table, sizeof(SMC_SISLANDS_SPLL_DIV_TABLE), + si_pi->sram_end); + + if (ret) + ni_pi->enable_power_containment = false; + + kfree(spll_table); + + return ret; +} + +struct si_dpm_quirk { + u32 chip_vendor; + u32 chip_device; + u32 subsys_vendor; + u32 subsys_device; + u32 max_sclk; + u32 max_mclk; +}; + +/* cards with dpm stability problems */ +static struct si_dpm_quirk si_dpm_quirk_list[] = { + /* PITCAIRN - https://bugs.freedesktop.org/show_bug.cgi?id=76490 */ + { PCI_VENDOR_ID_ATI, 0x6810, 0x1462, 0x3036, 0, 120000 }, + { PCI_VENDOR_ID_ATI, 0x6811, 0x174b, 0xe271, 0, 120000 }, + { PCI_VENDOR_ID_ATI, 0x6811, 0x174b, 0x2015, 0, 120000 }, + { PCI_VENDOR_ID_ATI, 0x6810, 0x174b, 0xe271, 85000, 90000 }, + { PCI_VENDOR_ID_ATI, 0x6811, 0x1462, 0x2015, 0, 120000 }, + { PCI_VENDOR_ID_ATI, 0x6811, 0x1043, 0x2015, 0, 120000 }, + { PCI_VENDOR_ID_ATI, 0x6811, 0x148c, 0x2015, 0, 120000 }, + { PCI_VENDOR_ID_ATI, 0x6810, 0x1682, 0x9275, 0, 120000 }, + { 0, 0, 0, 0 }, +}; + +static u16 si_get_lower_of_leakage_and_vce_voltage(struct radeon_device *rdev, + u16 vce_voltage) +{ + u16 highest_leakage = 0; + struct si_power_info *si_pi = si_get_pi(rdev); + int i; + + for (i = 0; i < si_pi->leakage_voltage.count; i++){ + if (highest_leakage < si_pi->leakage_voltage.entries[i].voltage) + highest_leakage = si_pi->leakage_voltage.entries[i].voltage; + } + + if (si_pi->leakage_voltage.count && (highest_leakage < vce_voltage)) + return highest_leakage; + + return vce_voltage; +} + +static int si_get_vce_clock_voltage(struct radeon_device *rdev, + u32 evclk, u32 ecclk, u16 *voltage) +{ + u32 i; + int ret = -EINVAL; + struct radeon_vce_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table; + + if (((evclk == 0) && (ecclk == 0)) || + (table && (table->count == 0))) { + *voltage = 0; + return 0; + } + + for (i = 0; i < table->count; i++) { + if ((evclk <= table->entries[i].evclk) && + (ecclk <= table->entries[i].ecclk)) { + *voltage = table->entries[i].v; + ret = 0; + break; + } + } + + /* if no match return the highest voltage */ + if (ret) + *voltage = table->entries[table->count - 1].v; + + *voltage = si_get_lower_of_leakage_and_vce_voltage(rdev, *voltage); + + return ret; +} + +static void si_apply_state_adjust_rules(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct ni_ps *ps = ni_get_ps(rps); + struct radeon_clock_and_voltage_limits *max_limits; + bool disable_mclk_switching = false; + bool disable_sclk_switching = false; + u32 mclk, sclk; + u16 vddc, vddci, min_vce_voltage = 0; + u32 max_sclk_vddc, max_mclk_vddci, max_mclk_vddc; + u32 max_sclk = 0, max_mclk = 0; + int i; + struct si_dpm_quirk *p = si_dpm_quirk_list; + + /* limit all SI kickers */ + if (rdev->family == CHIP_PITCAIRN) { + if ((rdev->pdev->revision == 0x81) || + (rdev->pdev->device == 0x6810) || + (rdev->pdev->device == 0x6811) || + (rdev->pdev->device == 0x6816) || + (rdev->pdev->device == 0x6817) || + (rdev->pdev->device == 0x6806)) + max_mclk = 120000; + } else if (rdev->family == CHIP_OLAND) { + if ((rdev->pdev->revision == 0xC7) || + (rdev->pdev->revision == 0x80) || + (rdev->pdev->revision == 0x81) || + (rdev->pdev->revision == 0x83) || + (rdev->pdev->revision == 0x87) || + (rdev->pdev->device == 0x6604) || + (rdev->pdev->device == 0x6605)) { + max_sclk = 75000; + max_mclk = 80000; + } + } else if (rdev->family == CHIP_HAINAN) { + if ((rdev->pdev->revision == 0x81) || + (rdev->pdev->revision == 0x83) || + (rdev->pdev->revision == 0xC3) || + (rdev->pdev->device == 0x6664) || + (rdev->pdev->device == 0x6665) || + (rdev->pdev->device == 0x6667)) { + max_sclk = 75000; + max_mclk = 80000; + } + } else if (rdev->family == CHIP_OLAND) { + if ((rdev->pdev->revision == 0xC7) || + (rdev->pdev->revision == 0x80) || + (rdev->pdev->revision == 0x81) || + (rdev->pdev->revision == 0x83) || + (rdev->pdev->revision == 0x87) || + (rdev->pdev->device == 0x6604) || + (rdev->pdev->device == 0x6605)) { + max_sclk = 75000; + } + } + /* Apply dpm quirks */ + while (p && p->chip_device != 0) { + if (rdev->pdev->vendor == p->chip_vendor && + rdev->pdev->device == p->chip_device && + rdev->pdev->subsystem_vendor == p->subsys_vendor && + rdev->pdev->subsystem_device == p->subsys_device) { + max_sclk = p->max_sclk; + max_mclk = p->max_mclk; + break; + } + ++p; + } + + if (rps->vce_active) { + rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk; + rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk; + si_get_vce_clock_voltage(rdev, rps->evclk, rps->ecclk, + &min_vce_voltage); + } else { + rps->evclk = 0; + rps->ecclk = 0; + } + + if ((rdev->pm.dpm.new_active_crtc_count > 1) || + ni_dpm_vblank_too_short(rdev)) + disable_mclk_switching = true; + + if (rps->vclk || rps->dclk) { + disable_mclk_switching = true; + disable_sclk_switching = true; + } + + if (rdev->pm.dpm.ac_power) + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + else + max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc; + + for (i = ps->performance_level_count - 2; i >= 0; i--) { + if (ps->performance_levels[i].vddc > ps->performance_levels[i+1].vddc) + ps->performance_levels[i].vddc = ps->performance_levels[i+1].vddc; + } + if (rdev->pm.dpm.ac_power == false) { + for (i = 0; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].mclk > max_limits->mclk) + ps->performance_levels[i].mclk = max_limits->mclk; + if (ps->performance_levels[i].sclk > max_limits->sclk) + ps->performance_levels[i].sclk = max_limits->sclk; + if (ps->performance_levels[i].vddc > max_limits->vddc) + ps->performance_levels[i].vddc = max_limits->vddc; + if (ps->performance_levels[i].vddci > max_limits->vddci) + ps->performance_levels[i].vddci = max_limits->vddci; + } + } + + /* limit clocks to max supported clocks based on voltage dependency tables */ + btc_get_max_clock_from_voltage_dependency_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk, + &max_sclk_vddc); + btc_get_max_clock_from_voltage_dependency_table(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + &max_mclk_vddci); + btc_get_max_clock_from_voltage_dependency_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + &max_mclk_vddc); + + for (i = 0; i < ps->performance_level_count; i++) { + if (max_sclk_vddc) { + if (ps->performance_levels[i].sclk > max_sclk_vddc) + ps->performance_levels[i].sclk = max_sclk_vddc; + } + if (max_mclk_vddci) { + if (ps->performance_levels[i].mclk > max_mclk_vddci) + ps->performance_levels[i].mclk = max_mclk_vddci; + } + if (max_mclk_vddc) { + if (ps->performance_levels[i].mclk > max_mclk_vddc) + ps->performance_levels[i].mclk = max_mclk_vddc; + } + if (max_mclk) { + if (ps->performance_levels[i].mclk > max_mclk) + ps->performance_levels[i].mclk = max_mclk; + } + if (max_sclk) { + if (ps->performance_levels[i].sclk > max_sclk) + ps->performance_levels[i].sclk = max_sclk; + } + } + + /* XXX validate the min clocks required for display */ + + if (disable_mclk_switching) { + mclk = ps->performance_levels[ps->performance_level_count - 1].mclk; + vddci = ps->performance_levels[ps->performance_level_count - 1].vddci; + } else { + mclk = ps->performance_levels[0].mclk; + vddci = ps->performance_levels[0].vddci; + } + + if (disable_sclk_switching) { + sclk = ps->performance_levels[ps->performance_level_count - 1].sclk; + vddc = ps->performance_levels[ps->performance_level_count - 1].vddc; + } else { + sclk = ps->performance_levels[0].sclk; + vddc = ps->performance_levels[0].vddc; + } + + if (rps->vce_active) { + if (sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk) + sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk; + if (mclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk) + mclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk; + } + + /* adjusted low state */ + ps->performance_levels[0].sclk = sclk; + ps->performance_levels[0].mclk = mclk; + ps->performance_levels[0].vddc = vddc; + ps->performance_levels[0].vddci = vddci; + + if (disable_sclk_switching) { + sclk = ps->performance_levels[0].sclk; + for (i = 1; i < ps->performance_level_count; i++) { + if (sclk < ps->performance_levels[i].sclk) + sclk = ps->performance_levels[i].sclk; + } + for (i = 0; i < ps->performance_level_count; i++) { + ps->performance_levels[i].sclk = sclk; + ps->performance_levels[i].vddc = vddc; + } + } else { + for (i = 1; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk) + ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk; + if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc) + ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc; + } + } + + if (disable_mclk_switching) { + mclk = ps->performance_levels[0].mclk; + for (i = 1; i < ps->performance_level_count; i++) { + if (mclk < ps->performance_levels[i].mclk) + mclk = ps->performance_levels[i].mclk; + } + for (i = 0; i < ps->performance_level_count; i++) { + ps->performance_levels[i].mclk = mclk; + ps->performance_levels[i].vddci = vddci; + } + } else { + for (i = 1; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk) + ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk; + if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci) + ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci; + } + } + + for (i = 0; i < ps->performance_level_count; i++) + btc_adjust_clock_combinations(rdev, max_limits, + &ps->performance_levels[i]); + + for (i = 0; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].vddc < min_vce_voltage) + ps->performance_levels[i].vddc = min_vce_voltage; + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk, + ps->performance_levels[i].sclk, + max_limits->vddc, &ps->performance_levels[i].vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + ps->performance_levels[i].mclk, + max_limits->vddci, &ps->performance_levels[i].vddci); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + ps->performance_levels[i].mclk, + max_limits->vddc, &ps->performance_levels[i].vddc); + btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk, + rdev->clock.current_dispclk, + max_limits->vddc, &ps->performance_levels[i].vddc); + } + + for (i = 0; i < ps->performance_level_count; i++) { + btc_apply_voltage_delta_rules(rdev, + max_limits->vddc, max_limits->vddci, + &ps->performance_levels[i].vddc, + &ps->performance_levels[i].vddci); + } + + ps->dc_compatible = true; + for (i = 0; i < ps->performance_level_count; i++) { + if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc) + ps->dc_compatible = false; + } +} + +#if 0 +static int si_read_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 *value) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + + return si_read_smc_sram_dword(rdev, + si_pi->soft_regs_start + reg_offset, value, + si_pi->sram_end); +} +#endif + +static int si_write_smc_soft_register(struct radeon_device *rdev, + u16 reg_offset, u32 value) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + + return si_write_smc_sram_dword(rdev, + si_pi->soft_regs_start + reg_offset, + value, si_pi->sram_end); +} + +static bool si_is_special_1gb_platform(struct radeon_device *rdev) +{ + bool ret = false; + u32 tmp, width, row, column, bank, density; + bool is_memory_gddr5, is_special; + + tmp = RREG32(MC_SEQ_MISC0); + is_memory_gddr5 = (MC_SEQ_MISC0_GDDR5_VALUE == ((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT)); + is_special = (MC_SEQ_MISC0_REV_ID_VALUE == ((tmp & MC_SEQ_MISC0_REV_ID_MASK) >> MC_SEQ_MISC0_REV_ID_SHIFT)) + & (MC_SEQ_MISC0_VEN_ID_VALUE == ((tmp & MC_SEQ_MISC0_VEN_ID_MASK) >> MC_SEQ_MISC0_VEN_ID_SHIFT)); + + WREG32(MC_SEQ_IO_DEBUG_INDEX, 0xb); + width = ((RREG32(MC_SEQ_IO_DEBUG_DATA) >> 1) & 1) ? 16 : 32; + + tmp = RREG32(MC_ARB_RAMCFG); + row = ((tmp & NOOFROWS_MASK) >> NOOFROWS_SHIFT) + 10; + column = ((tmp & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) + 8; + bank = ((tmp & NOOFBANK_MASK) >> NOOFBANK_SHIFT) + 2; + + density = (1 << (row + column - 20 + bank)) * width; + + if ((rdev->pdev->device == 0x6819) && + is_memory_gddr5 && is_special && (density == 0x400)) + ret = true; + + return ret; +} + +static void si_get_leakage_vddc(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u16 vddc, count = 0; + int i, ret; + + for (i = 0; i < SISLANDS_MAX_LEAKAGE_COUNT; i++) { + ret = radeon_atom_get_leakage_vddc_based_on_leakage_idx(rdev, &vddc, SISLANDS_LEAKAGE_INDEX0 + i); + + if (!ret && (vddc > 0) && (vddc != (SISLANDS_LEAKAGE_INDEX0 + i))) { + si_pi->leakage_voltage.entries[count].voltage = vddc; + si_pi->leakage_voltage.entries[count].leakage_index = + SISLANDS_LEAKAGE_INDEX0 + i; + count++; + } + } + si_pi->leakage_voltage.count = count; +} + +static int si_get_leakage_voltage_from_leakage_index(struct radeon_device *rdev, + u32 index, u16 *leakage_voltage) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + int i; + + if (leakage_voltage == NULL) + return -EINVAL; + + if ((index & 0xff00) != 0xff00) + return -EINVAL; + + if ((index & 0xff) > SISLANDS_MAX_LEAKAGE_COUNT + 1) + return -EINVAL; + + if (index < SISLANDS_LEAKAGE_INDEX0) + return -EINVAL; + + for (i = 0; i < si_pi->leakage_voltage.count; i++) { + if (si_pi->leakage_voltage.entries[i].leakage_index == index) { + *leakage_voltage = si_pi->leakage_voltage.entries[i].voltage; + return 0; + } + } + return -EAGAIN; +} + +static void si_set_dpm_event_sources(struct radeon_device *rdev, u32 sources) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + bool want_thermal_protection; + enum radeon_dpm_event_src dpm_event_src; + + switch (sources) { + case 0: + default: + want_thermal_protection = false; + break; + case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL; + break; + case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL; + break; + case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) | + (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)): + want_thermal_protection = true; + dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL; + break; + } + + if (want_thermal_protection) { + WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK); + if (pi->thermal_protection) + WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS); + } else { + WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS); + } +} + +static void si_enable_auto_throttle_source(struct radeon_device *rdev, + enum radeon_dpm_auto_throttle_src source, + bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (enable) { + if (!(pi->active_auto_throttle_sources & (1 << source))) { + pi->active_auto_throttle_sources |= 1 << source; + si_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources); + } + } else { + if (pi->active_auto_throttle_sources & (1 << source)) { + pi->active_auto_throttle_sources &= ~(1 << source); + si_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources); + } + } +} + +static void si_start_dpm(struct radeon_device *rdev) +{ + WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN); +} + +static void si_stop_dpm(struct radeon_device *rdev) +{ + WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN); +} + +static void si_enable_sclk_control(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF); + else + WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF); + +} + +#if 0 +static int si_notify_hardware_of_thermal_state(struct radeon_device *rdev, + u32 thermal_level) +{ + PPSMC_Result ret; + + if (thermal_level == 0) { + ret = si_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt); + if (ret == PPSMC_Result_OK) + return 0; + else + return -EINVAL; + } + return 0; +} + +static void si_notify_hardware_vpu_recovery_event(struct radeon_device *rdev) +{ + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_tdr_is_about_to_happen, true); +} +#endif + +#if 0 +static int si_notify_hw_of_powersource(struct radeon_device *rdev, bool ac_power) +{ + if (ac_power) + return (si_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ? + 0 : -EINVAL; + + return 0; +} +#endif + +static PPSMC_Result si_send_msg_to_smc_with_parameter(struct radeon_device *rdev, + PPSMC_Msg msg, u32 parameter) +{ + WREG32(SMC_SCRATCH0, parameter); + return si_send_msg_to_smc(rdev, msg); +} + +static int si_restrict_performance_levels_before_switch(struct radeon_device *rdev) +{ + if (si_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK) + return -EINVAL; + + return (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +int si_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level) +{ + struct radeon_ps *rps = rdev->pm.dpm.current_ps; + struct ni_ps *ps = ni_get_ps(rps); + u32 levels = ps->performance_level_count; + + if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + if (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, levels) != PPSMC_Result_OK) + return -EINVAL; + + if (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK) + return -EINVAL; + } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { + if (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK) + return -EINVAL; + + if (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK) + return -EINVAL; + } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) { + if (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK) + return -EINVAL; + + if (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, levels) != PPSMC_Result_OK) + return -EINVAL; + } + + rdev->pm.dpm.forced_level = level; + + return 0; +} + +#if 0 +static int si_set_boot_state(struct radeon_device *rdev) +{ + return (si_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToInitialState) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} +#endif + +static int si_set_sw_state(struct radeon_device *rdev) +{ + return (si_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToSwState) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +static int si_halt_smc(struct radeon_device *rdev) +{ + if (si_send_msg_to_smc(rdev, PPSMC_MSG_Halt) != PPSMC_Result_OK) + return -EINVAL; + + return (si_wait_for_smc_inactive(rdev) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +static int si_resume_smc(struct radeon_device *rdev) +{ + if (si_send_msg_to_smc(rdev, PPSMC_FlushDataCache) != PPSMC_Result_OK) + return -EINVAL; + + return (si_send_msg_to_smc(rdev, PPSMC_MSG_Resume) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +static void si_dpm_start_smc(struct radeon_device *rdev) +{ + si_program_jump_on_start(rdev); + si_start_smc(rdev); + si_start_smc_clock(rdev); +} + +static void si_dpm_stop_smc(struct radeon_device *rdev) +{ + si_reset_smc(rdev); + si_stop_smc_clock(rdev); +} + +static int si_process_firmware_header(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 tmp; + int ret; + + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_stateTable, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->state_table_start = tmp; + + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_softRegisters, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->soft_regs_start = tmp; + + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->mc_reg_table_start = tmp; + + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_fanTable, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->fan_table_start = tmp; + + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->arb_table_start = tmp; + + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_CacConfigTable, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->cac_table_start = tmp; + + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_DteConfiguration, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->dte_table_start = tmp; + + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_spllTable, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->spll_table_start = tmp; + + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_PAPMParameters, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->papm_cfg_table_start = tmp; + + return ret; +} + +static void si_read_clock_registers(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + + si_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL); + si_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2); + si_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3); + si_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4); + si_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM); + si_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2); + si_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL); + si_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL); + si_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL); + si_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL); + si_pi->clock_registers.mpll_func_cntl = RREG32(MPLL_FUNC_CNTL); + si_pi->clock_registers.mpll_func_cntl_1 = RREG32(MPLL_FUNC_CNTL_1); + si_pi->clock_registers.mpll_func_cntl_2 = RREG32(MPLL_FUNC_CNTL_2); + si_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1); + si_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2); +} + +static void si_enable_thermal_protection(struct radeon_device *rdev, + bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS); + else + WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS); +} + +static void si_enable_acpi_power_management(struct radeon_device *rdev) +{ + WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN); +} + +#if 0 +static int si_enter_ulp_state(struct radeon_device *rdev) +{ + WREG32(SMC_MESSAGE_0, PPSMC_MSG_SwitchToMinimumPower); + + udelay(25000); + + return 0; +} + +static int si_exit_ulp_state(struct radeon_device *rdev) +{ + int i; + + WREG32(SMC_MESSAGE_0, PPSMC_MSG_ResumeFromMinimumPower); + + udelay(7000); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(SMC_RESP_0) == 1) + break; + udelay(1000); + } + + return 0; +} +#endif + +static int si_notify_smc_display_change(struct radeon_device *rdev, + bool has_display) +{ + PPSMC_Msg msg = has_display ? + PPSMC_MSG_HasDisplay : PPSMC_MSG_NoDisplay; + + return (si_send_msg_to_smc(rdev, msg) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +static void si_program_response_times(struct radeon_device *rdev) +{ + u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out; + u32 vddc_dly, acpi_dly, vbi_dly; + u32 reference_clock; + + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_mvdd_chg_time, 1); + + voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time; + backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time; + + if (voltage_response_time == 0) + voltage_response_time = 1000; + + acpi_delay_time = 15000; + vbi_time_out = 100000; + + reference_clock = radeon_get_xclk(rdev); + + vddc_dly = (voltage_response_time * reference_clock) / 100; + acpi_dly = (acpi_delay_time * reference_clock) / 100; + vbi_dly = (vbi_time_out * reference_clock) / 100; + + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_delay_vreg, vddc_dly); + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_delay_acpi, acpi_dly); + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly); + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA); +} + +static void si_program_ds_registers(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + u32 tmp = 1; /* XXX: 0x10 on tahiti A0 */ + + if (eg_pi->sclk_deep_sleep) { + WREG32_P(MISC_CLK_CNTL, DEEP_SLEEP_CLK_SEL(tmp), ~DEEP_SLEEP_CLK_SEL_MASK); + WREG32_P(CG_SPLL_AUTOSCALE_CNTL, AUTOSCALE_ON_SS_CLEAR, + ~AUTOSCALE_ON_SS_CLEAR); + } +} + +static void si_program_display_gap(struct radeon_device *rdev) +{ + u32 tmp, pipe; + int i; + + tmp = RREG32(CG_DISPLAY_GAP_CNTL) & ~(DISP1_GAP_MASK | DISP2_GAP_MASK); + if (rdev->pm.dpm.new_active_crtc_count > 0) + tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM); + else + tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_IGNORE); + + if (rdev->pm.dpm.new_active_crtc_count > 1) + tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM); + else + tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_IGNORE); + + WREG32(CG_DISPLAY_GAP_CNTL, tmp); + + tmp = RREG32(DCCG_DISP_SLOW_SELECT_REG); + pipe = (tmp & DCCG_DISP1_SLOW_SELECT_MASK) >> DCCG_DISP1_SLOW_SELECT_SHIFT; + + if ((rdev->pm.dpm.new_active_crtc_count > 0) && + (!(rdev->pm.dpm.new_active_crtcs & (1 << pipe)))) { + /* find the first active crtc */ + for (i = 0; i < rdev->num_crtc; i++) { + if (rdev->pm.dpm.new_active_crtcs & (1 << i)) + break; + } + if (i == rdev->num_crtc) + pipe = 0; + else + pipe = i; + + tmp &= ~DCCG_DISP1_SLOW_SELECT_MASK; + tmp |= DCCG_DISP1_SLOW_SELECT(pipe); + WREG32(DCCG_DISP_SLOW_SELECT_REG, tmp); + } + + /* Setting this to false forces the performance state to low if the crtcs are disabled. + * This can be a problem on PowerXpress systems or if you want to use the card + * for offscreen rendering or compute if there are no crtcs enabled. + */ + si_notify_smc_display_change(rdev, rdev->pm.dpm.new_active_crtc_count > 0); +} + +static void si_enable_spread_spectrum(struct radeon_device *rdev, bool enable) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + if (enable) { + if (pi->sclk_ss) + WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN); + } else { + WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN); + WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN); + } +} + +static void si_setup_bsp(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u32 xclk = radeon_get_xclk(rdev); + + r600_calculate_u_and_p(pi->asi, + xclk, + 16, + &pi->bsp, + &pi->bsu); + + r600_calculate_u_and_p(pi->pasi, + xclk, + 16, + &pi->pbsp, + &pi->pbsu); + + + pi->dsp = BSP(pi->bsp) | BSU(pi->bsu); + pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu); + + WREG32(CG_BSP, pi->dsp); +} + +static void si_program_git(struct radeon_device *rdev) +{ + WREG32_P(CG_GIT, CG_GICST(R600_GICST_DFLT), ~CG_GICST_MASK); +} + +static void si_program_tp(struct radeon_device *rdev) +{ + int i; + enum r600_td td = R600_TD_DFLT; + + for (i = 0; i < R600_PM_NUMBER_OF_TC; i++) + WREG32(CG_FFCT_0 + (i * 4), (UTC_0(r600_utc[i]) | DTC_0(r600_dtc[i]))); + + if (td == R600_TD_AUTO) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL); + else + WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL); + + if (td == R600_TD_UP) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE); + + if (td == R600_TD_DOWN) + WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE); +} + +static void si_program_tpp(struct radeon_device *rdev) +{ + WREG32(CG_TPC, R600_TPC_DFLT); +} + +static void si_program_sstp(struct radeon_device *rdev) +{ + WREG32(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT))); +} + +static void si_enable_display_gap(struct radeon_device *rdev) +{ + u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL); + + tmp &= ~(DISP1_GAP_MASK | DISP2_GAP_MASK); + tmp |= (DISP1_GAP(R600_PM_DISPLAY_GAP_IGNORE) | + DISP2_GAP(R600_PM_DISPLAY_GAP_IGNORE)); + + tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK); + tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK) | + DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE)); + WREG32(CG_DISPLAY_GAP_CNTL, tmp); +} + +static void si_program_vc(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + WREG32(CG_FTV, pi->vrc); +} + +static void si_clear_vc(struct radeon_device *rdev) +{ + WREG32(CG_FTV, 0); +} + +u8 si_get_ddr3_mclk_frequency_ratio(u32 memory_clock) +{ + u8 mc_para_index; + + if (memory_clock < 10000) + mc_para_index = 0; + else if (memory_clock >= 80000) + mc_para_index = 0x0f; + else + mc_para_index = (u8)((memory_clock - 10000) / 5000 + 1); + return mc_para_index; +} + +u8 si_get_mclk_frequency_ratio(u32 memory_clock, bool strobe_mode) +{ + u8 mc_para_index; + + if (strobe_mode) { + if (memory_clock < 12500) + mc_para_index = 0x00; + else if (memory_clock > 47500) + mc_para_index = 0x0f; + else + mc_para_index = (u8)((memory_clock - 10000) / 2500); + } else { + if (memory_clock < 65000) + mc_para_index = 0x00; + else if (memory_clock > 135000) + mc_para_index = 0x0f; + else + mc_para_index = (u8)((memory_clock - 60000) / 5000); + } + return mc_para_index; +} + +static u8 si_get_strobe_mode_settings(struct radeon_device *rdev, u32 mclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + bool strobe_mode = false; + u8 result = 0; + + if (mclk <= pi->mclk_strobe_mode_threshold) + strobe_mode = true; + + if (pi->mem_gddr5) + result = si_get_mclk_frequency_ratio(mclk, strobe_mode); + else + result = si_get_ddr3_mclk_frequency_ratio(mclk); + + if (strobe_mode) + result |= SISLANDS_SMC_STROBE_ENABLE; + + return result; +} + +static int si_upload_firmware(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + int ret; + + si_reset_smc(rdev); + si_stop_smc_clock(rdev); + + ret = si_load_smc_ucode(rdev, si_pi->sram_end); + + return ret; +} + +static bool si_validate_phase_shedding_tables(struct radeon_device *rdev, + const struct atom_voltage_table *table, + const struct radeon_phase_shedding_limits_table *limits) +{ + u32 data, num_bits, num_levels; + + if ((table == NULL) || (limits == NULL)) + return false; + + data = table->mask_low; + + num_bits = hweight32(data); + + if (num_bits == 0) + return false; + + num_levels = (1 << num_bits); + + if (table->count != num_levels) + return false; + + if (limits->count != (num_levels - 1)) + return false; + + return true; +} + +void si_trim_voltage_table_to_fit_state_table(struct radeon_device *rdev, + u32 max_voltage_steps, + struct atom_voltage_table *voltage_table) +{ + unsigned int i, diff; + + if (voltage_table->count <= max_voltage_steps) + return; + + diff = voltage_table->count - max_voltage_steps; + + for (i= 0; i < max_voltage_steps; i++) + voltage_table->entries[i] = voltage_table->entries[i + diff]; + + voltage_table->count = max_voltage_steps; +} + +static int si_get_svi2_voltage_table(struct radeon_device *rdev, + struct radeon_clock_voltage_dependency_table *voltage_dependency_table, + struct atom_voltage_table *voltage_table) +{ + u32 i; + + if (voltage_dependency_table == NULL) + return -EINVAL; + + voltage_table->mask_low = 0; + voltage_table->phase_delay = 0; + + voltage_table->count = voltage_dependency_table->count; + for (i = 0; i < voltage_table->count; i++) { + voltage_table->entries[i].value = voltage_dependency_table->entries[i].v; + voltage_table->entries[i].smio_low = 0; + } + + return 0; +} + +static int si_construct_voltage_tables(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + int ret; + + if (pi->voltage_control) { + ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDC, + VOLTAGE_OBJ_GPIO_LUT, &eg_pi->vddc_voltage_table); + if (ret) + return ret; + + if (eg_pi->vddc_voltage_table.count > SISLANDS_MAX_NO_VREG_STEPS) + si_trim_voltage_table_to_fit_state_table(rdev, + SISLANDS_MAX_NO_VREG_STEPS, + &eg_pi->vddc_voltage_table); + } else if (si_pi->voltage_control_svi2) { + ret = si_get_svi2_voltage_table(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + &eg_pi->vddc_voltage_table); + if (ret) + return ret; + } else { + return -EINVAL; + } + + if (eg_pi->vddci_control) { + ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDCI, + VOLTAGE_OBJ_GPIO_LUT, &eg_pi->vddci_voltage_table); + if (ret) + return ret; + + if (eg_pi->vddci_voltage_table.count > SISLANDS_MAX_NO_VREG_STEPS) + si_trim_voltage_table_to_fit_state_table(rdev, + SISLANDS_MAX_NO_VREG_STEPS, + &eg_pi->vddci_voltage_table); + } + if (si_pi->vddci_control_svi2) { + ret = si_get_svi2_voltage_table(rdev, + &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + &eg_pi->vddci_voltage_table); + if (ret) + return ret; + } + + if (pi->mvdd_control) { + ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_MVDDC, + VOLTAGE_OBJ_GPIO_LUT, &si_pi->mvdd_voltage_table); + + if (ret) { + pi->mvdd_control = false; + return ret; + } + + if (si_pi->mvdd_voltage_table.count == 0) { + pi->mvdd_control = false; + return -EINVAL; + } + + if (si_pi->mvdd_voltage_table.count > SISLANDS_MAX_NO_VREG_STEPS) + si_trim_voltage_table_to_fit_state_table(rdev, + SISLANDS_MAX_NO_VREG_STEPS, + &si_pi->mvdd_voltage_table); + } + + if (si_pi->vddc_phase_shed_control) { + ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDC, + VOLTAGE_OBJ_PHASE_LUT, &si_pi->vddc_phase_shed_table); + if (ret) + si_pi->vddc_phase_shed_control = false; + + if ((si_pi->vddc_phase_shed_table.count == 0) || + (si_pi->vddc_phase_shed_table.count > SISLANDS_MAX_NO_VREG_STEPS)) + si_pi->vddc_phase_shed_control = false; + } + + return 0; +} + +static void si_populate_smc_voltage_table(struct radeon_device *rdev, + const struct atom_voltage_table *voltage_table, + SISLANDS_SMC_STATETABLE *table) +{ + unsigned int i; + + for (i = 0; i < voltage_table->count; i++) + table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low); +} + +static int si_populate_smc_voltage_tables(struct radeon_device *rdev, + SISLANDS_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + u8 i; + + if (si_pi->voltage_control_svi2) { + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svc, + si_pi->svc_gpio_id); + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svd, + si_pi->svd_gpio_id); + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_svi_rework_plat_type, + 2); + } else { + if (eg_pi->vddc_voltage_table.count) { + si_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table); + table->voltageMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDC] = + cpu_to_be32(eg_pi->vddc_voltage_table.mask_low); + + for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) { + if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) { + table->maxVDDCIndexInPPTable = i; + break; + } + } + } + + if (eg_pi->vddci_voltage_table.count) { + si_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table); + + table->voltageMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDCI] = + cpu_to_be32(eg_pi->vddci_voltage_table.mask_low); + } + + + if (si_pi->mvdd_voltage_table.count) { + si_populate_smc_voltage_table(rdev, &si_pi->mvdd_voltage_table, table); + + table->voltageMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_MVDD] = + cpu_to_be32(si_pi->mvdd_voltage_table.mask_low); + } + + if (si_pi->vddc_phase_shed_control) { + if (si_validate_phase_shedding_tables(rdev, &si_pi->vddc_phase_shed_table, + &rdev->pm.dpm.dyn_state.phase_shedding_limits_table)) { + si_populate_smc_voltage_table(rdev, &si_pi->vddc_phase_shed_table, table); + + table->phaseMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDC_PHASE_SHEDDING] = + cpu_to_be32(si_pi->vddc_phase_shed_table.mask_low); + + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_phase_shedding_delay, + (u32)si_pi->vddc_phase_shed_table.phase_delay); + } else { + si_pi->vddc_phase_shed_control = false; + } + } + } + + return 0; +} + +static int si_populate_voltage_value(struct radeon_device *rdev, + const struct atom_voltage_table *table, + u16 value, SISLANDS_SMC_VOLTAGE_VALUE *voltage) +{ + unsigned int i; + + for (i = 0; i < table->count; i++) { + if (value <= table->entries[i].value) { + voltage->index = (u8)i; + voltage->value = cpu_to_be16(table->entries[i].value); + break; + } + } + + if (i >= table->count) + return -EINVAL; + + return 0; +} + +static int si_populate_mvdd_value(struct radeon_device *rdev, u32 mclk, + SISLANDS_SMC_VOLTAGE_VALUE *voltage) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + + if (pi->mvdd_control) { + if (mclk <= pi->mvdd_split_frequency) + voltage->index = 0; + else + voltage->index = (u8)(si_pi->mvdd_voltage_table.count) - 1; + + voltage->value = cpu_to_be16(si_pi->mvdd_voltage_table.entries[voltage->index].value); + } + return 0; +} + +static int si_get_std_voltage_value(struct radeon_device *rdev, + SISLANDS_SMC_VOLTAGE_VALUE *voltage, + u16 *std_voltage) +{ + u16 v_index; + bool voltage_found = false; + *std_voltage = be16_to_cpu(voltage->value); + + if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries) { + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_NEW_CAC_VOLTAGE) { + if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries == NULL) + return -EINVAL; + + for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) { + if (be16_to_cpu(voltage->value) == + (u16)rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) { + voltage_found = true; + if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count) + *std_voltage = + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[v_index].vddc; + else + *std_voltage = + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[rdev->pm.dpm.dyn_state.cac_leakage_table.count-1].vddc; + break; + } + } + + if (!voltage_found) { + for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) { + if (be16_to_cpu(voltage->value) <= + (u16)rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) { + voltage_found = true; + if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count) + *std_voltage = + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[v_index].vddc; + else + *std_voltage = + rdev->pm.dpm.dyn_state.cac_leakage_table.entries[rdev->pm.dpm.dyn_state.cac_leakage_table.count-1].vddc; + break; + } + } + } + } else { + if ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count) + *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc; + } + } + + return 0; +} + +static int si_populate_std_voltage_value(struct radeon_device *rdev, + u16 value, u8 index, + SISLANDS_SMC_VOLTAGE_VALUE *voltage) +{ + voltage->index = index; + voltage->value = cpu_to_be16(value); + + return 0; +} + +static int si_populate_phase_shedding_value(struct radeon_device *rdev, + const struct radeon_phase_shedding_limits_table *limits, + u16 voltage, u32 sclk, u32 mclk, + SISLANDS_SMC_VOLTAGE_VALUE *smc_voltage) +{ + unsigned int i; + + for (i = 0; i < limits->count; i++) { + if ((voltage <= limits->entries[i].voltage) && + (sclk <= limits->entries[i].sclk) && + (mclk <= limits->entries[i].mclk)) + break; + } + + smc_voltage->phase_settings = (u8)i; + + return 0; +} + +static int si_init_arb_table_index(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 tmp; + int ret; + + ret = si_read_smc_sram_dword(rdev, si_pi->arb_table_start, &tmp, si_pi->sram_end); + if (ret) + return ret; + + tmp &= 0x00FFFFFF; + tmp |= MC_CG_ARB_FREQ_F1 << 24; + + return si_write_smc_sram_dword(rdev, si_pi->arb_table_start, tmp, si_pi->sram_end); +} + +static int si_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev) +{ + return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1); +} + +static int si_reset_to_default(struct radeon_device *rdev) +{ + return (si_send_msg_to_smc(rdev, PPSMC_MSG_ResetToDefaults) == PPSMC_Result_OK) ? + 0 : -EINVAL; +} + +static int si_force_switch_to_arb_f0(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 tmp; + int ret; + + ret = si_read_smc_sram_dword(rdev, si_pi->arb_table_start, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + tmp = (tmp >> 24) & 0xff; + + if (tmp == MC_CG_ARB_FREQ_F0) + return 0; + + return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0); +} + +static u32 si_calculate_memory_refresh_rate(struct radeon_device *rdev, + u32 engine_clock) +{ + u32 dram_rows; + u32 dram_refresh_rate; + u32 mc_arb_rfsh_rate; + u32 tmp = (RREG32(MC_ARB_RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT; + + if (tmp >= 4) + dram_rows = 16384; + else + dram_rows = 1 << (tmp + 10); + + dram_refresh_rate = 1 << ((RREG32(MC_SEQ_MISC0) & 0x3) + 3); + mc_arb_rfsh_rate = ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64; + + return mc_arb_rfsh_rate; +} + +static int si_populate_memory_timing_parameters(struct radeon_device *rdev, + struct rv7xx_pl *pl, + SMC_SIslands_MCArbDramTimingRegisterSet *arb_regs) +{ + u32 dram_timing; + u32 dram_timing2; + u32 burst_time; + + arb_regs->mc_arb_rfsh_rate = + (u8)si_calculate_memory_refresh_rate(rdev, pl->sclk); + + radeon_atom_set_engine_dram_timings(rdev, + pl->sclk, + pl->mclk); + + dram_timing = RREG32(MC_ARB_DRAM_TIMING); + dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2); + burst_time = RREG32(MC_ARB_BURST_TIME) & STATE0_MASK; + + arb_regs->mc_arb_dram_timing = cpu_to_be32(dram_timing); + arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2); + arb_regs->mc_arb_burst_time = (u8)burst_time; + + return 0; +} + +static int si_do_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + unsigned int first_arb_set) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + SMC_SIslands_MCArbDramTimingRegisterSet arb_regs = { 0 }; + int i, ret = 0; + + for (i = 0; i < state->performance_level_count; i++) { + ret = si_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs); + if (ret) + break; + ret = si_copy_bytes_to_smc(rdev, + si_pi->arb_table_start + + offsetof(SMC_SIslands_MCArbDramTimingRegisters, data) + + sizeof(SMC_SIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i), + (u8 *)&arb_regs, + sizeof(SMC_SIslands_MCArbDramTimingRegisterSet), + si_pi->sram_end); + if (ret) + break; + } + + return ret; +} + +static int si_program_memory_timing_parameters(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + return si_do_program_memory_timing_parameters(rdev, radeon_new_state, + SISLANDS_DRIVER_STATE_ARB_INDEX); +} + +static int si_populate_initial_mvdd_value(struct radeon_device *rdev, + struct SISLANDS_SMC_VOLTAGE_VALUE *voltage) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + + if (pi->mvdd_control) + return si_populate_voltage_value(rdev, &si_pi->mvdd_voltage_table, + si_pi->mvdd_bootup_value, voltage); + + return 0; +} + +static int si_populate_smc_initial_state(struct radeon_device *rdev, + struct radeon_ps *radeon_initial_state, + SISLANDS_SMC_STATETABLE *table) +{ + struct ni_ps *initial_state = ni_get_ps(radeon_initial_state); + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + u32 reg; + int ret; + + table->initialState.levels[0].mclk.vDLL_CNTL = + cpu_to_be32(si_pi->clock_registers.dll_cntl); + table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL = + cpu_to_be32(si_pi->clock_registers.mclk_pwrmgt_cntl); + table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = + cpu_to_be32(si_pi->clock_registers.mpll_ad_func_cntl); + table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = + cpu_to_be32(si_pi->clock_registers.mpll_dq_func_cntl); + table->initialState.levels[0].mclk.vMPLL_FUNC_CNTL = + cpu_to_be32(si_pi->clock_registers.mpll_func_cntl); + table->initialState.levels[0].mclk.vMPLL_FUNC_CNTL_1 = + cpu_to_be32(si_pi->clock_registers.mpll_func_cntl_1); + table->initialState.levels[0].mclk.vMPLL_FUNC_CNTL_2 = + cpu_to_be32(si_pi->clock_registers.mpll_func_cntl_2); + table->initialState.levels[0].mclk.vMPLL_SS = + cpu_to_be32(si_pi->clock_registers.mpll_ss1); + table->initialState.levels[0].mclk.vMPLL_SS2 = + cpu_to_be32(si_pi->clock_registers.mpll_ss2); + + table->initialState.levels[0].mclk.mclk_value = + cpu_to_be32(initial_state->performance_levels[0].mclk); + + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = + cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = + cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl_2); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = + cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl_3); + table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = + cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl_4); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM = + cpu_to_be32(si_pi->clock_registers.cg_spll_spread_spectrum); + table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 = + cpu_to_be32(si_pi->clock_registers.cg_spll_spread_spectrum_2); + + table->initialState.levels[0].sclk.sclk_value = + cpu_to_be32(initial_state->performance_levels[0].sclk); + + table->initialState.levels[0].arbRefreshState = + SISLANDS_INITIAL_STATE_ARB_INDEX; + + table->initialState.levels[0].ACIndex = 0; + + ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table, + initial_state->performance_levels[0].vddc, + &table->initialState.levels[0].vddc); + + if (!ret) { + u16 std_vddc; + + ret = si_get_std_voltage_value(rdev, + &table->initialState.levels[0].vddc, + &std_vddc); + if (!ret) + si_populate_std_voltage_value(rdev, std_vddc, + table->initialState.levels[0].vddc.index, + &table->initialState.levels[0].std_vddc); + } + + if (eg_pi->vddci_control) + si_populate_voltage_value(rdev, + &eg_pi->vddci_voltage_table, + initial_state->performance_levels[0].vddci, + &table->initialState.levels[0].vddci); + + if (si_pi->vddc_phase_shed_control) + si_populate_phase_shedding_value(rdev, + &rdev->pm.dpm.dyn_state.phase_shedding_limits_table, + initial_state->performance_levels[0].vddc, + initial_state->performance_levels[0].sclk, + initial_state->performance_levels[0].mclk, + &table->initialState.levels[0].vddc); + + si_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd); + + reg = CG_R(0xffff) | CG_L(0); + table->initialState.levels[0].aT = cpu_to_be32(reg); + + table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp); + + table->initialState.levels[0].gen2PCIE = (u8)si_pi->boot_pcie_gen; + + if (pi->mem_gddr5) { + table->initialState.levels[0].strobeMode = + si_get_strobe_mode_settings(rdev, + initial_state->performance_levels[0].mclk); + + if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold) + table->initialState.levels[0].mcFlags = SISLANDS_SMC_MC_EDC_RD_FLAG | SISLANDS_SMC_MC_EDC_WR_FLAG; + else + table->initialState.levels[0].mcFlags = 0; + } + + table->initialState.levelCount = 1; + + table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC; + + table->initialState.levels[0].dpm2.MaxPS = 0; + table->initialState.levels[0].dpm2.NearTDPDec = 0; + table->initialState.levels[0].dpm2.AboveSafeInc = 0; + table->initialState.levels[0].dpm2.BelowSafeInc = 0; + table->initialState.levels[0].dpm2.PwrEfficiencyRatio = 0; + + reg = MIN_POWER_MASK | MAX_POWER_MASK; + table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg); + + reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK; + table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg); + + return 0; +} + +static int si_populate_smc_acpi_state(struct radeon_device *rdev, + SISLANDS_SMC_STATETABLE *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + u32 spll_func_cntl = si_pi->clock_registers.cg_spll_func_cntl; + u32 spll_func_cntl_2 = si_pi->clock_registers.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = si_pi->clock_registers.cg_spll_func_cntl_3; + u32 spll_func_cntl_4 = si_pi->clock_registers.cg_spll_func_cntl_4; + u32 dll_cntl = si_pi->clock_registers.dll_cntl; + u32 mclk_pwrmgt_cntl = si_pi->clock_registers.mclk_pwrmgt_cntl; + u32 mpll_ad_func_cntl = si_pi->clock_registers.mpll_ad_func_cntl; + u32 mpll_dq_func_cntl = si_pi->clock_registers.mpll_dq_func_cntl; + u32 mpll_func_cntl = si_pi->clock_registers.mpll_func_cntl; + u32 mpll_func_cntl_1 = si_pi->clock_registers.mpll_func_cntl_1; + u32 mpll_func_cntl_2 = si_pi->clock_registers.mpll_func_cntl_2; + u32 reg; + int ret; + + table->ACPIState = table->initialState; + + table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC; + + if (pi->acpi_vddc) { + ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table, + pi->acpi_vddc, &table->ACPIState.levels[0].vddc); + if (!ret) { + u16 std_vddc; + + ret = si_get_std_voltage_value(rdev, + &table->ACPIState.levels[0].vddc, &std_vddc); + if (!ret) + si_populate_std_voltage_value(rdev, std_vddc, + table->ACPIState.levels[0].vddc.index, + &table->ACPIState.levels[0].std_vddc); + } + table->ACPIState.levels[0].gen2PCIE = si_pi->acpi_pcie_gen; + + if (si_pi->vddc_phase_shed_control) { + si_populate_phase_shedding_value(rdev, + &rdev->pm.dpm.dyn_state.phase_shedding_limits_table, + pi->acpi_vddc, + 0, + 0, + &table->ACPIState.levels[0].vddc); + } + } else { + ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table, + pi->min_vddc_in_table, &table->ACPIState.levels[0].vddc); + if (!ret) { + u16 std_vddc; + + ret = si_get_std_voltage_value(rdev, + &table->ACPIState.levels[0].vddc, &std_vddc); + + if (!ret) + si_populate_std_voltage_value(rdev, std_vddc, + table->ACPIState.levels[0].vddc.index, + &table->ACPIState.levels[0].std_vddc); + } + table->ACPIState.levels[0].gen2PCIE = (u8)r600_get_pcie_gen_support(rdev, + si_pi->sys_pcie_mask, + si_pi->boot_pcie_gen, + RADEON_PCIE_GEN1); + + if (si_pi->vddc_phase_shed_control) + si_populate_phase_shedding_value(rdev, + &rdev->pm.dpm.dyn_state.phase_shedding_limits_table, + pi->min_vddc_in_table, + 0, + 0, + &table->ACPIState.levels[0].vddc); + } + + if (pi->acpi_vddc) { + if (eg_pi->acpi_vddci) + si_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table, + eg_pi->acpi_vddci, + &table->ACPIState.levels[0].vddci); + } + + mclk_pwrmgt_cntl |= MRDCK0_RESET | MRDCK1_RESET; + mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB); + + dll_cntl &= ~(MRDCK0_BYPASS | MRDCK1_BYPASS); + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(4); + + table->ACPIState.levels[0].mclk.vDLL_CNTL = + cpu_to_be32(dll_cntl); + table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = + cpu_to_be32(mclk_pwrmgt_cntl); + table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = + cpu_to_be32(mpll_ad_func_cntl); + table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = + cpu_to_be32(mpll_dq_func_cntl); + table->ACPIState.levels[0].mclk.vMPLL_FUNC_CNTL = + cpu_to_be32(mpll_func_cntl); + table->ACPIState.levels[0].mclk.vMPLL_FUNC_CNTL_1 = + cpu_to_be32(mpll_func_cntl_1); + table->ACPIState.levels[0].mclk.vMPLL_FUNC_CNTL_2 = + cpu_to_be32(mpll_func_cntl_2); + table->ACPIState.levels[0].mclk.vMPLL_SS = + cpu_to_be32(si_pi->clock_registers.mpll_ss1); + table->ACPIState.levels[0].mclk.vMPLL_SS2 = + cpu_to_be32(si_pi->clock_registers.mpll_ss2); + + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = + cpu_to_be32(spll_func_cntl); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = + cpu_to_be32(spll_func_cntl_2); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = + cpu_to_be32(spll_func_cntl_3); + table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = + cpu_to_be32(spll_func_cntl_4); + + table->ACPIState.levels[0].mclk.mclk_value = 0; + table->ACPIState.levels[0].sclk.sclk_value = 0; + + si_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd); + + if (eg_pi->dynamic_ac_timing) + table->ACPIState.levels[0].ACIndex = 0; + + table->ACPIState.levels[0].dpm2.MaxPS = 0; + table->ACPIState.levels[0].dpm2.NearTDPDec = 0; + table->ACPIState.levels[0].dpm2.AboveSafeInc = 0; + table->ACPIState.levels[0].dpm2.BelowSafeInc = 0; + table->ACPIState.levels[0].dpm2.PwrEfficiencyRatio = 0; + + reg = MIN_POWER_MASK | MAX_POWER_MASK; + table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg); + + reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK; + table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg); + + return 0; +} + +static int si_populate_ulv_state(struct radeon_device *rdev, + SISLANDS_SMC_SWSTATE *state) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + struct si_ulv_param *ulv = &si_pi->ulv; + u32 sclk_in_sr = 1350; /* ??? */ + int ret; + + ret = si_convert_power_level_to_smc(rdev, &ulv->pl, + &state->levels[0]); + if (!ret) { + if (eg_pi->sclk_deep_sleep) { + if (sclk_in_sr <= SCLK_MIN_DEEPSLEEP_FREQ) + state->levels[0].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_BYPASS; + else + state->levels[0].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE; + } + if (ulv->one_pcie_lane_in_ulv) + state->flags |= PPSMC_SWSTATE_FLAG_PCIE_X1; + state->levels[0].arbRefreshState = (u8)(SISLANDS_ULV_STATE_ARB_INDEX); + state->levels[0].ACIndex = 1; + state->levels[0].std_vddc = state->levels[0].vddc; + state->levelCount = 1; + + state->flags |= PPSMC_SWSTATE_FLAG_DC; + } + + return ret; +} + +static int si_program_ulv_memory_timing_parameters(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + struct si_ulv_param *ulv = &si_pi->ulv; + SMC_SIslands_MCArbDramTimingRegisterSet arb_regs = { 0 }; + int ret; + + ret = si_populate_memory_timing_parameters(rdev, &ulv->pl, + &arb_regs); + if (ret) + return ret; + + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_ulv_volt_change_delay, + ulv->volt_change_delay); + + ret = si_copy_bytes_to_smc(rdev, + si_pi->arb_table_start + + offsetof(SMC_SIslands_MCArbDramTimingRegisters, data) + + sizeof(SMC_SIslands_MCArbDramTimingRegisterSet) * SISLANDS_ULV_STATE_ARB_INDEX, + (u8 *)&arb_regs, + sizeof(SMC_SIslands_MCArbDramTimingRegisterSet), + si_pi->sram_end); + + return ret; +} + +static void si_get_mvdd_configuration(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + + pi->mvdd_split_frequency = 30000; +} + +static int si_init_smc_table(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps; + const struct si_ulv_param *ulv = &si_pi->ulv; + SISLANDS_SMC_STATETABLE *table = &si_pi->smc_statetable; + int ret; + u32 lane_width; + u32 vr_hot_gpio; + + si_populate_smc_voltage_tables(rdev, table); + + switch (rdev->pm.int_thermal_type) { + case THERMAL_TYPE_SI: + case THERMAL_TYPE_EMC2103_WITH_INTERNAL: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL; + break; + case THERMAL_TYPE_NONE: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE; + break; + default: + table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL; + break; + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) + table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT) { + if ((rdev->pdev->device != 0x6818) && (rdev->pdev->device != 0x6819)) + table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT; + } + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) + table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC; + + if (pi->mem_gddr5) + table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REVERT_GPIO5_POLARITY) + table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_GPIO5_POLARITY_HIGH; + + if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_VRHOT_GPIO_CONFIGURABLE) { + table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT_PROG_GPIO; + vr_hot_gpio = rdev->pm.dpm.backbias_response_time; + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_vr_hot_gpio, + vr_hot_gpio); + } + + ret = si_populate_smc_initial_state(rdev, radeon_boot_state, table); + if (ret) + return ret; + + ret = si_populate_smc_acpi_state(rdev, table); + if (ret) + return ret; + + table->driverState = table->initialState; + + ret = si_do_program_memory_timing_parameters(rdev, radeon_boot_state, + SISLANDS_INITIAL_STATE_ARB_INDEX); + if (ret) + return ret; + + if (ulv->supported && ulv->pl.vddc) { + ret = si_populate_ulv_state(rdev, &table->ULVState); + if (ret) + return ret; + + ret = si_program_ulv_memory_timing_parameters(rdev); + if (ret) + return ret; + + WREG32(CG_ULV_CONTROL, ulv->cg_ulv_control); + WREG32(CG_ULV_PARAMETER, ulv->cg_ulv_parameter); + + lane_width = radeon_get_pcie_lanes(rdev); + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_non_ulv_pcie_link_width, lane_width); + } else { + table->ULVState = table->initialState; + } + + return si_copy_bytes_to_smc(rdev, si_pi->state_table_start, + (u8 *)table, sizeof(SISLANDS_SMC_STATETABLE), + si_pi->sram_end); +} + +static int si_calculate_sclk_params(struct radeon_device *rdev, + u32 engine_clock, + SISLANDS_SMC_SCLK_VALUE *sclk) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + struct atom_clock_dividers dividers; + u32 spll_func_cntl = si_pi->clock_registers.cg_spll_func_cntl; + u32 spll_func_cntl_2 = si_pi->clock_registers.cg_spll_func_cntl_2; + u32 spll_func_cntl_3 = si_pi->clock_registers.cg_spll_func_cntl_3; + u32 spll_func_cntl_4 = si_pi->clock_registers.cg_spll_func_cntl_4; + u32 cg_spll_spread_spectrum = si_pi->clock_registers.cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2 = si_pi->clock_registers.cg_spll_spread_spectrum_2; + u64 tmp; + u32 reference_clock = rdev->clock.spll.reference_freq; + u32 reference_divider; + u32 fbdiv; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + engine_clock, false, ÷rs); + if (ret) + return ret; + + reference_divider = 1 + dividers.ref_div; + + tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16384; + do_div(tmp, reference_clock); + fbdiv = (u32) tmp; + + spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK); + spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div); + spll_func_cntl |= SPLL_PDIV_A(dividers.post_div); + + spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK; + spll_func_cntl_2 |= SCLK_MUX_SEL(2); + + spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK; + spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv); + spll_func_cntl_3 |= SPLL_DITHEN; + + if (pi->sclk_ss) { + struct radeon_atom_ss ss; + u32 vco_freq = engine_clock * dividers.post_div; + + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_ENGINE_SS, vco_freq)) { + u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate); + u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000); + + cg_spll_spread_spectrum &= ~CLK_S_MASK; + cg_spll_spread_spectrum |= CLK_S(clk_s); + cg_spll_spread_spectrum |= SSEN; + + cg_spll_spread_spectrum_2 &= ~CLK_V_MASK; + cg_spll_spread_spectrum_2 |= CLK_V(clk_v); + } + } + + sclk->sclk_value = engine_clock; + sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl; + sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2; + sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3; + sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4; + sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum; + sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2; + + return 0; +} + +static int si_populate_sclk_value(struct radeon_device *rdev, + u32 engine_clock, + SISLANDS_SMC_SCLK_VALUE *sclk) +{ + SISLANDS_SMC_SCLK_VALUE sclk_tmp; + int ret; + + ret = si_calculate_sclk_params(rdev, engine_clock, &sclk_tmp); + if (!ret) { + sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value); + sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL); + sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2); + sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3); + sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4); + sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM); + sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2); + } + + return ret; +} + +static int si_populate_mclk_value(struct radeon_device *rdev, + u32 engine_clock, + u32 memory_clock, + SISLANDS_SMC_MCLK_VALUE *mclk, + bool strobe_mode, + bool dll_state_on) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + u32 dll_cntl = si_pi->clock_registers.dll_cntl; + u32 mclk_pwrmgt_cntl = si_pi->clock_registers.mclk_pwrmgt_cntl; + u32 mpll_ad_func_cntl = si_pi->clock_registers.mpll_ad_func_cntl; + u32 mpll_dq_func_cntl = si_pi->clock_registers.mpll_dq_func_cntl; + u32 mpll_func_cntl = si_pi->clock_registers.mpll_func_cntl; + u32 mpll_func_cntl_1 = si_pi->clock_registers.mpll_func_cntl_1; + u32 mpll_func_cntl_2 = si_pi->clock_registers.mpll_func_cntl_2; + u32 mpll_ss1 = si_pi->clock_registers.mpll_ss1; + u32 mpll_ss2 = si_pi->clock_registers.mpll_ss2; + struct atom_mpll_param mpll_param; + int ret; + + ret = radeon_atom_get_memory_pll_dividers(rdev, memory_clock, strobe_mode, &mpll_param); + if (ret) + return ret; + + mpll_func_cntl &= ~BWCTRL_MASK; + mpll_func_cntl |= BWCTRL(mpll_param.bwcntl); + + mpll_func_cntl_1 &= ~(CLKF_MASK | CLKFRAC_MASK | VCO_MODE_MASK); + mpll_func_cntl_1 |= CLKF(mpll_param.clkf) | + CLKFRAC(mpll_param.clkfrac) | VCO_MODE(mpll_param.vco_mode); + + mpll_ad_func_cntl &= ~YCLK_POST_DIV_MASK; + mpll_ad_func_cntl |= YCLK_POST_DIV(mpll_param.post_div); + + if (pi->mem_gddr5) { + mpll_dq_func_cntl &= ~(YCLK_SEL_MASK | YCLK_POST_DIV_MASK); + mpll_dq_func_cntl |= YCLK_SEL(mpll_param.yclk_sel) | + YCLK_POST_DIV(mpll_param.post_div); + } + + if (pi->mclk_ss) { + struct radeon_atom_ss ss; + u32 freq_nom; + u32 tmp; + u32 reference_clock = rdev->clock.mpll.reference_freq; + + if (pi->mem_gddr5) + freq_nom = memory_clock * 4; + else + freq_nom = memory_clock * 2; + + tmp = freq_nom / reference_clock; + tmp = tmp * tmp; + if (radeon_atombios_get_asic_ss_info(rdev, &ss, + ASIC_INTERNAL_MEMORY_SS, freq_nom)) { + u32 clks = reference_clock * 5 / ss.rate; + u32 clkv = (u32)((((131 * ss.percentage * ss.rate) / 100) * tmp) / freq_nom); + + mpll_ss1 &= ~CLKV_MASK; + mpll_ss1 |= CLKV(clkv); + + mpll_ss2 &= ~CLKS_MASK; + mpll_ss2 |= CLKS(clks); + } + } + + mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK; + mclk_pwrmgt_cntl |= DLL_SPEED(mpll_param.dll_speed); + + if (dll_state_on) + mclk_pwrmgt_cntl |= MRDCK0_PDNB | MRDCK1_PDNB; + else + mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB); + + mclk->mclk_value = cpu_to_be32(memory_clock); + mclk->vMPLL_FUNC_CNTL = cpu_to_be32(mpll_func_cntl); + mclk->vMPLL_FUNC_CNTL_1 = cpu_to_be32(mpll_func_cntl_1); + mclk->vMPLL_FUNC_CNTL_2 = cpu_to_be32(mpll_func_cntl_2); + mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl); + mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl); + mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl); + mclk->vDLL_CNTL = cpu_to_be32(dll_cntl); + mclk->vMPLL_SS = cpu_to_be32(mpll_ss1); + mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2); + + return 0; +} + +static void si_populate_smc_sp(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + SISLANDS_SMC_SWSTATE *smc_state) +{ + struct ni_ps *ps = ni_get_ps(radeon_state); + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + int i; + + for (i = 0; i < ps->performance_level_count - 1; i++) + smc_state->levels[i].bSP = cpu_to_be32(pi->dsp); + + smc_state->levels[ps->performance_level_count - 1].bSP = + cpu_to_be32(pi->psp); +} + +static int si_convert_power_level_to_smc(struct radeon_device *rdev, + struct rv7xx_pl *pl, + SISLANDS_SMC_HW_PERFORMANCE_LEVEL *level) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + int ret; + bool dll_state_on; + u16 std_vddc; + bool gmc_pg = false; + + if (eg_pi->pcie_performance_request && + (si_pi->force_pcie_gen != RADEON_PCIE_GEN_INVALID)) + level->gen2PCIE = (u8)si_pi->force_pcie_gen; + else + level->gen2PCIE = (u8)pl->pcie_gen; + + ret = si_populate_sclk_value(rdev, pl->sclk, &level->sclk); + if (ret) + return ret; + + level->mcFlags = 0; + + if (pi->mclk_stutter_mode_threshold && + (pl->mclk <= pi->mclk_stutter_mode_threshold) && + !eg_pi->uvd_enabled && + (RREG32(DPG_PIPE_STUTTER_CONTROL) & STUTTER_ENABLE) && + (rdev->pm.dpm.new_active_crtc_count <= 2)) { + level->mcFlags |= SISLANDS_SMC_MC_STUTTER_EN; + + if (gmc_pg) + level->mcFlags |= SISLANDS_SMC_MC_PG_EN; + } + + if (pi->mem_gddr5) { + if (pl->mclk > pi->mclk_edc_enable_threshold) + level->mcFlags |= SISLANDS_SMC_MC_EDC_RD_FLAG; + + if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold) + level->mcFlags |= SISLANDS_SMC_MC_EDC_WR_FLAG; + + level->strobeMode = si_get_strobe_mode_settings(rdev, pl->mclk); + + if (level->strobeMode & SISLANDS_SMC_STROBE_ENABLE) { + if (si_get_mclk_frequency_ratio(pl->mclk, true) >= + ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf)) + dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false; + else + dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false; + } else { + dll_state_on = false; + } + } else { + level->strobeMode = si_get_strobe_mode_settings(rdev, + pl->mclk); + + dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false; + } + + ret = si_populate_mclk_value(rdev, + pl->sclk, + pl->mclk, + &level->mclk, + (level->strobeMode & SISLANDS_SMC_STROBE_ENABLE) != 0, dll_state_on); + if (ret) + return ret; + + ret = si_populate_voltage_value(rdev, + &eg_pi->vddc_voltage_table, + pl->vddc, &level->vddc); + if (ret) + return ret; + + + ret = si_get_std_voltage_value(rdev, &level->vddc, &std_vddc); + if (ret) + return ret; + + ret = si_populate_std_voltage_value(rdev, std_vddc, + level->vddc.index, &level->std_vddc); + if (ret) + return ret; + + if (eg_pi->vddci_control) { + ret = si_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table, + pl->vddci, &level->vddci); + if (ret) + return ret; + } + + if (si_pi->vddc_phase_shed_control) { + ret = si_populate_phase_shedding_value(rdev, + &rdev->pm.dpm.dyn_state.phase_shedding_limits_table, + pl->vddc, + pl->sclk, + pl->mclk, + &level->vddc); + if (ret) + return ret; + } + + level->MaxPoweredUpCU = si_pi->max_cu; + + ret = si_populate_mvdd_value(rdev, pl->mclk, &level->mvdd); + + return ret; +} + +static int si_populate_smc_t(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + SISLANDS_SMC_SWSTATE *smc_state) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + u32 a_t; + u32 t_l, t_h; + u32 high_bsp; + int i, ret; + + if (state->performance_level_count >= 9) + return -EINVAL; + + if (state->performance_level_count < 2) { + a_t = CG_R(0xffff) | CG_L(0); + smc_state->levels[0].aT = cpu_to_be32(a_t); + return 0; + } + + smc_state->levels[0].aT = cpu_to_be32(0); + + for (i = 0; i <= state->performance_level_count - 2; i++) { + ret = r600_calculate_at( + (50 / SISLANDS_MAX_HARDWARE_POWERLEVELS) * 100 * (i + 1), + 100 * R600_AH_DFLT, + state->performance_levels[i + 1].sclk, + state->performance_levels[i].sclk, + &t_l, + &t_h); + + if (ret) { + t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT; + t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT; + } + + a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK; + a_t |= CG_R(t_l * pi->bsp / 20000); + smc_state->levels[i].aT = cpu_to_be32(a_t); + + high_bsp = (i == state->performance_level_count - 2) ? + pi->pbsp : pi->bsp; + a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000); + smc_state->levels[i + 1].aT = cpu_to_be32(a_t); + } + + return 0; +} + +static int si_disable_ulv(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + struct si_ulv_param *ulv = &si_pi->ulv; + + if (ulv->supported) + return (si_send_msg_to_smc(rdev, PPSMC_MSG_DisableULV) == PPSMC_Result_OK) ? + 0 : -EINVAL; + + return 0; +} + +static bool si_is_state_ulv_compatible(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + const struct si_power_info *si_pi = si_get_pi(rdev); + const struct si_ulv_param *ulv = &si_pi->ulv; + const struct ni_ps *state = ni_get_ps(radeon_state); + int i; + + if (state->performance_levels[0].mclk != ulv->pl.mclk) + return false; + + /* XXX validate against display requirements! */ + + for (i = 0; i < rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count; i++) { + if (rdev->clock.current_dispclk <= + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[i].clk) { + if (ulv->pl.vddc < + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[i].v) + return false; + } + } + + if ((radeon_state->vclk != 0) || (radeon_state->dclk != 0)) + return false; + + return true; +} + +static int si_set_power_state_conditionally_enable_ulv(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + const struct si_power_info *si_pi = si_get_pi(rdev); + const struct si_ulv_param *ulv = &si_pi->ulv; + + if (ulv->supported) { + if (si_is_state_ulv_compatible(rdev, radeon_new_state)) + return (si_send_msg_to_smc(rdev, PPSMC_MSG_EnableULV) == PPSMC_Result_OK) ? + 0 : -EINVAL; + } + return 0; +} + +static int si_convert_power_state_to_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + SISLANDS_SMC_SWSTATE *smc_state) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct ni_power_info *ni_pi = ni_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + struct ni_ps *state = ni_get_ps(radeon_state); + int i, ret; + u32 threshold; + u32 sclk_in_sr = 1350; /* ??? */ + + if (state->performance_level_count > SISLANDS_MAX_HARDWARE_POWERLEVELS) + return -EINVAL; + + threshold = state->performance_levels[state->performance_level_count-1].sclk * 100 / 100; + + if (radeon_state->vclk && radeon_state->dclk) { + eg_pi->uvd_enabled = true; + if (eg_pi->smu_uvd_hs) + smc_state->flags |= PPSMC_SWSTATE_FLAG_UVD; + } else { + eg_pi->uvd_enabled = false; + } + + if (state->dc_compatible) + smc_state->flags |= PPSMC_SWSTATE_FLAG_DC; + + smc_state->levelCount = 0; + for (i = 0; i < state->performance_level_count; i++) { + if (eg_pi->sclk_deep_sleep) { + if ((i == 0) || si_pi->sclk_deep_sleep_above_low) { + if (sclk_in_sr <= SCLK_MIN_DEEPSLEEP_FREQ) + smc_state->levels[i].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_BYPASS; + else + smc_state->levels[i].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE; + } + } + + ret = si_convert_power_level_to_smc(rdev, &state->performance_levels[i], + &smc_state->levels[i]); + smc_state->levels[i].arbRefreshState = + (u8)(SISLANDS_DRIVER_STATE_ARB_INDEX + i); + + if (ret) + return ret; + + if (ni_pi->enable_power_containment) + smc_state->levels[i].displayWatermark = + (state->performance_levels[i].sclk < threshold) ? + PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH; + else + smc_state->levels[i].displayWatermark = (i < 2) ? + PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH; + + if (eg_pi->dynamic_ac_timing) + smc_state->levels[i].ACIndex = SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i; + else + smc_state->levels[i].ACIndex = 0; + + smc_state->levelCount++; + } + + si_write_smc_soft_register(rdev, + SI_SMC_SOFT_REGISTER_watermark_threshold, + threshold / 512); + + si_populate_smc_sp(rdev, radeon_state, smc_state); + + ret = si_populate_power_containment_values(rdev, radeon_state, smc_state); + if (ret) + ni_pi->enable_power_containment = false; + + ret = si_populate_sq_ramping_values(rdev, radeon_state, smc_state); + if (ret) + ni_pi->enable_sq_ramping = false; + + return si_populate_smc_t(rdev, radeon_state, smc_state); +} + +static int si_upload_sw_state(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + struct ni_ps *new_state = ni_get_ps(radeon_new_state); + int ret; + u32 address = si_pi->state_table_start + + offsetof(SISLANDS_SMC_STATETABLE, driverState); + u32 state_size = sizeof(SISLANDS_SMC_SWSTATE) + + ((new_state->performance_level_count - 1) * + sizeof(SISLANDS_SMC_HW_PERFORMANCE_LEVEL)); + SISLANDS_SMC_SWSTATE *smc_state = &si_pi->smc_statetable.driverState; + + memset(smc_state, 0, state_size); + + ret = si_convert_power_state_to_smc(rdev, radeon_new_state, smc_state); + if (ret) + return ret; + + ret = si_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, + state_size, si_pi->sram_end); + + return ret; +} + +static int si_upload_ulv_state(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + struct si_ulv_param *ulv = &si_pi->ulv; + int ret = 0; + + if (ulv->supported && ulv->pl.vddc) { + u32 address = si_pi->state_table_start + + offsetof(SISLANDS_SMC_STATETABLE, ULVState); + SISLANDS_SMC_SWSTATE *smc_state = &si_pi->smc_statetable.ULVState; + u32 state_size = sizeof(SISLANDS_SMC_SWSTATE); + + memset(smc_state, 0, state_size); + + ret = si_populate_ulv_state(rdev, smc_state); + if (!ret) + ret = si_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, + state_size, si_pi->sram_end); + } + + return ret; +} + +static int si_upload_smc_data(struct radeon_device *rdev) +{ + struct radeon_crtc *radeon_crtc = NULL; + int i; + + if (rdev->pm.dpm.new_active_crtc_count == 0) + return 0; + + for (i = 0; i < rdev->num_crtc; i++) { + if (rdev->pm.dpm.new_active_crtcs & (1 << i)) { + radeon_crtc = rdev->mode_info.crtcs[i]; + break; + } + } + + if (radeon_crtc == NULL) + return 0; + + if (radeon_crtc->line_time <= 0) + return 0; + + if (si_write_smc_soft_register(rdev, + SI_SMC_SOFT_REGISTER_crtc_index, + radeon_crtc->crtc_id) != PPSMC_Result_OK) + return 0; + + if (si_write_smc_soft_register(rdev, + SI_SMC_SOFT_REGISTER_mclk_change_block_cp_min, + radeon_crtc->wm_high / radeon_crtc->line_time) != PPSMC_Result_OK) + return 0; + + if (si_write_smc_soft_register(rdev, + SI_SMC_SOFT_REGISTER_mclk_change_block_cp_max, + radeon_crtc->wm_low / radeon_crtc->line_time) != PPSMC_Result_OK) + return 0; + + return 0; +} + +static int si_set_mc_special_registers(struct radeon_device *rdev, + struct si_mc_reg_table *table) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + u8 i, j, k; + u32 temp_reg; + + for (i = 0, j = table->last; i < table->last; i++) { + if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + switch (table->mc_reg_address[i].s1 << 2) { + case MC_SEQ_MISC1: + temp_reg = RREG32(MC_PMG_CMD_EMRS); + table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2; + for (k = 0; k < table->num_entries; k++) + table->mc_reg_table_entry[k].mc_data[j] = + ((temp_reg & 0xffff0000)) | + ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16); + j++; + if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + + temp_reg = RREG32(MC_PMG_CMD_MRS); + table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2; + for (k = 0; k < table->num_entries; k++) { + table->mc_reg_table_entry[k].mc_data[j] = + (temp_reg & 0xffff0000) | + (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); + if (!pi->mem_gddr5) + table->mc_reg_table_entry[k].mc_data[j] |= 0x100; + } + j++; + if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + + if (!pi->mem_gddr5) { + table->mc_reg_address[j].s1 = MC_PMG_AUTO_CMD >> 2; + table->mc_reg_address[j].s0 = MC_PMG_AUTO_CMD >> 2; + for (k = 0; k < table->num_entries; k++) + table->mc_reg_table_entry[k].mc_data[j] = + (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16; + j++; + if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + } + break; + case MC_SEQ_RESERVE_M: + temp_reg = RREG32(MC_PMG_CMD_MRS1); + table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2; + table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2; + for(k = 0; k < table->num_entries; k++) + table->mc_reg_table_entry[k].mc_data[j] = + (temp_reg & 0xffff0000) | + (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff); + j++; + if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + break; + default: + break; + } + } + + table->last = j; + + return 0; +} + +static bool si_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg) +{ + bool result = true; + + switch (in_reg) { + case MC_SEQ_RAS_TIMING >> 2: + *out_reg = MC_SEQ_RAS_TIMING_LP >> 2; + break; + case MC_SEQ_CAS_TIMING >> 2: + *out_reg = MC_SEQ_CAS_TIMING_LP >> 2; + break; + case MC_SEQ_MISC_TIMING >> 2: + *out_reg = MC_SEQ_MISC_TIMING_LP >> 2; + break; + case MC_SEQ_MISC_TIMING2 >> 2: + *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2; + break; + case MC_SEQ_RD_CTL_D0 >> 2: + *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2; + break; + case MC_SEQ_RD_CTL_D1 >> 2: + *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2; + break; + case MC_SEQ_WR_CTL_D0 >> 2: + *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2; + break; + case MC_SEQ_WR_CTL_D1 >> 2: + *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2; + break; + case MC_PMG_CMD_EMRS >> 2: + *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2; + break; + case MC_PMG_CMD_MRS >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2; + break; + case MC_PMG_CMD_MRS1 >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2; + break; + case MC_SEQ_PMG_TIMING >> 2: + *out_reg = MC_SEQ_PMG_TIMING_LP >> 2; + break; + case MC_PMG_CMD_MRS2 >> 2: + *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2; + break; + case MC_SEQ_WR_CTL_2 >> 2: + *out_reg = MC_SEQ_WR_CTL_2_LP >> 2; + break; + default: + result = false; + break; + } + + return result; +} + +static void si_set_valid_flag(struct si_mc_reg_table *table) +{ + u8 i, j; + + for (i = 0; i < table->last; i++) { + for (j = 1; j < table->num_entries; j++) { + if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) { + table->valid_flag |= 1 << i; + break; + } + } + } +} + +static void si_set_s0_mc_reg_index(struct si_mc_reg_table *table) +{ + u32 i; + u16 address; + + for (i = 0; i < table->last; i++) + table->mc_reg_address[i].s0 = si_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ? + address : table->mc_reg_address[i].s1; + +} + +static int si_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table, + struct si_mc_reg_table *si_table) +{ + u8 i, j; + + if (table->last > SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + if (table->num_entries > MAX_AC_TIMING_ENTRIES) + return -EINVAL; + + for (i = 0; i < table->last; i++) + si_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1; + si_table->last = table->last; + + for (i = 0; i < table->num_entries; i++) { + si_table->mc_reg_table_entry[i].mclk_max = + table->mc_reg_table_entry[i].mclk_max; + for (j = 0; j < table->last; j++) { + si_table->mc_reg_table_entry[i].mc_data[j] = + table->mc_reg_table_entry[i].mc_data[j]; + } + } + si_table->num_entries = table->num_entries; + + return 0; +} + +static int si_initialize_mc_reg_table(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + struct atom_mc_reg_table *table; + struct si_mc_reg_table *si_table = &si_pi->mc_reg_table; + u8 module_index = rv770_get_memory_module_index(rdev); + int ret; + + table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING)); + WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING)); + WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING)); + WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2)); + WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS)); + WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS)); + WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1)); + WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0)); + WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1)); + WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0)); + WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1)); + WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING)); + WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2)); + WREG32(MC_SEQ_WR_CTL_2_LP, RREG32(MC_SEQ_WR_CTL_2)); + + ret = radeon_atom_init_mc_reg_table(rdev, module_index, table); + if (ret) + goto init_mc_done; + + ret = si_copy_vbios_mc_reg_table(table, si_table); + if (ret) + goto init_mc_done; + + si_set_s0_mc_reg_index(si_table); + + ret = si_set_mc_special_registers(rdev, si_table); + if (ret) + goto init_mc_done; + + si_set_valid_flag(si_table); + +init_mc_done: + kfree(table); + + return ret; + +} + +static void si_populate_mc_reg_addresses(struct radeon_device *rdev, + SMC_SIslands_MCRegisters *mc_reg_table) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 i, j; + + for (i = 0, j = 0; j < si_pi->mc_reg_table.last; j++) { + if (si_pi->mc_reg_table.valid_flag & (1 << j)) { + if (i >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE) + break; + mc_reg_table->address[i].s0 = + cpu_to_be16(si_pi->mc_reg_table.mc_reg_address[j].s0); + mc_reg_table->address[i].s1 = + cpu_to_be16(si_pi->mc_reg_table.mc_reg_address[j].s1); + i++; + } + } + mc_reg_table->last = (u8)i; +} + +static void si_convert_mc_registers(const struct si_mc_reg_entry *entry, + SMC_SIslands_MCRegisterSet *data, + u32 num_entries, u32 valid_flag) +{ + u32 i, j; + + for(i = 0, j = 0; j < num_entries; j++) { + if (valid_flag & (1 << j)) { + data->value[i] = cpu_to_be32(entry->mc_data[j]); + i++; + } + } +} + +static void si_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev, + struct rv7xx_pl *pl, + SMC_SIslands_MCRegisterSet *mc_reg_table_data) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 i = 0; + + for (i = 0; i < si_pi->mc_reg_table.num_entries; i++) { + if (pl->mclk <= si_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max) + break; + } + + if ((i == si_pi->mc_reg_table.num_entries) && (i > 0)) + --i; + + si_convert_mc_registers(&si_pi->mc_reg_table.mc_reg_table_entry[i], + mc_reg_table_data, si_pi->mc_reg_table.last, + si_pi->mc_reg_table.valid_flag); +} + +static void si_convert_mc_reg_table_to_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_state, + SMC_SIslands_MCRegisters *mc_reg_table) +{ + struct ni_ps *state = ni_get_ps(radeon_state); + int i; + + for (i = 0; i < state->performance_level_count; i++) { + si_convert_mc_reg_table_entry_to_smc(rdev, + &state->performance_levels[i], + &mc_reg_table->data[SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]); + } +} + +static int si_populate_mc_reg_table(struct radeon_device *rdev, + struct radeon_ps *radeon_boot_state) +{ + struct ni_ps *boot_state = ni_get_ps(radeon_boot_state); + struct si_power_info *si_pi = si_get_pi(rdev); + struct si_ulv_param *ulv = &si_pi->ulv; + SMC_SIslands_MCRegisters *smc_mc_reg_table = &si_pi->smc_mc_reg_table; + + memset(smc_mc_reg_table, 0, sizeof(SMC_SIslands_MCRegisters)); + + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_seq_index, 1); + + si_populate_mc_reg_addresses(rdev, smc_mc_reg_table); + + si_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0], + &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_INITIAL_SLOT]); + + si_convert_mc_registers(&si_pi->mc_reg_table.mc_reg_table_entry[0], + &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_ACPI_SLOT], + si_pi->mc_reg_table.last, + si_pi->mc_reg_table.valid_flag); + + if (ulv->supported && ulv->pl.vddc != 0) + si_convert_mc_reg_table_entry_to_smc(rdev, &ulv->pl, + &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_ULV_SLOT]); + else + si_convert_mc_registers(&si_pi->mc_reg_table.mc_reg_table_entry[0], + &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_ULV_SLOT], + si_pi->mc_reg_table.last, + si_pi->mc_reg_table.valid_flag); + + si_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, smc_mc_reg_table); + + return si_copy_bytes_to_smc(rdev, si_pi->mc_reg_table_start, + (u8 *)smc_mc_reg_table, + sizeof(SMC_SIslands_MCRegisters), si_pi->sram_end); +} + +static int si_upload_mc_reg_table(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state) +{ + struct ni_ps *new_state = ni_get_ps(radeon_new_state); + struct si_power_info *si_pi = si_get_pi(rdev); + u32 address = si_pi->mc_reg_table_start + + offsetof(SMC_SIslands_MCRegisters, + data[SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]); + SMC_SIslands_MCRegisters *smc_mc_reg_table = &si_pi->smc_mc_reg_table; + + memset(smc_mc_reg_table, 0, sizeof(SMC_SIslands_MCRegisters)); + + si_convert_mc_reg_table_to_smc(rdev, radeon_new_state, smc_mc_reg_table); + + + return si_copy_bytes_to_smc(rdev, address, + (u8 *)&smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT], + sizeof(SMC_SIslands_MCRegisterSet) * new_state->performance_level_count, + si_pi->sram_end); + +} + +static void si_enable_voltage_control(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN); + else + WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN); +} + +static enum radeon_pcie_gen si_get_maximum_link_speed(struct radeon_device *rdev, + struct radeon_ps *radeon_state) +{ + struct ni_ps *state = ni_get_ps(radeon_state); + int i; + u16 pcie_speed, max_speed = 0; + + for (i = 0; i < state->performance_level_count; i++) { + pcie_speed = state->performance_levels[i].pcie_gen; + if (max_speed < pcie_speed) + max_speed = pcie_speed; + } + return max_speed; +} + +static u16 si_get_current_pcie_speed(struct radeon_device *rdev) +{ + u32 speed_cntl; + + speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE_MASK; + speed_cntl >>= LC_CURRENT_DATA_RATE_SHIFT; + + return (u16)speed_cntl; +} + +static void si_request_link_speed_change_before_state_change(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + enum radeon_pcie_gen target_link_speed = si_get_maximum_link_speed(rdev, radeon_new_state); + enum radeon_pcie_gen current_link_speed; + + if (si_pi->force_pcie_gen == RADEON_PCIE_GEN_INVALID) + current_link_speed = si_get_maximum_link_speed(rdev, radeon_current_state); + else + current_link_speed = si_pi->force_pcie_gen; + + si_pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID; + si_pi->pspp_notify_required = false; + if (target_link_speed > current_link_speed) { + switch (target_link_speed) { +#if defined(CONFIG_ACPI) + case RADEON_PCIE_GEN3: + if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN3, false) == 0) + break; + si_pi->force_pcie_gen = RADEON_PCIE_GEN2; + if (current_link_speed == RADEON_PCIE_GEN2) + break; + case RADEON_PCIE_GEN2: + if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, false) == 0) + break; +#endif + default: + si_pi->force_pcie_gen = si_get_current_pcie_speed(rdev); + break; + } + } else { + if (target_link_speed < current_link_speed) + si_pi->pspp_notify_required = true; + } +} + +static void si_notify_link_speed_change_after_state_change(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + enum radeon_pcie_gen target_link_speed = si_get_maximum_link_speed(rdev, radeon_new_state); + u8 request; + + if (si_pi->pspp_notify_required) { + if (target_link_speed == RADEON_PCIE_GEN3) + request = PCIE_PERF_REQ_PECI_GEN3; + else if (target_link_speed == RADEON_PCIE_GEN2) + request = PCIE_PERF_REQ_PECI_GEN2; + else + request = PCIE_PERF_REQ_PECI_GEN1; + + if ((request == PCIE_PERF_REQ_PECI_GEN1) && + (si_get_current_pcie_speed(rdev) > 0)) + return; + +#if defined(CONFIG_ACPI) + radeon_acpi_pcie_performance_request(rdev, request, false); +#endif + } +} + +#if 0 +static int si_ds_request(struct radeon_device *rdev, + bool ds_status_on, u32 count_write) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + + if (eg_pi->sclk_deep_sleep) { + if (ds_status_on) + return (si_send_msg_to_smc(rdev, PPSMC_MSG_CancelThrottleOVRDSCLKDS) == + PPSMC_Result_OK) ? + 0 : -EINVAL; + else + return (si_send_msg_to_smc(rdev, PPSMC_MSG_ThrottleOVRDSCLKDS) == + PPSMC_Result_OK) ? 0 : -EINVAL; + } + return 0; +} +#endif + +static void si_set_max_cu_value(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + + if (rdev->family == CHIP_VERDE) { + switch (rdev->pdev->device) { + case 0x6820: + case 0x6825: + case 0x6821: + case 0x6823: + case 0x6827: + si_pi->max_cu = 10; + break; + case 0x682D: + case 0x6824: + case 0x682F: + case 0x6826: + si_pi->max_cu = 8; + break; + case 0x6828: + case 0x6830: + case 0x6831: + case 0x6838: + case 0x6839: + case 0x683D: + si_pi->max_cu = 10; + break; + case 0x683B: + case 0x683F: + case 0x6829: + si_pi->max_cu = 8; + break; + default: + si_pi->max_cu = 0; + break; + } + } else { + si_pi->max_cu = 0; + } +} + +static int si_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev, + struct radeon_clock_voltage_dependency_table *table) +{ + u32 i; + int j; + u16 leakage_voltage; + + if (table) { + for (i = 0; i < table->count; i++) { + switch (si_get_leakage_voltage_from_leakage_index(rdev, + table->entries[i].v, + &leakage_voltage)) { + case 0: + table->entries[i].v = leakage_voltage; + break; + case -EAGAIN: + return -EINVAL; + case -EINVAL: + default: + break; + } + } + + for (j = (table->count - 2); j >= 0; j--) { + table->entries[j].v = (table->entries[j].v <= table->entries[j + 1].v) ? + table->entries[j].v : table->entries[j + 1].v; + } + } + return 0; +} + +static int si_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev) +{ + int ret = 0; + + ret = si_patch_single_dependency_table_based_on_leakage(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk); + ret = si_patch_single_dependency_table_based_on_leakage(rdev, + &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk); + ret = si_patch_single_dependency_table_based_on_leakage(rdev, + &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk); + return ret; +} + +static void si_set_pcie_lane_width_in_smc(struct radeon_device *rdev, + struct radeon_ps *radeon_new_state, + struct radeon_ps *radeon_current_state) +{ + u32 lane_width; + u32 new_lane_width = + ((radeon_new_state->caps & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >> ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT) + 1; + u32 current_lane_width = + ((radeon_current_state->caps & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >> ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT) + 1; + + if (new_lane_width != current_lane_width) { + radeon_set_pcie_lanes(rdev, new_lane_width); + lane_width = radeon_get_pcie_lanes(rdev); + si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_non_ulv_pcie_link_width, lane_width); + } +} + +static void si_set_vce_clock(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + if ((old_rps->evclk != new_rps->evclk) || + (old_rps->ecclk != new_rps->ecclk)) { + /* turn the clocks on when encoding, off otherwise */ + if (new_rps->evclk || new_rps->ecclk) + vce_v1_0_enable_mgcg(rdev, false); + else + vce_v1_0_enable_mgcg(rdev, true); + radeon_set_vce_clocks(rdev, new_rps->evclk, new_rps->ecclk); + } +} + +void si_dpm_setup_asic(struct radeon_device *rdev) +{ + int r; + + r = si_mc_load_microcode(rdev); + if (r) + DRM_ERROR("Failed to load MC firmware!\n"); + rv770_get_memory_type(rdev); + si_read_clock_registers(rdev); + si_enable_acpi_power_management(rdev); +} + +static int si_thermal_enable_alert(struct radeon_device *rdev, + bool enable) +{ + u32 thermal_int = RREG32(CG_THERMAL_INT); + + if (enable) { + PPSMC_Result result; + + thermal_int &= ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + WREG32(CG_THERMAL_INT, thermal_int); + rdev->irq.dpm_thermal = false; + result = si_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt); + if (result != PPSMC_Result_OK) { + DRM_DEBUG_KMS("Could not enable thermal interrupts.\n"); + return -EINVAL; + } + } else { + thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; + WREG32(CG_THERMAL_INT, thermal_int); + rdev->irq.dpm_thermal = true; + } + + return 0; +} + +static int si_thermal_set_temperature_range(struct radeon_device *rdev, + int min_temp, int max_temp) +{ + int low_temp = 0 * 1000; + int high_temp = 255 * 1000; + + if (low_temp < min_temp) + low_temp = min_temp; + if (high_temp > max_temp) + high_temp = max_temp; + if (high_temp < low_temp) { + DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp); + return -EINVAL; + } + + WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK); + WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK); + WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK); + + rdev->pm.dpm.thermal.min_temp = low_temp; + rdev->pm.dpm.thermal.max_temp = high_temp; + + return 0; +} + +static void si_fan_ctrl_set_static_mode(struct radeon_device *rdev, u32 mode) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 tmp; + + if (si_pi->fan_ctrl_is_in_default_mode) { + tmp = (RREG32(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK) >> FDO_PWM_MODE_SHIFT; + si_pi->fan_ctrl_default_mode = tmp; + tmp = (RREG32(CG_FDO_CTRL2) & TMIN_MASK) >> TMIN_SHIFT; + si_pi->t_min = tmp; + si_pi->fan_ctrl_is_in_default_mode = false; + } + + tmp = RREG32(CG_FDO_CTRL2) & ~TMIN_MASK; + tmp |= TMIN(0); + WREG32(CG_FDO_CTRL2, tmp); + + tmp = RREG32(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK; + tmp |= FDO_PWM_MODE(mode); + WREG32(CG_FDO_CTRL2, tmp); +} + +static int si_thermal_setup_fan_table(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + PP_SIslands_FanTable fan_table = { FDO_MODE_HARDWARE }; + u32 duty100; + u32 t_diff1, t_diff2, pwm_diff1, pwm_diff2; + u16 fdo_min, slope1, slope2; + u32 reference_clock, tmp; + int ret; + u64 tmp64; + + if (!si_pi->fan_table_start) { + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; + } + + duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + + if (duty100 == 0) { + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; + } + + tmp64 = (u64)rdev->pm.dpm.fan.pwm_min * duty100; + do_div(tmp64, 10000); + fdo_min = (u16)tmp64; + + t_diff1 = rdev->pm.dpm.fan.t_med - rdev->pm.dpm.fan.t_min; + t_diff2 = rdev->pm.dpm.fan.t_high - rdev->pm.dpm.fan.t_med; + + pwm_diff1 = rdev->pm.dpm.fan.pwm_med - rdev->pm.dpm.fan.pwm_min; + pwm_diff2 = rdev->pm.dpm.fan.pwm_high - rdev->pm.dpm.fan.pwm_med; + + slope1 = (u16)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100); + slope2 = (u16)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100); + + fan_table.temp_min = cpu_to_be16((50 + rdev->pm.dpm.fan.t_min) / 100); + fan_table.temp_med = cpu_to_be16((50 + rdev->pm.dpm.fan.t_med) / 100); + fan_table.temp_max = cpu_to_be16((50 + rdev->pm.dpm.fan.t_max) / 100); + + fan_table.slope1 = cpu_to_be16(slope1); + fan_table.slope2 = cpu_to_be16(slope2); + + fan_table.fdo_min = cpu_to_be16(fdo_min); + + fan_table.hys_down = cpu_to_be16(rdev->pm.dpm.fan.t_hyst); + + fan_table.hys_up = cpu_to_be16(1); + + fan_table.hys_slope = cpu_to_be16(1); + + fan_table.temp_resp_lim = cpu_to_be16(5); + + reference_clock = radeon_get_xclk(rdev); + + fan_table.refresh_period = cpu_to_be32((rdev->pm.dpm.fan.cycle_delay * + reference_clock) / 1600); + + fan_table.fdo_max = cpu_to_be16((u16)duty100); + + tmp = (RREG32(CG_MULT_THERMAL_CTRL) & TEMP_SEL_MASK) >> TEMP_SEL_SHIFT; + fan_table.temp_src = (uint8_t)tmp; + + ret = si_copy_bytes_to_smc(rdev, + si_pi->fan_table_start, + (u8 *)(&fan_table), + sizeof(fan_table), + si_pi->sram_end); + + if (ret) { + DRM_ERROR("Failed to load fan table to the SMC."); + rdev->pm.dpm.fan.ucode_fan_control = false; + } + + return 0; +} + +static int si_fan_ctrl_start_smc_fan_control(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + PPSMC_Result ret; + + ret = si_send_msg_to_smc(rdev, PPSMC_StartFanControl); + if (ret == PPSMC_Result_OK) { + si_pi->fan_is_controlled_by_smc = true; + return 0; + } else { + return -EINVAL; + } +} + +static int si_fan_ctrl_stop_smc_fan_control(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + PPSMC_Result ret; + + ret = si_send_msg_to_smc(rdev, PPSMC_StopFanControl); + + if (ret == PPSMC_Result_OK) { + si_pi->fan_is_controlled_by_smc = false; + return 0; + } else { + return -EINVAL; + } +} + +int si_fan_ctrl_get_fan_speed_percent(struct radeon_device *rdev, + u32 *speed) +{ + u32 duty, duty100; + u64 tmp64; + + if (rdev->pm.no_fan) + return -ENOENT; + + duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + duty = (RREG32(CG_THERMAL_STATUS) & FDO_PWM_DUTY_MASK) >> FDO_PWM_DUTY_SHIFT; + + if (duty100 == 0) + return -EINVAL; + + tmp64 = (u64)duty * 100; + do_div(tmp64, duty100); + *speed = (u32)tmp64; + + if (*speed > 100) + *speed = 100; + + return 0; +} + +int si_fan_ctrl_set_fan_speed_percent(struct radeon_device *rdev, + u32 speed) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 tmp; + u32 duty, duty100; + u64 tmp64; + + if (rdev->pm.no_fan) + return -ENOENT; + + if (si_pi->fan_is_controlled_by_smc) + return -EINVAL; + + if (speed > 100) + return -EINVAL; + + duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + + if (duty100 == 0) + return -EINVAL; + + tmp64 = (u64)speed * duty100; + do_div(tmp64, 100); + duty = (u32)tmp64; + + tmp = RREG32(CG_FDO_CTRL0) & ~FDO_STATIC_DUTY_MASK; + tmp |= FDO_STATIC_DUTY(duty); + WREG32(CG_FDO_CTRL0, tmp); + + return 0; +} + +void si_fan_ctrl_set_mode(struct radeon_device *rdev, u32 mode) +{ + if (mode) { + /* stop auto-manage */ + if (rdev->pm.dpm.fan.ucode_fan_control) + si_fan_ctrl_stop_smc_fan_control(rdev); + si_fan_ctrl_set_static_mode(rdev, mode); + } else { + /* restart auto-manage */ + if (rdev->pm.dpm.fan.ucode_fan_control) + si_thermal_start_smc_fan_control(rdev); + else + si_fan_ctrl_set_default_mode(rdev); + } +} + +u32 si_fan_ctrl_get_mode(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 tmp; + + if (si_pi->fan_is_controlled_by_smc) + return 0; + + tmp = RREG32(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK; + return (tmp >> FDO_PWM_MODE_SHIFT); +} + +#if 0 +static int si_fan_ctrl_get_fan_speed_rpm(struct radeon_device *rdev, + u32 *speed) +{ + u32 tach_period; + u32 xclk = radeon_get_xclk(rdev); + + if (rdev->pm.no_fan) + return -ENOENT; + + if (rdev->pm.fan_pulses_per_revolution == 0) + return -ENOENT; + + tach_period = (RREG32(CG_TACH_STATUS) & TACH_PERIOD_MASK) >> TACH_PERIOD_SHIFT; + if (tach_period == 0) + return -ENOENT; + + *speed = 60 * xclk * 10000 / tach_period; + + return 0; +} + +static int si_fan_ctrl_set_fan_speed_rpm(struct radeon_device *rdev, + u32 speed) +{ + u32 tach_period, tmp; + u32 xclk = radeon_get_xclk(rdev); + + if (rdev->pm.no_fan) + return -ENOENT; + + if (rdev->pm.fan_pulses_per_revolution == 0) + return -ENOENT; + + if ((speed < rdev->pm.fan_min_rpm) || + (speed > rdev->pm.fan_max_rpm)) + return -EINVAL; + + if (rdev->pm.dpm.fan.ucode_fan_control) + si_fan_ctrl_stop_smc_fan_control(rdev); + + tach_period = 60 * xclk * 10000 / (8 * speed); + tmp = RREG32(CG_TACH_CTRL) & ~TARGET_PERIOD_MASK; + tmp |= TARGET_PERIOD(tach_period); + WREG32(CG_TACH_CTRL, tmp); + + si_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC_RPM); + + return 0; +} +#endif + +static void si_fan_ctrl_set_default_mode(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 tmp; + + if (!si_pi->fan_ctrl_is_in_default_mode) { + tmp = RREG32(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK; + tmp |= FDO_PWM_MODE(si_pi->fan_ctrl_default_mode); + WREG32(CG_FDO_CTRL2, tmp); + + tmp = RREG32(CG_FDO_CTRL2) & ~TMIN_MASK; + tmp |= TMIN(si_pi->t_min); + WREG32(CG_FDO_CTRL2, tmp); + si_pi->fan_ctrl_is_in_default_mode = true; + } +} + +static void si_thermal_start_smc_fan_control(struct radeon_device *rdev) +{ + if (rdev->pm.dpm.fan.ucode_fan_control) { + si_fan_ctrl_start_smc_fan_control(rdev); + si_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC); + } +} + +static void si_thermal_initialize(struct radeon_device *rdev) +{ + u32 tmp; + + if (rdev->pm.fan_pulses_per_revolution) { + tmp = RREG32(CG_TACH_CTRL) & ~EDGE_PER_REV_MASK; + tmp |= EDGE_PER_REV(rdev->pm.fan_pulses_per_revolution -1); + WREG32(CG_TACH_CTRL, tmp); + } + + tmp = RREG32(CG_FDO_CTRL2) & ~TACH_PWM_RESP_RATE_MASK; + tmp |= TACH_PWM_RESP_RATE(0x28); + WREG32(CG_FDO_CTRL2, tmp); +} + +static int si_thermal_start_thermal_controller(struct radeon_device *rdev) +{ + int ret; + + si_thermal_initialize(rdev); + ret = si_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) + return ret; + ret = si_thermal_enable_alert(rdev, true); + if (ret) + return ret; + if (rdev->pm.dpm.fan.ucode_fan_control) { + ret = si_halt_smc(rdev); + if (ret) + return ret; + ret = si_thermal_setup_fan_table(rdev); + if (ret) + return ret; + ret = si_resume_smc(rdev); + if (ret) + return ret; + si_thermal_start_smc_fan_control(rdev); + } + + return 0; +} + +static void si_thermal_stop_thermal_controller(struct radeon_device *rdev) +{ + if (!rdev->pm.no_fan) { + si_fan_ctrl_set_default_mode(rdev); + si_fan_ctrl_stop_smc_fan_control(rdev); + } +} + +int si_dpm_enable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + int ret; + + if (si_is_smc_running(rdev)) + return -EINVAL; + if (pi->voltage_control || si_pi->voltage_control_svi2) + si_enable_voltage_control(rdev, true); + if (pi->mvdd_control) + si_get_mvdd_configuration(rdev); + if (pi->voltage_control || si_pi->voltage_control_svi2) { + ret = si_construct_voltage_tables(rdev); + if (ret) { + DRM_ERROR("si_construct_voltage_tables failed\n"); + return ret; + } + } + if (eg_pi->dynamic_ac_timing) { + ret = si_initialize_mc_reg_table(rdev); + if (ret) + eg_pi->dynamic_ac_timing = false; + } + if (pi->dynamic_ss) + si_enable_spread_spectrum(rdev, true); + if (pi->thermal_protection) + si_enable_thermal_protection(rdev, true); + si_setup_bsp(rdev); + si_program_git(rdev); + si_program_tp(rdev); + si_program_tpp(rdev); + si_program_sstp(rdev); + si_enable_display_gap(rdev); + si_program_vc(rdev); + ret = si_upload_firmware(rdev); + if (ret) { + DRM_ERROR("si_upload_firmware failed\n"); + return ret; + } + ret = si_process_firmware_header(rdev); + if (ret) { + DRM_ERROR("si_process_firmware_header failed\n"); + return ret; + } + ret = si_initial_switch_from_arb_f0_to_f1(rdev); + if (ret) { + DRM_ERROR("si_initial_switch_from_arb_f0_to_f1 failed\n"); + return ret; + } + ret = si_init_smc_table(rdev); + if (ret) { + DRM_ERROR("si_init_smc_table failed\n"); + return ret; + } + ret = si_init_smc_spll_table(rdev); + if (ret) { + DRM_ERROR("si_init_smc_spll_table failed\n"); + return ret; + } + ret = si_init_arb_table_index(rdev); + if (ret) { + DRM_ERROR("si_init_arb_table_index failed\n"); + return ret; + } + if (eg_pi->dynamic_ac_timing) { + ret = si_populate_mc_reg_table(rdev, boot_ps); + if (ret) { + DRM_ERROR("si_populate_mc_reg_table failed\n"); + return ret; + } + } + ret = si_initialize_smc_cac_tables(rdev); + if (ret) { + DRM_ERROR("si_initialize_smc_cac_tables failed\n"); + return ret; + } + ret = si_initialize_hardware_cac_manager(rdev); + if (ret) { + DRM_ERROR("si_initialize_hardware_cac_manager failed\n"); + return ret; + } + ret = si_initialize_smc_dte_tables(rdev); + if (ret) { + DRM_ERROR("si_initialize_smc_dte_tables failed\n"); + return ret; + } + ret = si_populate_smc_tdp_limits(rdev, boot_ps); + if (ret) { + DRM_ERROR("si_populate_smc_tdp_limits failed\n"); + return ret; + } + ret = si_populate_smc_tdp_limits_2(rdev, boot_ps); + if (ret) { + DRM_ERROR("si_populate_smc_tdp_limits_2 failed\n"); + return ret; + } + si_program_response_times(rdev); + si_program_ds_registers(rdev); + si_dpm_start_smc(rdev); + ret = si_notify_smc_display_change(rdev, false); + if (ret) { + DRM_ERROR("si_notify_smc_display_change failed\n"); + return ret; + } + si_enable_sclk_control(rdev, true); + si_start_dpm(rdev); + + si_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + + si_thermal_start_thermal_controller(rdev); + + ni_update_current_ps(rdev, boot_ps); + + return 0; +} + +static int si_set_temperature_range(struct radeon_device *rdev) +{ + int ret; + + ret = si_thermal_enable_alert(rdev, false); + if (ret) + return ret; + ret = si_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) + return ret; + ret = si_thermal_enable_alert(rdev, true); + if (ret) + return ret; + + return ret; +} + +int si_dpm_late_enable(struct radeon_device *rdev) +{ + int ret; + + ret = si_set_temperature_range(rdev); + if (ret) + return ret; + + return ret; +} + +void si_dpm_disable(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps; + + if (!si_is_smc_running(rdev)) + return; + si_thermal_stop_thermal_controller(rdev); + si_disable_ulv(rdev); + si_clear_vc(rdev); + if (pi->thermal_protection) + si_enable_thermal_protection(rdev, false); + si_enable_power_containment(rdev, boot_ps, false); + si_enable_smc_cac(rdev, boot_ps, false); + si_enable_spread_spectrum(rdev, false); + si_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false); + si_stop_dpm(rdev); + si_reset_to_default(rdev); + si_dpm_stop_smc(rdev); + si_force_switch_to_arb_f0(rdev); + + ni_update_current_ps(rdev, boot_ps); +} + +int si_dpm_pre_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps; + struct radeon_ps *new_ps = &requested_ps; + + ni_update_requested_ps(rdev, new_ps); + + si_apply_state_adjust_rules(rdev, &eg_pi->requested_rps); + + return 0; +} + +static int si_power_control_set_level(struct radeon_device *rdev) +{ + struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps; + int ret; + + ret = si_restrict_performance_levels_before_switch(rdev); + if (ret) + return ret; + ret = si_halt_smc(rdev); + if (ret) + return ret; + ret = si_populate_smc_tdp_limits(rdev, new_ps); + if (ret) + return ret; + ret = si_populate_smc_tdp_limits_2(rdev, new_ps); + if (ret) + return ret; + ret = si_resume_smc(rdev); + if (ret) + return ret; + ret = si_set_sw_state(rdev); + if (ret) + return ret; + return 0; +} + +int si_dpm_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *new_ps = &eg_pi->requested_rps; + struct radeon_ps *old_ps = &eg_pi->current_rps; + int ret; + + ret = si_disable_ulv(rdev); + if (ret) { + DRM_ERROR("si_disable_ulv failed\n"); + return ret; + } + ret = si_restrict_performance_levels_before_switch(rdev); + if (ret) { + DRM_ERROR("si_restrict_performance_levels_before_switch failed\n"); + return ret; + } + if (eg_pi->pcie_performance_request) + si_request_link_speed_change_before_state_change(rdev, new_ps, old_ps); + ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps); + ret = si_enable_power_containment(rdev, new_ps, false); + if (ret) { + DRM_ERROR("si_enable_power_containment failed\n"); + return ret; + } + ret = si_enable_smc_cac(rdev, new_ps, false); + if (ret) { + DRM_ERROR("si_enable_smc_cac failed\n"); + return ret; + } + ret = si_halt_smc(rdev); + if (ret) { + DRM_ERROR("si_halt_smc failed\n"); + return ret; + } + ret = si_upload_sw_state(rdev, new_ps); + if (ret) { + DRM_ERROR("si_upload_sw_state failed\n"); + return ret; + } + ret = si_upload_smc_data(rdev); + if (ret) { + DRM_ERROR("si_upload_smc_data failed\n"); + return ret; + } + ret = si_upload_ulv_state(rdev); + if (ret) { + DRM_ERROR("si_upload_ulv_state failed\n"); + return ret; + } + if (eg_pi->dynamic_ac_timing) { + ret = si_upload_mc_reg_table(rdev, new_ps); + if (ret) { + DRM_ERROR("si_upload_mc_reg_table failed\n"); + return ret; + } + } + ret = si_program_memory_timing_parameters(rdev, new_ps); + if (ret) { + DRM_ERROR("si_program_memory_timing_parameters failed\n"); + return ret; + } + si_set_pcie_lane_width_in_smc(rdev, new_ps, old_ps); + + ret = si_resume_smc(rdev); + if (ret) { + DRM_ERROR("si_resume_smc failed\n"); + return ret; + } + ret = si_set_sw_state(rdev); + if (ret) { + DRM_ERROR("si_set_sw_state failed\n"); + return ret; + } + ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps); + si_set_vce_clock(rdev, new_ps, old_ps); + if (eg_pi->pcie_performance_request) + si_notify_link_speed_change_after_state_change(rdev, new_ps, old_ps); + ret = si_set_power_state_conditionally_enable_ulv(rdev, new_ps); + if (ret) { + DRM_ERROR("si_set_power_state_conditionally_enable_ulv failed\n"); + return ret; + } + ret = si_enable_smc_cac(rdev, new_ps, true); + if (ret) { + DRM_ERROR("si_enable_smc_cac failed\n"); + return ret; + } + ret = si_enable_power_containment(rdev, new_ps, true); + if (ret) { + DRM_ERROR("si_enable_power_containment failed\n"); + return ret; + } + + ret = si_power_control_set_level(rdev); + if (ret) { + DRM_ERROR("si_power_control_set_level failed\n"); + return ret; + } + + return 0; +} + +void si_dpm_post_set_power_state(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *new_ps = &eg_pi->requested_rps; + + ni_update_current_ps(rdev, new_ps); +} + +#if 0 +void si_dpm_reset_asic(struct radeon_device *rdev) +{ + si_restrict_performance_levels_before_switch(rdev); + si_disable_ulv(rdev); + si_set_boot_state(rdev); +} +#endif + +void si_dpm_display_configuration_changed(struct radeon_device *rdev) +{ + si_program_display_gap(rdev); +} + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; +}; + +union pplib_clock_info { + struct _ATOM_PPLIB_R600_CLOCK_INFO r600; + struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; + struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; + struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; + struct _ATOM_PPLIB_SI_CLOCK_INFO si; +}; + +union pplib_power_state { + struct _ATOM_PPLIB_STATE v1; + struct _ATOM_PPLIB_STATE_V2 v2; +}; + +static void si_parse_pplib_non_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info, + u8 table_rev) +{ + rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); + rps->class = le16_to_cpu(non_clock_info->usClassification); + rps->class2 = le16_to_cpu(non_clock_info->usClassification2); + + if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) { + rps->vclk = le32_to_cpu(non_clock_info->ulVCLK); + rps->dclk = le32_to_cpu(non_clock_info->ulDCLK); + } else if (r600_is_uvd_state(rps->class, rps->class2)) { + rps->vclk = RV770_DEFAULT_VCLK_FREQ; + rps->dclk = RV770_DEFAULT_DCLK_FREQ; + } else { + rps->vclk = 0; + rps->dclk = 0; + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) + rdev->pm.dpm.boot_ps = rps; + if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + rdev->pm.dpm.uvd_ps = rps; +} + +static void si_parse_pplib_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, int index, + union pplib_clock_info *clock_info) +{ + struct rv7xx_power_info *pi = rv770_get_pi(rdev); + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct si_power_info *si_pi = si_get_pi(rdev); + struct ni_ps *ps = ni_get_ps(rps); + u16 leakage_voltage; + struct rv7xx_pl *pl = &ps->performance_levels[index]; + int ret; + + ps->performance_level_count = index + 1; + + pl->sclk = le16_to_cpu(clock_info->si.usEngineClockLow); + pl->sclk |= clock_info->si.ucEngineClockHigh << 16; + pl->mclk = le16_to_cpu(clock_info->si.usMemoryClockLow); + pl->mclk |= clock_info->si.ucMemoryClockHigh << 16; + + pl->vddc = le16_to_cpu(clock_info->si.usVDDC); + pl->vddci = le16_to_cpu(clock_info->si.usVDDCI); + pl->flags = le32_to_cpu(clock_info->si.ulFlags); + pl->pcie_gen = r600_get_pcie_gen_support(rdev, + si_pi->sys_pcie_mask, + si_pi->boot_pcie_gen, + clock_info->si.ucPCIEGen); + + /* patch up vddc if necessary */ + ret = si_get_leakage_voltage_from_leakage_index(rdev, pl->vddc, + &leakage_voltage); + if (ret == 0) + pl->vddc = leakage_voltage; + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) { + pi->acpi_vddc = pl->vddc; + eg_pi->acpi_vddci = pl->vddci; + si_pi->acpi_pcie_gen = pl->pcie_gen; + } + + if ((rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) && + index == 0) { + /* XXX disable for A0 tahiti */ + si_pi->ulv.supported = false; + si_pi->ulv.pl = *pl; + si_pi->ulv.one_pcie_lane_in_ulv = false; + si_pi->ulv.volt_change_delay = SISLANDS_ULVVOLTAGECHANGEDELAY_DFLT; + si_pi->ulv.cg_ulv_parameter = SISLANDS_CGULVPARAMETER_DFLT; + si_pi->ulv.cg_ulv_control = SISLANDS_CGULVCONTROL_DFLT; + } + + if (pi->min_vddc_in_table > pl->vddc) + pi->min_vddc_in_table = pl->vddc; + + if (pi->max_vddc_in_table < pl->vddc) + pi->max_vddc_in_table = pl->vddc; + + /* patch up boot state */ + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { + u16 vddc, vddci, mvdd; + radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd); + pl->mclk = rdev->clock.default_mclk; + pl->sclk = rdev->clock.default_sclk; + pl->vddc = vddc; + pl->vddci = vddci; + si_pi->mvdd_bootup_value = mvdd; + } + + if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == + ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) { + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc; + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci; + } +} + +static int si_parse_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; + union pplib_power_state *power_state; + int i, j, k, non_clock_array_index, clock_array_index; + union pplib_clock_info *clock_info; + struct _StateArray *state_array; + struct _ClockInfoArray *clock_info_array; + struct _NonClockInfoArray *non_clock_info_array; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + u8 *power_state_offset; + struct ni_ps *ps; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + state_array = (struct _StateArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usStateArrayOffset)); + clock_info_array = (struct _ClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usClockInfoArrayOffset)); + non_clock_info_array = (struct _NonClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset)); + + rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) * + state_array->ucNumEntries, GFP_KERNEL); + if (!rdev->pm.dpm.ps) + return -ENOMEM; + power_state_offset = (u8 *)state_array->states; + for (i = 0; i < state_array->ucNumEntries; i++) { + u8 *idx; + power_state = (union pplib_power_state *)power_state_offset; + non_clock_array_index = power_state->v2.nonClockInfoIndex; + non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) + &non_clock_info_array->nonClockInfo[non_clock_array_index]; + if (!rdev->pm.power_state[i].clock_info) + return -EINVAL; + ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL); + if (ps == NULL) { + kfree(rdev->pm.dpm.ps); + return -ENOMEM; + } + rdev->pm.dpm.ps[i].ps_priv = ps; + si_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i], + non_clock_info, + non_clock_info_array->ucEntrySize); + k = 0; + idx = (u8 *)&power_state->v2.clockInfoIndex[0]; + for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) { + clock_array_index = idx[j]; + if (clock_array_index >= clock_info_array->ucNumEntries) + continue; + if (k >= SISLANDS_MAX_HARDWARE_POWERLEVELS) + break; + clock_info = (union pplib_clock_info *) + ((u8 *)&clock_info_array->clockInfo[0] + + (clock_array_index * clock_info_array->ucEntrySize)); + si_parse_pplib_clock_info(rdev, + &rdev->pm.dpm.ps[i], k, + clock_info); + k++; + } + power_state_offset += 2 + power_state->v2.ucNumDPMLevels; + } + rdev->pm.dpm.num_ps = state_array->ucNumEntries; + + /* fill in the vce power states */ + for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) { + u32 sclk, mclk; + clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx; + clock_info = (union pplib_clock_info *) + &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize]; + sclk = le16_to_cpu(clock_info->si.usEngineClockLow); + sclk |= clock_info->si.ucEngineClockHigh << 16; + mclk = le16_to_cpu(clock_info->si.usMemoryClockLow); + mclk |= clock_info->si.ucMemoryClockHigh << 16; + rdev->pm.dpm.vce_states[i].sclk = sclk; + rdev->pm.dpm.vce_states[i].mclk = mclk; + } + + return 0; +} + +int si_dpm_init(struct radeon_device *rdev) +{ + struct rv7xx_power_info *pi; + struct evergreen_power_info *eg_pi; + struct ni_power_info *ni_pi; + struct si_power_info *si_pi; + struct atom_clock_dividers dividers; + int ret; + u32 mask; + + si_pi = kzalloc(sizeof(struct si_power_info), GFP_KERNEL); + if (si_pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = si_pi; + ni_pi = &si_pi->ni; + eg_pi = &ni_pi->eg; + pi = &eg_pi->rv7xx; + + ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); + if (ret) + si_pi->sys_pcie_mask = 0; + else + si_pi->sys_pcie_mask = mask; + si_pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID; + si_pi->boot_pcie_gen = si_get_current_pcie_speed(rdev); + + si_set_max_cu_value(rdev); + + rv770_get_max_vddc(rdev); + si_get_leakage_vddc(rdev); + si_patch_dependency_tables_based_on_leakage(rdev); + + pi->acpi_vddc = 0; + eg_pi->acpi_vddci = 0; + pi->min_vddc_in_table = 0; + pi->max_vddc_in_table = 0; + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = r600_parse_extended_power_table(rdev); + if (ret) + return ret; + + ret = si_parse_power_table(rdev); + if (ret) + return ret; + + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries = + kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL); + if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) { + r600_free_extended_power_table(rdev); + return -ENOMEM; + } + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000; + rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900; + + if (rdev->pm.dpm.voltage_response_time == 0) + rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT; + if (rdev->pm.dpm.backbias_response_time == 0) + rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + 0, false, ÷rs); + if (ret) + pi->ref_div = dividers.ref_div + 1; + else + pi->ref_div = R600_REFERENCEDIVIDER_DFLT; + + eg_pi->smu_uvd_hs = false; + + pi->mclk_strobe_mode_threshold = 40000; + if (si_is_special_1gb_platform(rdev)) + pi->mclk_stutter_mode_threshold = 0; + else + pi->mclk_stutter_mode_threshold = pi->mclk_strobe_mode_threshold; + pi->mclk_edc_enable_threshold = 40000; + eg_pi->mclk_edc_wr_enable_threshold = 40000; + + ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold; + + pi->voltage_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, + VOLTAGE_OBJ_GPIO_LUT); + if (!pi->voltage_control) { + si_pi->voltage_control_svi2 = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, + VOLTAGE_OBJ_SVID2); + if (si_pi->voltage_control_svi2) + radeon_atom_get_svi2_info(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, + &si_pi->svd_gpio_id, &si_pi->svc_gpio_id); + } + + pi->mvdd_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, + VOLTAGE_OBJ_GPIO_LUT); + + eg_pi->vddci_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, + VOLTAGE_OBJ_GPIO_LUT); + if (!eg_pi->vddci_control) + si_pi->vddci_control_svi2 = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, + VOLTAGE_OBJ_SVID2); + + si_pi->vddc_phase_shed_control = + radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, + VOLTAGE_OBJ_PHASE_LUT); + + rv770_get_engine_memory_ss(rdev); + + pi->asi = RV770_ASI_DFLT; + pi->pasi = CYPRESS_HASI_DFLT; + pi->vrc = SISLANDS_VRC_DFLT; + + pi->gfx_clock_gating = true; + + eg_pi->sclk_deep_sleep = true; + si_pi->sclk_deep_sleep_above_low = false; + + if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE) + pi->thermal_protection = true; + else + pi->thermal_protection = false; + + eg_pi->dynamic_ac_timing = true; + + eg_pi->light_sleep = true; +#if defined(CONFIG_ACPI) + eg_pi->pcie_performance_request = + radeon_acpi_is_pcie_performance_request_supported(rdev); +#else + eg_pi->pcie_performance_request = false; +#endif + + si_pi->sram_end = SMC_RAM_END; + + rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 4; + rdev->pm.dpm.dyn_state.sclk_mclk_delta = 15000; + rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200; + rdev->pm.dpm.dyn_state.valid_sclk_values.count = 0; + rdev->pm.dpm.dyn_state.valid_sclk_values.values = NULL; + rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0; + rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL; + + si_initialize_powertune_defaults(rdev); + + /* make sure dc limits are valid */ + if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) || + (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0)) + rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc = + rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + + si_pi->fan_ctrl_is_in_default_mode = true; + + return 0; +} + +void si_dpm_fini(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); + kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries); + r600_free_extended_power_table(rdev); +} + +void si_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *rps = &eg_pi->current_rps; + struct ni_ps *ps = ni_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >> + CURRENT_STATE_INDEX_SHIFT; + + if (current_index >= ps->performance_level_count) { + seq_printf(m, "invalid dpm profile %d\n", current_index); + } else { + pl = &ps->performance_levels[current_index]; + seq_printf(m, "uvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + seq_printf(m, "power level %d sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n", + current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1); + } +} + +u32 si_dpm_get_current_sclk(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *rps = &eg_pi->current_rps; + struct ni_ps *ps = ni_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >> + CURRENT_STATE_INDEX_SHIFT; + + if (current_index >= ps->performance_level_count) { + return 0; + } else { + pl = &ps->performance_levels[current_index]; + return pl->sclk; + } +} + +u32 si_dpm_get_current_mclk(struct radeon_device *rdev) +{ + struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev); + struct radeon_ps *rps = &eg_pi->current_rps; + struct ni_ps *ps = ni_get_ps(rps); + struct rv7xx_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >> + CURRENT_STATE_INDEX_SHIFT; + + if (current_index >= ps->performance_level_count) { + return 0; + } else { + pl = &ps->performance_levels[current_index]; + return pl->mclk; + } +} diff --git a/sys/dev/pci/drm/radeon/si_dpm.h b/sys/dev/pci/drm/radeon/si_dpm.h new file mode 100644 index 00000000000..1032a68be79 --- /dev/null +++ b/sys/dev/pci/drm/radeon/si_dpm.h @@ -0,0 +1,238 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __SI_DPM_H__ +#define __SI_DPM_H__ + +#include "ni_dpm.h" +#include "sislands_smc.h" + +enum si_cac_config_reg_type +{ + SISLANDS_CACCONFIG_MMR = 0, + SISLANDS_CACCONFIG_CGIND, + SISLANDS_CACCONFIG_MAX +}; + +struct si_cac_config_reg +{ + u32 offset; + u32 mask; + u32 shift; + u32 value; + enum si_cac_config_reg_type type; +}; + +struct si_powertune_data +{ + u32 cac_window; + u32 l2_lta_window_size_default; + u8 lts_truncate_default; + u8 shift_n_default; + u8 operating_temp; + struct ni_leakage_coeffients leakage_coefficients; + u32 fixed_kt; + u32 lkge_lut_v0_percent; + u8 dc_cac[NISLANDS_DCCAC_MAX_LEVELS]; + bool enable_powertune_by_default; +}; + +struct si_dyn_powertune_data +{ + u32 cac_leakage; + s32 leakage_minimum_temperature; + u32 wintime; + u32 l2_lta_window_size; + u8 lts_truncate; + u8 shift_n; + u8 dc_pwr_value; + bool disable_uvd_powertune; +}; + +struct si_dte_data +{ + u32 tau[SMC_SISLANDS_DTE_MAX_FILTER_STAGES]; + u32 r[SMC_SISLANDS_DTE_MAX_FILTER_STAGES]; + u32 k; + u32 t0; + u32 max_t; + u8 window_size; + u8 temp_select; + u8 dte_mode; + u8 tdep_count; + u8 t_limits[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE]; + u32 tdep_tau[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE]; + u32 tdep_r[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE]; + u32 t_threshold; + bool enable_dte_by_default; +}; + +struct si_clock_registers { + u32 cg_spll_func_cntl; + u32 cg_spll_func_cntl_2; + u32 cg_spll_func_cntl_3; + u32 cg_spll_func_cntl_4; + u32 cg_spll_spread_spectrum; + u32 cg_spll_spread_spectrum_2; + u32 dll_cntl; + u32 mclk_pwrmgt_cntl; + u32 mpll_ad_func_cntl; + u32 mpll_dq_func_cntl; + u32 mpll_func_cntl; + u32 mpll_func_cntl_1; + u32 mpll_func_cntl_2; + u32 mpll_ss1; + u32 mpll_ss2; +}; + +struct si_mc_reg_entry { + u32 mclk_max; + u32 mc_data[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE]; +}; + +struct si_mc_reg_table { + u8 last; + u8 num_entries; + u16 valid_flag; + struct si_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES]; + SMC_NIslands_MCRegisterAddress mc_reg_address[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE]; +}; + +#define SISLANDS_MCREGISTERTABLE_INITIAL_SLOT 0 +#define SISLANDS_MCREGISTERTABLE_ACPI_SLOT 1 +#define SISLANDS_MCREGISTERTABLE_ULV_SLOT 2 +#define SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT 3 + +struct si_leakage_voltage_entry +{ + u16 voltage; + u16 leakage_index; +}; + +#define SISLANDS_LEAKAGE_INDEX0 0xff01 +#define SISLANDS_MAX_LEAKAGE_COUNT 4 + +struct si_leakage_voltage +{ + u16 count; + struct si_leakage_voltage_entry entries[SISLANDS_MAX_LEAKAGE_COUNT]; +}; + +#define SISLANDS_MAX_HARDWARE_POWERLEVELS 5 + +struct si_ulv_param { + bool supported; + u32 cg_ulv_control; + u32 cg_ulv_parameter; + u32 volt_change_delay; + struct rv7xx_pl pl; + bool one_pcie_lane_in_ulv; +}; + +struct si_power_info { + /* must be first! */ + struct ni_power_info ni; + struct si_clock_registers clock_registers; + struct si_mc_reg_table mc_reg_table; + struct atom_voltage_table mvdd_voltage_table; + struct atom_voltage_table vddc_phase_shed_table; + struct si_leakage_voltage leakage_voltage; + u16 mvdd_bootup_value; + struct si_ulv_param ulv; + u32 max_cu; + /* pcie gen */ + enum radeon_pcie_gen force_pcie_gen; + enum radeon_pcie_gen boot_pcie_gen; + enum radeon_pcie_gen acpi_pcie_gen; + u32 sys_pcie_mask; + /* flags */ + bool enable_dte; + bool enable_ppm; + bool vddc_phase_shed_control; + bool pspp_notify_required; + bool sclk_deep_sleep_above_low; + bool voltage_control_svi2; + bool vddci_control_svi2; + /* smc offsets */ + u32 sram_end; + u32 state_table_start; + u32 soft_regs_start; + u32 mc_reg_table_start; + u32 arb_table_start; + u32 cac_table_start; + u32 dte_table_start; + u32 spll_table_start; + u32 papm_cfg_table_start; + u32 fan_table_start; + /* CAC stuff */ + const struct si_cac_config_reg *cac_weights; + const struct si_cac_config_reg *lcac_config; + const struct si_cac_config_reg *cac_override; + const struct si_powertune_data *powertune_data; + struct si_dyn_powertune_data dyn_powertune_data; + /* DTE stuff */ + struct si_dte_data dte_data; + /* scratch structs */ + SMC_SIslands_MCRegisters smc_mc_reg_table; + SISLANDS_SMC_STATETABLE smc_statetable; + PP_SIslands_PAPMParameters papm_parm; + /* SVI2 */ + u8 svd_gpio_id; + u8 svc_gpio_id; + /* fan control */ + bool fan_ctrl_is_in_default_mode; + u32 t_min; + u32 fan_ctrl_default_mode; + bool fan_is_controlled_by_smc; +}; + +#define SISLANDS_INITIAL_STATE_ARB_INDEX 0 +#define SISLANDS_ACPI_STATE_ARB_INDEX 1 +#define SISLANDS_ULV_STATE_ARB_INDEX 2 +#define SISLANDS_DRIVER_STATE_ARB_INDEX 3 + +#define SISLANDS_DPM2_MAX_PULSE_SKIP 256 + +#define SISLANDS_DPM2_NEAR_TDP_DEC 10 +#define SISLANDS_DPM2_ABOVE_SAFE_INC 5 +#define SISLANDS_DPM2_BELOW_SAFE_INC 20 + +#define SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT 80 + +#define SISLANDS_DPM2_MAXPS_PERCENT_H 99 +#define SISLANDS_DPM2_MAXPS_PERCENT_M 99 + +#define SISLANDS_DPM2_SQ_RAMP_MAX_POWER 0x3FFF +#define SISLANDS_DPM2_SQ_RAMP_MIN_POWER 0x12 +#define SISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA 0x15 +#define SISLANDS_DPM2_SQ_RAMP_STI_SIZE 0x1E +#define SISLANDS_DPM2_SQ_RAMP_LTI_RATIO 0xF + +#define SISLANDS_DPM2_PWREFFICIENCYRATIO_MARGIN 10 + +#define SISLANDS_VRC_DFLT 0xC000B3 +#define SISLANDS_ULVVOLTAGECHANGEDELAY_DFLT 1687 +#define SISLANDS_CGULVPARAMETER_DFLT 0x00040035 +#define SISLANDS_CGULVCONTROL_DFLT 0x1f007550 + + +#endif diff --git a/sys/dev/pci/drm/radeon/si_reg.h b/sys/dev/pci/drm/radeon/si_reg.h index ba6918a46cf..501f9d431d5 100644 --- a/sys/dev/pci/drm/radeon/si_reg.h +++ b/sys/dev/pci/drm/radeon/si_reg.h @@ -1,4 +1,3 @@ -/* $OpenBSD: si_reg.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2010 Advanced Micro Devices, Inc. * diff --git a/sys/dev/pci/drm/radeon/si_smc.c b/sys/dev/pci/drm/radeon/si_smc.c new file mode 100644 index 00000000000..1e76cfb7aee --- /dev/null +++ b/sys/dev/pci/drm/radeon/si_smc.c @@ -0,0 +1,309 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include +#include "radeon.h" +#include "sid.h" +#include "ppsmc.h" +#include "radeon_ucode.h" +#include "sislands_smc.h" + +static int si_set_smc_sram_address(struct radeon_device *rdev, + u32 smc_address, u32 limit) +{ + if (smc_address & 3) + return -EINVAL; + if ((smc_address + 3) > limit) + return -EINVAL; + + WREG32(SMC_IND_INDEX_0, smc_address); + WREG32_P(SMC_IND_ACCESS_CNTL, 0, ~AUTO_INCREMENT_IND_0); + + return 0; +} + +int si_copy_bytes_to_smc(struct radeon_device *rdev, + u32 smc_start_address, + const u8 *src, u32 byte_count, u32 limit) +{ + unsigned long flags; + int ret = 0; + u32 data, original_data, addr, extra_shift; + + if (smc_start_address & 3) + return -EINVAL; + if ((smc_start_address + byte_count) > limit) + return -EINVAL; + + addr = smc_start_address; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + while (byte_count >= 4) { + /* SMC address space is BE */ + data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3]; + + ret = si_set_smc_sram_address(rdev, addr, limit); + if (ret) + goto done; + + WREG32(SMC_IND_DATA_0, data); + + src += 4; + byte_count -= 4; + addr += 4; + } + + /* RMW for the final bytes */ + if (byte_count > 0) { + data = 0; + + ret = si_set_smc_sram_address(rdev, addr, limit); + if (ret) + goto done; + + original_data = RREG32(SMC_IND_DATA_0); + + extra_shift = 8 * (4 - byte_count); + + while (byte_count > 0) { + /* SMC address space is BE */ + data = (data << 8) + *src++; + byte_count--; + } + + data <<= extra_shift; + + data |= (original_data & ~((~0UL) << extra_shift)); + + ret = si_set_smc_sram_address(rdev, addr, limit); + if (ret) + goto done; + + WREG32(SMC_IND_DATA_0, data); + } + +done: + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return ret; +} + +void si_start_smc(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(SMC_SYSCON_RESET_CNTL); + + tmp &= ~RST_REG; + + WREG32_SMC(SMC_SYSCON_RESET_CNTL, tmp); +} + +void si_reset_smc(struct radeon_device *rdev) +{ + u32 tmp; + + RREG32(CB_CGTT_SCLK_CTRL); + RREG32(CB_CGTT_SCLK_CTRL); + RREG32(CB_CGTT_SCLK_CTRL); + RREG32(CB_CGTT_SCLK_CTRL); + + tmp = RREG32_SMC(SMC_SYSCON_RESET_CNTL); + tmp |= RST_REG; + WREG32_SMC(SMC_SYSCON_RESET_CNTL, tmp); +} + +int si_program_jump_on_start(struct radeon_device *rdev) +{ + static const u8 data[] = { 0x0E, 0x00, 0x40, 0x40 }; + + return si_copy_bytes_to_smc(rdev, 0x0, data, 4, sizeof(data)+1); +} + +void si_stop_smc_clock(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0); + + tmp |= CK_DISABLE; + + WREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0, tmp); +} + +void si_start_smc_clock(struct radeon_device *rdev) +{ + u32 tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0); + + tmp &= ~CK_DISABLE; + + WREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0, tmp); +} + +bool si_is_smc_running(struct radeon_device *rdev) +{ + u32 rst = RREG32_SMC(SMC_SYSCON_RESET_CNTL); + u32 clk = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0); + + if (!(rst & RST_REG) && !(clk & CK_DISABLE)) + return true; + + return false; +} + +PPSMC_Result si_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg) +{ + u32 tmp; + int i; + + if (!si_is_smc_running(rdev)) + return PPSMC_Result_Failed; + + WREG32(SMC_MESSAGE_0, msg); + + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = RREG32(SMC_RESP_0); + if (tmp != 0) + break; + udelay(1); + } + tmp = RREG32(SMC_RESP_0); + + return (PPSMC_Result)tmp; +} + +PPSMC_Result si_wait_for_smc_inactive(struct radeon_device *rdev) +{ + u32 tmp; + int i; + + if (!si_is_smc_running(rdev)) + return PPSMC_Result_OK; + + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0); + if ((tmp & CKEN) == 0) + break; + udelay(1); + } + + return PPSMC_Result_OK; +} + +int si_load_smc_ucode(struct radeon_device *rdev, u32 limit) +{ + unsigned long flags; + u32 ucode_start_address; + u32 ucode_size; + const u8 *src; + u32 data; + + if (!rdev->smc_fw) + return -EINVAL; + + if (rdev->new_fw) { + const struct smc_firmware_header_v1_0 *hdr = + (const struct smc_firmware_header_v1_0 *)rdev->smc_fw->data; + + radeon_ucode_print_smc_hdr(&hdr->header); + + ucode_start_address = le32_to_cpu(hdr->ucode_start_addr); + ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes); + src = (const u8 *) + (rdev->smc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + } else { + switch (rdev->family) { + case CHIP_TAHITI: + ucode_start_address = TAHITI_SMC_UCODE_START; + ucode_size = TAHITI_SMC_UCODE_SIZE; + break; + case CHIP_PITCAIRN: + ucode_start_address = PITCAIRN_SMC_UCODE_START; + ucode_size = PITCAIRN_SMC_UCODE_SIZE; + break; + case CHIP_VERDE: + ucode_start_address = VERDE_SMC_UCODE_START; + ucode_size = VERDE_SMC_UCODE_SIZE; + break; + case CHIP_OLAND: + ucode_start_address = OLAND_SMC_UCODE_START; + ucode_size = OLAND_SMC_UCODE_SIZE; + break; + case CHIP_HAINAN: + ucode_start_address = HAINAN_SMC_UCODE_START; + ucode_size = HAINAN_SMC_UCODE_SIZE; + break; + default: + DRM_ERROR("unknown asic in smc ucode loader\n"); + BUG(); + } + src = (const u8 *)rdev->smc_fw->data; + } + + if (ucode_size & 3) + return -EINVAL; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + WREG32(SMC_IND_INDEX_0, ucode_start_address); + WREG32_P(SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, ~AUTO_INCREMENT_IND_0); + while (ucode_size >= 4) { + /* SMC address space is BE */ + data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3]; + + WREG32(SMC_IND_DATA_0, data); + + src += 4; + ucode_size -= 4; + } + WREG32_P(SMC_IND_ACCESS_CNTL, 0, ~AUTO_INCREMENT_IND_0); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return 0; +} + +int si_read_smc_sram_dword(struct radeon_device *rdev, u32 smc_address, + u32 *value, u32 limit) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + ret = si_set_smc_sram_address(rdev, smc_address, limit); + if (ret == 0) + *value = RREG32(SMC_IND_DATA_0); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return ret; +} + +int si_write_smc_sram_dword(struct radeon_device *rdev, u32 smc_address, + u32 value, u32 limit) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&rdev->smc_idx_lock, flags); + ret = si_set_smc_sram_address(rdev, smc_address, limit); + if (ret == 0) + WREG32(SMC_IND_DATA_0, value); + spin_unlock_irqrestore(&rdev->smc_idx_lock, flags); + + return ret; +} diff --git a/sys/dev/pci/drm/radeon/sid.h b/sys/dev/pci/drm/radeon/sid.h index 132d48ff337..d1a7b58dd29 100644 --- a/sys/dev/pci/drm/radeon/sid.h +++ b/sys/dev/pci/drm/radeon/sid.h @@ -1,4 +1,3 @@ -/* $OpenBSD: sid.h,v 1.6 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright 2011 Advanced Micro Devices, Inc. * @@ -29,14 +28,7 @@ #define TAHITI_GB_ADDR_CONFIG_GOLDEN 0x12011003 #define VERDE_GB_ADDR_CONFIG_GOLDEN 0x12010002 - -#define CG_MULT_THERMAL_STATUS 0x714 -#define ASIC_MAX_TEMP(x) ((x) << 0) -#define ASIC_MAX_TEMP_MASK 0x000001ff -#define ASIC_MAX_TEMP_SHIFT 0 -#define CTF_TEMP(x) ((x) << 9) -#define CTF_TEMP_MASK 0x0003fe00 -#define CTF_TEMP_SHIFT 9 +#define HAINAN_GB_ADDR_CONFIG_GOLDEN 0x02010001 #define SI_MAX_SH_GPRS 256 #define SI_MAX_TEMP_GPRS 16 @@ -56,9 +48,279 @@ #define SI_MAX_TCC 16 #define SI_MAX_TCC_MASK 0xFFFF +/* SMC IND accessor regs */ +#define SMC_IND_INDEX_0 0x200 +#define SMC_IND_DATA_0 0x204 + +#define SMC_IND_ACCESS_CNTL 0x228 +# define AUTO_INCREMENT_IND_0 (1 << 0) +#define SMC_MESSAGE_0 0x22c +#define SMC_RESP_0 0x230 + +/* CG IND registers are accessed via SMC indirect space + SMC_CG_IND_START */ +#define SMC_CG_IND_START 0xc0030000 +#define SMC_CG_IND_END 0xc0040000 + +#define CG_CGTT_LOCAL_0 0x400 +#define CG_CGTT_LOCAL_1 0x401 + +/* SMC IND registers */ +#define SMC_SYSCON_RESET_CNTL 0x80000000 +# define RST_REG (1 << 0) +#define SMC_SYSCON_CLOCK_CNTL_0 0x80000004 +# define CK_DISABLE (1 << 0) +# define CKEN (1 << 24) + #define VGA_HDP_CONTROL 0x328 #define VGA_MEMORY_DISABLE (1 << 4) +#define DCCG_DISP_SLOW_SELECT_REG 0x4fc +#define DCCG_DISP1_SLOW_SELECT(x) ((x) << 0) +#define DCCG_DISP1_SLOW_SELECT_MASK (7 << 0) +#define DCCG_DISP1_SLOW_SELECT_SHIFT 0 +#define DCCG_DISP2_SLOW_SELECT(x) ((x) << 4) +#define DCCG_DISP2_SLOW_SELECT_MASK (7 << 4) +#define DCCG_DISP2_SLOW_SELECT_SHIFT 4 + +#define CG_SPLL_FUNC_CNTL 0x600 +#define SPLL_RESET (1 << 0) +#define SPLL_SLEEP (1 << 1) +#define SPLL_BYPASS_EN (1 << 3) +#define SPLL_REF_DIV(x) ((x) << 4) +#define SPLL_REF_DIV_MASK (0x3f << 4) +#define SPLL_PDIV_A(x) ((x) << 20) +#define SPLL_PDIV_A_MASK (0x7f << 20) +#define SPLL_PDIV_A_SHIFT 20 +#define CG_SPLL_FUNC_CNTL_2 0x604 +#define SCLK_MUX_SEL(x) ((x) << 0) +#define SCLK_MUX_SEL_MASK (0x1ff << 0) +#define SPLL_CTLREQ_CHG (1 << 23) +#define SCLK_MUX_UPDATE (1 << 26) +#define CG_SPLL_FUNC_CNTL_3 0x608 +#define SPLL_FB_DIV(x) ((x) << 0) +#define SPLL_FB_DIV_MASK (0x3ffffff << 0) +#define SPLL_FB_DIV_SHIFT 0 +#define SPLL_DITHEN (1 << 28) +#define CG_SPLL_FUNC_CNTL_4 0x60c + +#define SPLL_STATUS 0x614 +#define SPLL_CHG_STATUS (1 << 1) +#define SPLL_CNTL_MODE 0x618 +#define SPLL_SW_DIR_CONTROL (1 << 0) +# define SPLL_REFCLK_SEL(x) ((x) << 26) +# define SPLL_REFCLK_SEL_MASK (3 << 26) + +#define CG_SPLL_SPREAD_SPECTRUM 0x620 +#define SSEN (1 << 0) +#define CLK_S(x) ((x) << 4) +#define CLK_S_MASK (0xfff << 4) +#define CLK_S_SHIFT 4 +#define CG_SPLL_SPREAD_SPECTRUM_2 0x624 +#define CLK_V(x) ((x) << 0) +#define CLK_V_MASK (0x3ffffff << 0) +#define CLK_V_SHIFT 0 + +#define CG_SPLL_AUTOSCALE_CNTL 0x62c +# define AUTOSCALE_ON_SS_CLEAR (1 << 9) + +/* discrete uvd clocks */ +#define CG_UPLL_FUNC_CNTL 0x634 +# define UPLL_RESET_MASK 0x00000001 +# define UPLL_SLEEP_MASK 0x00000002 +# define UPLL_BYPASS_EN_MASK 0x00000004 +# define UPLL_CTLREQ_MASK 0x00000008 +# define UPLL_VCO_MODE_MASK 0x00000600 +# define UPLL_REF_DIV_MASK 0x003F0000 +# define UPLL_CTLACK_MASK 0x40000000 +# define UPLL_CTLACK2_MASK 0x80000000 +#define CG_UPLL_FUNC_CNTL_2 0x638 +# define UPLL_PDIV_A(x) ((x) << 0) +# define UPLL_PDIV_A_MASK 0x0000007F +# define UPLL_PDIV_B(x) ((x) << 8) +# define UPLL_PDIV_B_MASK 0x00007F00 +# define VCLK_SRC_SEL(x) ((x) << 20) +# define VCLK_SRC_SEL_MASK 0x01F00000 +# define DCLK_SRC_SEL(x) ((x) << 25) +# define DCLK_SRC_SEL_MASK 0x3E000000 +#define CG_UPLL_FUNC_CNTL_3 0x63C +# define UPLL_FB_DIV(x) ((x) << 0) +# define UPLL_FB_DIV_MASK 0x01FFFFFF +#define CG_UPLL_FUNC_CNTL_4 0x644 +# define UPLL_SPARE_ISPARE9 0x00020000 +#define CG_UPLL_FUNC_CNTL_5 0x648 +# define RESET_ANTI_MUX_MASK 0x00000200 +#define CG_UPLL_SPREAD_SPECTRUM 0x650 +# define SSEN_MASK 0x00000001 + +#define MPLL_BYPASSCLK_SEL 0x65c +# define MPLL_CLKOUT_SEL(x) ((x) << 8) +# define MPLL_CLKOUT_SEL_MASK 0xFF00 + +#define CG_CLKPIN_CNTL 0x660 +# define XTALIN_DIVIDE (1 << 1) +# define BCLK_AS_XCLK (1 << 2) +#define CG_CLKPIN_CNTL_2 0x664 +# define FORCE_BIF_REFCLK_EN (1 << 3) +# define MUX_TCLK_TO_XCLK (1 << 8) + +#define THM_CLK_CNTL 0x66c +# define CMON_CLK_SEL(x) ((x) << 0) +# define CMON_CLK_SEL_MASK 0xFF +# define TMON_CLK_SEL(x) ((x) << 8) +# define TMON_CLK_SEL_MASK 0xFF00 +#define MISC_CLK_CNTL 0x670 +# define DEEP_SLEEP_CLK_SEL(x) ((x) << 0) +# define DEEP_SLEEP_CLK_SEL_MASK 0xFF +# define ZCLK_SEL(x) ((x) << 8) +# define ZCLK_SEL_MASK 0xFF00 + +#define CG_THERMAL_CTRL 0x700 +#define DPM_EVENT_SRC(x) ((x) << 0) +#define DPM_EVENT_SRC_MASK (7 << 0) +#define DIG_THERM_DPM(x) ((x) << 14) +#define DIG_THERM_DPM_MASK 0x003FC000 +#define DIG_THERM_DPM_SHIFT 14 +#define CG_THERMAL_STATUS 0x704 +#define FDO_PWM_DUTY(x) ((x) << 9) +#define FDO_PWM_DUTY_MASK (0xff << 9) +#define FDO_PWM_DUTY_SHIFT 9 +#define CG_THERMAL_INT 0x708 +#define DIG_THERM_INTH(x) ((x) << 8) +#define DIG_THERM_INTH_MASK 0x0000FF00 +#define DIG_THERM_INTH_SHIFT 8 +#define DIG_THERM_INTL(x) ((x) << 16) +#define DIG_THERM_INTL_MASK 0x00FF0000 +#define DIG_THERM_INTL_SHIFT 16 +#define THERM_INT_MASK_HIGH (1 << 24) +#define THERM_INT_MASK_LOW (1 << 25) + +#define CG_MULT_THERMAL_CTRL 0x710 +#define TEMP_SEL(x) ((x) << 20) +#define TEMP_SEL_MASK (0xff << 20) +#define TEMP_SEL_SHIFT 20 +#define CG_MULT_THERMAL_STATUS 0x714 +#define ASIC_MAX_TEMP(x) ((x) << 0) +#define ASIC_MAX_TEMP_MASK 0x000001ff +#define ASIC_MAX_TEMP_SHIFT 0 +#define CTF_TEMP(x) ((x) << 9) +#define CTF_TEMP_MASK 0x0003fe00 +#define CTF_TEMP_SHIFT 9 + +#define CG_FDO_CTRL0 0x754 +#define FDO_STATIC_DUTY(x) ((x) << 0) +#define FDO_STATIC_DUTY_MASK 0x000000FF +#define FDO_STATIC_DUTY_SHIFT 0 +#define CG_FDO_CTRL1 0x758 +#define FMAX_DUTY100(x) ((x) << 0) +#define FMAX_DUTY100_MASK 0x000000FF +#define FMAX_DUTY100_SHIFT 0 +#define CG_FDO_CTRL2 0x75C +#define TMIN(x) ((x) << 0) +#define TMIN_MASK 0x000000FF +#define TMIN_SHIFT 0 +#define FDO_PWM_MODE(x) ((x) << 11) +#define FDO_PWM_MODE_MASK (7 << 11) +#define FDO_PWM_MODE_SHIFT 11 +#define TACH_PWM_RESP_RATE(x) ((x) << 25) +#define TACH_PWM_RESP_RATE_MASK (0x7f << 25) +#define TACH_PWM_RESP_RATE_SHIFT 25 + +#define CG_TACH_CTRL 0x770 +# define EDGE_PER_REV(x) ((x) << 0) +# define EDGE_PER_REV_MASK (0x7 << 0) +# define EDGE_PER_REV_SHIFT 0 +# define TARGET_PERIOD(x) ((x) << 3) +# define TARGET_PERIOD_MASK 0xfffffff8 +# define TARGET_PERIOD_SHIFT 3 +#define CG_TACH_STATUS 0x774 +# define TACH_PERIOD(x) ((x) << 0) +# define TACH_PERIOD_MASK 0xffffffff +# define TACH_PERIOD_SHIFT 0 + +#define GENERAL_PWRMGT 0x780 +# define GLOBAL_PWRMGT_EN (1 << 0) +# define STATIC_PM_EN (1 << 1) +# define THERMAL_PROTECTION_DIS (1 << 2) +# define THERMAL_PROTECTION_TYPE (1 << 3) +# define SW_SMIO_INDEX(x) ((x) << 6) +# define SW_SMIO_INDEX_MASK (1 << 6) +# define SW_SMIO_INDEX_SHIFT 6 +# define VOLT_PWRMGT_EN (1 << 10) +# define DYN_SPREAD_SPECTRUM_EN (1 << 23) +#define CG_TPC 0x784 +#define SCLK_PWRMGT_CNTL 0x788 +# define SCLK_PWRMGT_OFF (1 << 0) +# define SCLK_LOW_D1 (1 << 1) +# define FIR_RESET (1 << 4) +# define FIR_FORCE_TREND_SEL (1 << 5) +# define FIR_TREND_MODE (1 << 6) +# define DYN_GFX_CLK_OFF_EN (1 << 7) +# define GFX_CLK_FORCE_ON (1 << 8) +# define GFX_CLK_REQUEST_OFF (1 << 9) +# define GFX_CLK_FORCE_OFF (1 << 10) +# define GFX_CLK_OFF_ACPI_D1 (1 << 11) +# define GFX_CLK_OFF_ACPI_D2 (1 << 12) +# define GFX_CLK_OFF_ACPI_D3 (1 << 13) +# define DYN_LIGHT_SLEEP_EN (1 << 14) + +#define TARGET_AND_CURRENT_PROFILE_INDEX 0x798 +# define CURRENT_STATE_INDEX_MASK (0xf << 4) +# define CURRENT_STATE_INDEX_SHIFT 4 + +#define CG_FTV 0x7bc + +#define CG_FFCT_0 0x7c0 +# define UTC_0(x) ((x) << 0) +# define UTC_0_MASK (0x3ff << 0) +# define DTC_0(x) ((x) << 10) +# define DTC_0_MASK (0x3ff << 10) + +#define CG_BSP 0x7fc +# define BSP(x) ((x) << 0) +# define BSP_MASK (0xffff << 0) +# define BSU(x) ((x) << 16) +# define BSU_MASK (0xf << 16) +#define CG_AT 0x800 +# define CG_R(x) ((x) << 0) +# define CG_R_MASK (0xffff << 0) +# define CG_L(x) ((x) << 16) +# define CG_L_MASK (0xffff << 16) + +#define CG_GIT 0x804 +# define CG_GICST(x) ((x) << 0) +# define CG_GICST_MASK (0xffff << 0) +# define CG_GIPOT(x) ((x) << 16) +# define CG_GIPOT_MASK (0xffff << 16) + +#define CG_SSP 0x80c +# define SST(x) ((x) << 0) +# define SST_MASK (0xffff << 0) +# define SSTU(x) ((x) << 16) +# define SSTU_MASK (0xf << 16) + +#define CG_DISPLAY_GAP_CNTL 0x828 +# define DISP1_GAP(x) ((x) << 0) +# define DISP1_GAP_MASK (3 << 0) +# define DISP2_GAP(x) ((x) << 2) +# define DISP2_GAP_MASK (3 << 2) +# define VBI_TIMER_COUNT(x) ((x) << 4) +# define VBI_TIMER_COUNT_MASK (0x3fff << 4) +# define VBI_TIMER_UNIT(x) ((x) << 20) +# define VBI_TIMER_UNIT_MASK (7 << 20) +# define DISP1_GAP_MCHG(x) ((x) << 24) +# define DISP1_GAP_MCHG_MASK (3 << 24) +# define DISP2_GAP_MCHG(x) ((x) << 26) +# define DISP2_GAP_MCHG_MASK (3 << 26) + +#define CG_ULV_CONTROL 0x878 +#define CG_ULV_PARAMETER 0x87c + +#define SMC_SCRATCH0 0x884 + +#define CG_CAC_CTRL 0x8b8 +# define CAC_WINDOW(x) ((x) << 0) +# define CAC_WINDOW_MASK 0x00ffffff + #define DMIF_ADDR_CONFIG 0xBD4 #define DMIF_ADDR_CALC 0xC00 @@ -68,6 +330,14 @@ # define DMIF_BUFFERS_ALLOCATED_COMPLETED (1 << 4) #define SRBM_STATUS 0xE50 +#define GRBM_RQ_PENDING (1 << 5) +#define VMC_BUSY (1 << 8) +#define MCB_BUSY (1 << 9) +#define MCB_NON_DISPLAY_BUSY (1 << 10) +#define MCC_BUSY (1 << 11) +#define MCD_BUSY (1 << 12) +#define SEM_BUSY (1 << 14) +#define IH_BUSY (1 << 17) #define SRBM_SOFT_RESET 0x0E60 #define SOFT_RESET_BIF (1 << 1) @@ -88,6 +358,14 @@ #define CC_SYS_RB_BACKEND_DISABLE 0xe80 #define GC_USER_SYS_RB_BACKEND_DISABLE 0xe84 +#define SRBM_READ_ERROR 0xE98 +#define SRBM_INT_CNTL 0xEA0 +#define SRBM_INT_ACK 0xEA8 + +#define SRBM_STATUS2 0x0EC4 +#define DMA_BUSY (1 << 5) +#define DMA1_BUSY (1 << 6) + #define VM_L2_CNTL 0x1400 #define ENABLE_L2_CACHE (1 << 0) #define ENABLE_L2_FRAGMENT_PROCESSING (1 << 1) @@ -126,6 +404,7 @@ #define READ_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 16) #define WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT (1 << 18) #define WRITE_PROTECTION_FAULT_ENABLE_DEFAULT (1 << 19) +#define PAGE_TABLE_BLOCK_SIZE(x) (((x) & 0xF) << 24) #define VM_CONTEXT1_CNTL 0x1414 #define VM_CONTEXT0_CNTL2 0x1430 #define VM_CONTEXT1_CNTL2 0x1434 @@ -140,6 +419,20 @@ #define VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x14FC #define VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x14DC +#define PROTECTIONS_MASK (0xf << 0) +#define PROTECTIONS_SHIFT 0 + /* bit 0: range + * bit 1: pde0 + * bit 2: valid + * bit 3: read + * bit 4: write + */ +#define MEMORY_CLIENT_ID_MASK (0xff << 12) +#define MEMORY_CLIENT_ID_SHIFT 12 +#define MEMORY_CLIENT_RW_MASK (1 << 24) +#define MEMORY_CLIENT_RW_SHIFT 24 +#define FAULT_VMID_MASK (0xf << 25) +#define FAULT_VMID_SHIFT 25 #define VM_INVALIDATE_REQUEST 0x1478 #define VM_INVALIDATE_RESPONSE 0x147c @@ -161,6 +454,10 @@ #define VM_CONTEXT0_PAGE_TABLE_END_ADDR 0x157C #define VM_CONTEXT1_PAGE_TABLE_END_ADDR 0x1580 +#define VM_L2_CG 0x15c0 +#define MC_CG_ENABLE (1 << 18) +#define MC_LS_ENABLE (1 << 19) + #define MC_SHARED_CHMAP 0x2004 #define NOOFCHAN_SHIFT 12 #define NOOFCHAN_MASK 0x0000f000 @@ -186,6 +483,17 @@ #define MC_SHARED_BLACKOUT_CNTL 0x20ac +#define MC_HUB_MISC_HUB_CG 0x20b8 +#define MC_HUB_MISC_VM_CG 0x20bc + +#define MC_HUB_MISC_SIP_CG 0x20c0 + +#define MC_XPB_CLK_GAT 0x2478 + +#define MC_CITF_MISC_RD_CG 0x2648 +#define MC_CITF_MISC_WR_CG 0x264c +#define MC_CITF_MISC_VM_CG 0x2650 + #define MC_ARB_RAMCFG 0x2760 #define NOOFBANK_SHIFT 0 #define NOOFBANK_MASK 0x00000003 @@ -201,6 +509,23 @@ #define NOOFGROUPS_SHIFT 12 #define NOOFGROUPS_MASK 0x00001000 +#define MC_ARB_DRAM_TIMING 0x2774 +#define MC_ARB_DRAM_TIMING2 0x2778 + +#define MC_ARB_BURST_TIME 0x2808 +#define STATE0(x) ((x) << 0) +#define STATE0_MASK (0x1f << 0) +#define STATE0_SHIFT 0 +#define STATE1(x) ((x) << 5) +#define STATE1_MASK (0x1f << 5) +#define STATE1_SHIFT 5 +#define STATE2(x) ((x) << 10) +#define STATE2_MASK (0x1f << 10) +#define STATE2_SHIFT 10 +#define STATE3(x) ((x) << 15) +#define STATE3_MASK (0x1f << 15) +#define STATE3_SHIFT 15 + #define MC_SEQ_TRAIN_WAKEUP_CNTL 0x28e8 #define TRAIN_DONE_D0 (1 << 30) #define TRAIN_DONE_D1 (1 << 31) @@ -208,14 +533,109 @@ #define MC_SEQ_SUP_CNTL 0x28c8 #define RUN_MASK (1 << 0) #define MC_SEQ_SUP_PGM 0x28cc +#define MC_PMG_AUTO_CMD 0x28d0 #define MC_IO_PAD_CNTL_D0 0x29d0 #define MEM_FALL_OUT_CMD (1 << 8) +#define MC_SEQ_RAS_TIMING 0x28a0 +#define MC_SEQ_CAS_TIMING 0x28a4 +#define MC_SEQ_MISC_TIMING 0x28a8 +#define MC_SEQ_MISC_TIMING2 0x28ac +#define MC_SEQ_PMG_TIMING 0x28b0 +#define MC_SEQ_RD_CTL_D0 0x28b4 +#define MC_SEQ_RD_CTL_D1 0x28b8 +#define MC_SEQ_WR_CTL_D0 0x28bc +#define MC_SEQ_WR_CTL_D1 0x28c0 + +#define MC_SEQ_MISC0 0x2a00 +#define MC_SEQ_MISC0_VEN_ID_SHIFT 8 +#define MC_SEQ_MISC0_VEN_ID_MASK 0x00000f00 +#define MC_SEQ_MISC0_VEN_ID_VALUE 3 +#define MC_SEQ_MISC0_REV_ID_SHIFT 12 +#define MC_SEQ_MISC0_REV_ID_MASK 0x0000f000 +#define MC_SEQ_MISC0_REV_ID_VALUE 1 +#define MC_SEQ_MISC0_GDDR5_SHIFT 28 +#define MC_SEQ_MISC0_GDDR5_MASK 0xf0000000 +#define MC_SEQ_MISC0_GDDR5_VALUE 5 +#define MC_SEQ_MISC1 0x2a04 +#define MC_SEQ_RESERVE_M 0x2a08 +#define MC_PMG_CMD_EMRS 0x2a0c + #define MC_SEQ_IO_DEBUG_INDEX 0x2a44 #define MC_SEQ_IO_DEBUG_DATA 0x2a48 +#define MC_SEQ_MISC5 0x2a54 +#define MC_SEQ_MISC6 0x2a58 + +#define MC_SEQ_MISC7 0x2a64 + +#define MC_SEQ_RAS_TIMING_LP 0x2a6c +#define MC_SEQ_CAS_TIMING_LP 0x2a70 +#define MC_SEQ_MISC_TIMING_LP 0x2a74 +#define MC_SEQ_MISC_TIMING2_LP 0x2a78 +#define MC_SEQ_WR_CTL_D0_LP 0x2a7c +#define MC_SEQ_WR_CTL_D1_LP 0x2a80 +#define MC_SEQ_PMG_CMD_EMRS_LP 0x2a84 +#define MC_SEQ_PMG_CMD_MRS_LP 0x2a88 + +#define MC_PMG_CMD_MRS 0x2aac + +#define MC_SEQ_RD_CTL_D0_LP 0x2b1c +#define MC_SEQ_RD_CTL_D1_LP 0x2b20 + +#define MC_PMG_CMD_MRS1 0x2b44 +#define MC_SEQ_PMG_CMD_MRS1_LP 0x2b48 +#define MC_SEQ_PMG_TIMING_LP 0x2b4c + +#define MC_SEQ_WR_CTL_2 0x2b54 +#define MC_SEQ_WR_CTL_2_LP 0x2b58 +#define MC_PMG_CMD_MRS2 0x2b5c +#define MC_SEQ_PMG_CMD_MRS2_LP 0x2b60 + +#define MCLK_PWRMGT_CNTL 0x2ba0 +# define DLL_SPEED(x) ((x) << 0) +# define DLL_SPEED_MASK (0x1f << 0) +# define DLL_READY (1 << 6) +# define MC_INT_CNTL (1 << 7) +# define MRDCK0_PDNB (1 << 8) +# define MRDCK1_PDNB (1 << 9) +# define MRDCK0_RESET (1 << 16) +# define MRDCK1_RESET (1 << 17) +# define DLL_READY_READ (1 << 24) +#define DLL_CNTL 0x2ba4 +# define MRDCK0_BYPASS (1 << 24) +# define MRDCK1_BYPASS (1 << 25) + +#define MPLL_CNTL_MODE 0x2bb0 +# define MPLL_MCLK_SEL (1 << 11) +#define MPLL_FUNC_CNTL 0x2bb4 +#define BWCTRL(x) ((x) << 20) +#define BWCTRL_MASK (0xff << 20) +#define MPLL_FUNC_CNTL_1 0x2bb8 +#define VCO_MODE(x) ((x) << 0) +#define VCO_MODE_MASK (3 << 0) +#define CLKFRAC(x) ((x) << 4) +#define CLKFRAC_MASK (0xfff << 4) +#define CLKF(x) ((x) << 16) +#define CLKF_MASK (0xfff << 16) +#define MPLL_FUNC_CNTL_2 0x2bbc +#define MPLL_AD_FUNC_CNTL 0x2bc0 +#define YCLK_POST_DIV(x) ((x) << 0) +#define YCLK_POST_DIV_MASK (7 << 0) +#define MPLL_DQ_FUNC_CNTL 0x2bc4 +#define YCLK_SEL(x) ((x) << 4) +#define YCLK_SEL_MASK (1 << 4) + +#define MPLL_SS1 0x2bcc +#define CLKV(x) ((x) << 0) +#define CLKV_MASK (0x3ffffff << 0) +#define MPLL_SS2 0x2bd0 +#define CLKS(x) ((x) << 0) +#define CLKS_MASK (0xfff << 0) + #define HDP_HOST_PATH_CNTL 0x2C00 +#define CLOCK_GATING_DIS (1 << 23) #define HDP_NONSURFACE_BASE 0x2C04 #define HDP_NONSURFACE_INFO 0x2C08 #define HDP_NONSURFACE_SIZE 0x2C0C @@ -223,6 +643,10 @@ #define HDP_ADDR_CONFIG 0x2F48 #define HDP_MISC_CNTL 0x2F4C #define HDP_FLUSH_INVALIDATE_CACHE (1 << 0) +#define HDP_MEM_POWER_LS 0x2F50 +#define HDP_LS_ENABLE (1 << 0) + +#define ATC_MISC_CG 0x3350 #define IH_RB_CNTL 0x3e00 # define IH_RB_ENABLE (1 << 0) @@ -268,6 +692,99 @@ #define HDP_REG_COHERENCY_FLUSH_CNTL 0x54A0 +/* DCE6 ELD audio interface */ +#define AZ_F0_CODEC_ENDPOINT_INDEX 0x5E00 +# define AZ_ENDPOINT_REG_INDEX(x) (((x) & 0xff) << 0) +# define AZ_ENDPOINT_REG_WRITE_EN (1 << 8) +#define AZ_F0_CODEC_ENDPOINT_DATA 0x5E04 + +#define AZ_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x25 +#define SPEAKER_ALLOCATION(x) (((x) & 0x7f) << 0) +#define SPEAKER_ALLOCATION_MASK (0x7f << 0) +#define SPEAKER_ALLOCATION_SHIFT 0 +#define HDMI_CONNECTION (1 << 16) +#define DP_CONNECTION (1 << 17) + +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x28 /* LPCM */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x29 /* AC3 */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x2A /* MPEG1 */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x2B /* MP3 */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x2C /* MPEG2 */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x2D /* AAC */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x2E /* DTS */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x2F /* ATRAC */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x30 /* one bit audio - leave at 0 (default) */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x31 /* Dolby Digital */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x32 /* DTS-HD */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x33 /* MAT-MLP */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x34 /* DTS */ +#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x35 /* WMA Pro */ +# define MAX_CHANNELS(x) (((x) & 0x7) << 0) +/* max channels minus one. 7 = 8 channels */ +# define SUPPORTED_FREQUENCIES(x) (((x) & 0xff) << 8) +# define DESCRIPTOR_BYTE_2(x) (((x) & 0xff) << 16) +# define SUPPORTED_FREQUENCIES_STEREO(x) (((x) & 0xff) << 24) /* LPCM only */ +/* SUPPORTED_FREQUENCIES, SUPPORTED_FREQUENCIES_STEREO + * bit0 = 32 kHz + * bit1 = 44.1 kHz + * bit2 = 48 kHz + * bit3 = 88.2 kHz + * bit4 = 96 kHz + * bit5 = 176.4 kHz + * bit6 = 192 kHz + */ + +#define AZ_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x37 +# define VIDEO_LIPSYNC(x) (((x) & 0xff) << 0) +# define AUDIO_LIPSYNC(x) (((x) & 0xff) << 8) +/* VIDEO_LIPSYNC, AUDIO_LIPSYNC + * 0 = invalid + * x = legal delay value + * 255 = sync not supported + */ +#define AZ_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x38 +# define HBR_CAPABLE (1 << 0) /* enabled by default */ + +#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x3a +# define MANUFACTURER_ID(x) (((x) & 0xffff) << 0) +# define PRODUCT_ID(x) (((x) & 0xffff) << 16) +#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x3b +# define SINK_DESCRIPTION_LEN(x) (((x) & 0xff) << 0) +#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x3c +# define PORT_ID0(x) (((x) & 0xffffffff) << 0) +#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x3d +# define PORT_ID1(x) (((x) & 0xffffffff) << 0) +#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x3e +# define DESCRIPTION0(x) (((x) & 0xff) << 0) +# define DESCRIPTION1(x) (((x) & 0xff) << 8) +# define DESCRIPTION2(x) (((x) & 0xff) << 16) +# define DESCRIPTION3(x) (((x) & 0xff) << 24) +#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x3f +# define DESCRIPTION4(x) (((x) & 0xff) << 0) +# define DESCRIPTION5(x) (((x) & 0xff) << 8) +# define DESCRIPTION6(x) (((x) & 0xff) << 16) +# define DESCRIPTION7(x) (((x) & 0xff) << 24) +#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x40 +# define DESCRIPTION8(x) (((x) & 0xff) << 0) +# define DESCRIPTION9(x) (((x) & 0xff) << 8) +# define DESCRIPTION10(x) (((x) & 0xff) << 16) +# define DESCRIPTION11(x) (((x) & 0xff) << 24) +#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x41 +# define DESCRIPTION12(x) (((x) & 0xff) << 0) +# define DESCRIPTION13(x) (((x) & 0xff) << 8) +# define DESCRIPTION14(x) (((x) & 0xff) << 16) +# define DESCRIPTION15(x) (((x) & 0xff) << 24) +#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x42 +# define DESCRIPTION16(x) (((x) & 0xff) << 0) +# define DESCRIPTION17(x) (((x) & 0xff) << 8) + +#define AZ_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x54 +# define AUDIO_ENABLED (1 << 31) + +#define AZ_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x56 +#define PORT_CONNECTIVITY_MASK (3 << 30) +#define PORT_CONNECTIVITY_SHIFT 30 + #define DC_LB_MEMORY_SPLIT 0x6b0c #define DC_LB_MEMORY_CONFIG(x) ((x) << 20) @@ -348,7 +865,7 @@ # define GRPH_PFLIP_INT_MASK (1 << 0) # define GRPH_PFLIP_INT_TYPE (1 << 8) -#define DACA_AUTODETECT_INT_CONTROL 0x66c8 +#define DAC_AUTODETECT_INT_CONTROL 0x67c8 #define DC_HPD1_INT_STATUS 0x601c #define DC_HPD2_INT_STATUS 0x6028 @@ -382,9 +899,38 @@ # define DC_HPDx_RX_INT_TIMER(x) ((x) << 16) # define DC_HPDx_EN (1 << 28) +#define DPG_PIPE_STUTTER_CONTROL 0x6cd4 +# define STUTTER_ENABLE (1 << 0) + /* 0x6e98, 0x7a98, 0x10698, 0x11298, 0x11e98, 0x12a98 */ #define CRTC_STATUS_FRAME_COUNT 0x6e98 +/* Audio clocks */ +#define DCCG_AUDIO_DTO_SOURCE 0x05ac +# define DCCG_AUDIO_DTO0_SOURCE_SEL(x) ((x) << 0) /* crtc0 - crtc5 */ +# define DCCG_AUDIO_DTO_SEL (1 << 4) /* 0=dto0 1=dto1 */ + +#define DCCG_AUDIO_DTO0_PHASE 0x05b0 +#define DCCG_AUDIO_DTO0_MODULE 0x05b4 +#define DCCG_AUDIO_DTO1_PHASE 0x05c0 +#define DCCG_AUDIO_DTO1_MODULE 0x05c4 + +#define DENTIST_DISPCLK_CNTL 0x0490 +# define DENTIST_DPREFCLK_WDIVIDER(x) (((x) & 0x7f) << 24) +# define DENTIST_DPREFCLK_WDIVIDER_MASK (0x7f << 24) +# define DENTIST_DPREFCLK_WDIVIDER_SHIFT 24 + +#define AFMT_AUDIO_SRC_CONTROL 0x713c +#define AFMT_AUDIO_SRC_SELECT(x) (((x) & 7) << 0) +/* AFMT_AUDIO_SRC_SELECT + * 0 = stream0 + * 1 = stream1 + * 2 = stream2 + * 3 = stream3 + * 4 = stream4 + * 5 = stream5 + */ + #define GRBM_CNTL 0x8000 #define GRBM_READ_TIMEOUT(x) ((x) << 0) @@ -557,6 +1103,24 @@ #define SQC_CACHES 0x8C08 +#define SQ_POWER_THROTTLE 0x8e58 +#define MIN_POWER(x) ((x) << 0) +#define MIN_POWER_MASK (0x3fff << 0) +#define MIN_POWER_SHIFT 0 +#define MAX_POWER(x) ((x) << 16) +#define MAX_POWER_MASK (0x3fff << 16) +#define MAX_POWER_SHIFT 0 +#define SQ_POWER_THROTTLE2 0x8e5c +#define MAX_POWER_DELTA(x) ((x) << 0) +#define MAX_POWER_DELTA_MASK (0x3fff << 0) +#define MAX_POWER_DELTA_SHIFT 0 +#define STI_SIZE(x) ((x) << 16) +#define STI_SIZE_MASK (0x3ff << 16) +#define STI_SIZE_SHIFT 16 +#define LTI_RATIO(x) ((x) << 27) +#define LTI_RATIO_MASK (0xf << 27) +#define LTI_RATIO_SHIFT 27 + #define SX_DEBUG_1 0x9060 #define SPI_STATIC_THREAD_MGMT_1 0x90E0 @@ -574,6 +1138,11 @@ #define CGTS_USER_TCC_DISABLE 0x914C #define TCC_DISABLE_MASK 0xFFFF0000 #define TCC_DISABLE_SHIFT 16 +#define CGTS_SM_CTRL_REG 0x9150 +#define OVERRIDE (1 << 21) +#define LS_OVERRIDE (1 << 22) + +#define SPI_LB_CU_MASK 0x9354 #define TA_CNTL_AUX 0x9508 @@ -663,6 +1232,8 @@ #define CB_PERFCOUNTER3_SELECT0 0x9a38 #define CB_PERFCOUNTER3_SELECT1 0x9a3c +#define CB_CGTT_SCLK_CTRL 0x9a60 + #define GC_USER_RB_BACKEND_DISABLE 0x9B7C #define BACKEND_DISABLE_MASK 0x00FF0000 #define BACKEND_DISABLE_SHIFT 16 @@ -720,6 +1291,9 @@ # define CP_RINGID1_INT_STAT (1 << 30) # define CP_RINGID0_INT_STAT (1 << 31) +#define CP_MEM_SLP_CNTL 0xC1E4 +# define CP_MEM_LS_EN (1 << 0) + #define CP_DEBUG 0xC1FC #define RLC_CNTL 0xC300 @@ -727,6 +1301,7 @@ #define RLC_RL_BASE 0xC304 #define RLC_RL_SIZE 0xC308 #define RLC_LB_CNTL 0xC30C +# define LOAD_BALANCE_ENABLE (1 << 0) #define RLC_SAVE_AND_RESTORE_BASE 0xC310 #define RLC_LB_CNTR_MAX 0xC314 #define RLC_LB_CNTR_INIT 0xC318 @@ -741,6 +1316,56 @@ #define RLC_CAPTURE_GPU_CLOCK_COUNT 0xC340 #define RLC_MC_CNTL 0xC344 #define RLC_UCODE_CNTL 0xC348 +#define RLC_STAT 0xC34C +# define RLC_BUSY_STATUS (1 << 0) +# define GFX_POWER_STATUS (1 << 1) +# define GFX_CLOCK_STATUS (1 << 2) +# define GFX_LS_STATUS (1 << 3) + +#define RLC_PG_CNTL 0xC35C +# define GFX_PG_ENABLE (1 << 0) +# define GFX_PG_SRC (1 << 1) + +#define RLC_CGTT_MGCG_OVERRIDE 0xC400 +#define RLC_CGCG_CGLS_CTRL 0xC404 +# define CGCG_EN (1 << 0) +# define CGLS_EN (1 << 1) + +#define RLC_TTOP_D 0xC414 +# define RLC_PUD(x) ((x) << 0) +# define RLC_PUD_MASK (0xff << 0) +# define RLC_PDD(x) ((x) << 8) +# define RLC_PDD_MASK (0xff << 8) +# define RLC_TTPD(x) ((x) << 16) +# define RLC_TTPD_MASK (0xff << 16) +# define RLC_MSD(x) ((x) << 24) +# define RLC_MSD_MASK (0xff << 24) + +#define RLC_LB_INIT_CU_MASK 0xC41C + +#define RLC_PG_AO_CU_MASK 0xC42C +#define RLC_MAX_PG_CU 0xC430 +# define MAX_PU_CU(x) ((x) << 0) +# define MAX_PU_CU_MASK (0xff << 0) +#define RLC_AUTO_PG_CTRL 0xC434 +# define AUTO_PG_EN (1 << 0) +# define GRBM_REG_SGIT(x) ((x) << 3) +# define GRBM_REG_SGIT_MASK (0xffff << 3) +# define PG_AFTER_GRBM_REG_ST(x) ((x) << 19) +# define PG_AFTER_GRBM_REG_ST_MASK (0x1fff << 19) + +#define RLC_SERDES_WR_MASTER_MASK_0 0xC454 +#define RLC_SERDES_WR_MASTER_MASK_1 0xC458 +#define RLC_SERDES_WR_CTRL 0xC45C + +#define RLC_SERDES_MASTER_BUSY_0 0xC464 +#define RLC_SERDES_MASTER_BUSY_1 0xC468 + +#define RLC_GCPM_GENERAL_3 0xC478 + +#define DB_RENDER_CONTROL 0x28000 + +#define DB_DEPTH_INFO 0x2803c #define PA_SC_RASTER_CONFIG 0x28350 # define RASTER_CONFIG_RB_MAP_0 0 @@ -787,19 +1412,176 @@ # define THREAD_TRACE_FLUSH (54 << 0) # define THREAD_TRACE_FINISH (55 << 0) +/* PIF PHY0 registers idx/data 0x8/0xc */ +#define PB0_PIF_CNTL 0x10 +# define LS2_EXIT_TIME(x) ((x) << 17) +# define LS2_EXIT_TIME_MASK (0x7 << 17) +# define LS2_EXIT_TIME_SHIFT 17 +#define PB0_PIF_PAIRING 0x11 +# define MULTI_PIF (1 << 25) +#define PB0_PIF_PWRDOWN_0 0x12 +# define PLL_POWER_STATE_IN_TXS2_0(x) ((x) << 7) +# define PLL_POWER_STATE_IN_TXS2_0_MASK (0x7 << 7) +# define PLL_POWER_STATE_IN_TXS2_0_SHIFT 7 +# define PLL_POWER_STATE_IN_OFF_0(x) ((x) << 10) +# define PLL_POWER_STATE_IN_OFF_0_MASK (0x7 << 10) +# define PLL_POWER_STATE_IN_OFF_0_SHIFT 10 +# define PLL_RAMP_UP_TIME_0(x) ((x) << 24) +# define PLL_RAMP_UP_TIME_0_MASK (0x7 << 24) +# define PLL_RAMP_UP_TIME_0_SHIFT 24 +#define PB0_PIF_PWRDOWN_1 0x13 +# define PLL_POWER_STATE_IN_TXS2_1(x) ((x) << 7) +# define PLL_POWER_STATE_IN_TXS2_1_MASK (0x7 << 7) +# define PLL_POWER_STATE_IN_TXS2_1_SHIFT 7 +# define PLL_POWER_STATE_IN_OFF_1(x) ((x) << 10) +# define PLL_POWER_STATE_IN_OFF_1_MASK (0x7 << 10) +# define PLL_POWER_STATE_IN_OFF_1_SHIFT 10 +# define PLL_RAMP_UP_TIME_1(x) ((x) << 24) +# define PLL_RAMP_UP_TIME_1_MASK (0x7 << 24) +# define PLL_RAMP_UP_TIME_1_SHIFT 24 + +#define PB0_PIF_PWRDOWN_2 0x17 +# define PLL_POWER_STATE_IN_TXS2_2(x) ((x) << 7) +# define PLL_POWER_STATE_IN_TXS2_2_MASK (0x7 << 7) +# define PLL_POWER_STATE_IN_TXS2_2_SHIFT 7 +# define PLL_POWER_STATE_IN_OFF_2(x) ((x) << 10) +# define PLL_POWER_STATE_IN_OFF_2_MASK (0x7 << 10) +# define PLL_POWER_STATE_IN_OFF_2_SHIFT 10 +# define PLL_RAMP_UP_TIME_2(x) ((x) << 24) +# define PLL_RAMP_UP_TIME_2_MASK (0x7 << 24) +# define PLL_RAMP_UP_TIME_2_SHIFT 24 +#define PB0_PIF_PWRDOWN_3 0x18 +# define PLL_POWER_STATE_IN_TXS2_3(x) ((x) << 7) +# define PLL_POWER_STATE_IN_TXS2_3_MASK (0x7 << 7) +# define PLL_POWER_STATE_IN_TXS2_3_SHIFT 7 +# define PLL_POWER_STATE_IN_OFF_3(x) ((x) << 10) +# define PLL_POWER_STATE_IN_OFF_3_MASK (0x7 << 10) +# define PLL_POWER_STATE_IN_OFF_3_SHIFT 10 +# define PLL_RAMP_UP_TIME_3(x) ((x) << 24) +# define PLL_RAMP_UP_TIME_3_MASK (0x7 << 24) +# define PLL_RAMP_UP_TIME_3_SHIFT 24 +/* PIF PHY1 registers idx/data 0x10/0x14 */ +#define PB1_PIF_CNTL 0x10 +#define PB1_PIF_PAIRING 0x11 +#define PB1_PIF_PWRDOWN_0 0x12 +#define PB1_PIF_PWRDOWN_1 0x13 + +#define PB1_PIF_PWRDOWN_2 0x17 +#define PB1_PIF_PWRDOWN_3 0x18 +/* PCIE registers idx/data 0x30/0x34 */ +#define PCIE_CNTL2 0x1c /* PCIE */ +# define SLV_MEM_LS_EN (1 << 16) +# define SLV_MEM_AGGRESSIVE_LS_EN (1 << 17) +# define MST_MEM_LS_EN (1 << 18) +# define REPLAY_MEM_LS_EN (1 << 19) +#define PCIE_LC_STATUS1 0x28 /* PCIE */ +# define LC_REVERSE_RCVR (1 << 0) +# define LC_REVERSE_XMIT (1 << 1) +# define LC_OPERATING_LINK_WIDTH_MASK (0x7 << 2) +# define LC_OPERATING_LINK_WIDTH_SHIFT 2 +# define LC_DETECTED_LINK_WIDTH_MASK (0x7 << 5) +# define LC_DETECTED_LINK_WIDTH_SHIFT 5 + +#define PCIE_P_CNTL 0x40 /* PCIE */ +# define P_IGNORE_EDB_ERR (1 << 6) + +/* PCIE PORT registers idx/data 0x38/0x3c */ +#define PCIE_LC_CNTL 0xa0 +# define LC_L0S_INACTIVITY(x) ((x) << 8) +# define LC_L0S_INACTIVITY_MASK (0xf << 8) +# define LC_L0S_INACTIVITY_SHIFT 8 +# define LC_L1_INACTIVITY(x) ((x) << 12) +# define LC_L1_INACTIVITY_MASK (0xf << 12) +# define LC_L1_INACTIVITY_SHIFT 12 +# define LC_PMI_TO_L1_DIS (1 << 16) +# define LC_ASPM_TO_L1_DIS (1 << 24) +#define PCIE_LC_LINK_WIDTH_CNTL 0xa2 /* PCIE_P */ +# define LC_LINK_WIDTH_SHIFT 0 +# define LC_LINK_WIDTH_MASK 0x7 +# define LC_LINK_WIDTH_X0 0 +# define LC_LINK_WIDTH_X1 1 +# define LC_LINK_WIDTH_X2 2 +# define LC_LINK_WIDTH_X4 3 +# define LC_LINK_WIDTH_X8 4 +# define LC_LINK_WIDTH_X16 6 +# define LC_LINK_WIDTH_RD_SHIFT 4 +# define LC_LINK_WIDTH_RD_MASK 0x70 +# define LC_RECONFIG_ARC_MISSING_ESCAPE (1 << 7) +# define LC_RECONFIG_NOW (1 << 8) +# define LC_RENEGOTIATION_SUPPORT (1 << 9) +# define LC_RENEGOTIATE_EN (1 << 10) +# define LC_SHORT_RECONFIG_EN (1 << 11) +# define LC_UPCONFIGURE_SUPPORT (1 << 12) +# define LC_UPCONFIGURE_DIS (1 << 13) +# define LC_DYN_LANES_PWR_STATE(x) ((x) << 21) +# define LC_DYN_LANES_PWR_STATE_MASK (0x3 << 21) +# define LC_DYN_LANES_PWR_STATE_SHIFT 21 +#define PCIE_LC_N_FTS_CNTL 0xa3 /* PCIE_P */ +# define LC_XMIT_N_FTS(x) ((x) << 0) +# define LC_XMIT_N_FTS_MASK (0xff << 0) +# define LC_XMIT_N_FTS_SHIFT 0 +# define LC_XMIT_N_FTS_OVERRIDE_EN (1 << 8) +# define LC_N_FTS_MASK (0xff << 24) +#define PCIE_LC_SPEED_CNTL 0xa4 /* PCIE_P */ +# define LC_GEN2_EN_STRAP (1 << 0) +# define LC_GEN3_EN_STRAP (1 << 1) +# define LC_TARGET_LINK_SPEED_OVERRIDE_EN (1 << 2) +# define LC_TARGET_LINK_SPEED_OVERRIDE_MASK (0x3 << 3) +# define LC_TARGET_LINK_SPEED_OVERRIDE_SHIFT 3 +# define LC_FORCE_EN_SW_SPEED_CHANGE (1 << 5) +# define LC_FORCE_DIS_SW_SPEED_CHANGE (1 << 6) +# define LC_FORCE_EN_HW_SPEED_CHANGE (1 << 7) +# define LC_FORCE_DIS_HW_SPEED_CHANGE (1 << 8) +# define LC_INITIATE_LINK_SPEED_CHANGE (1 << 9) +# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK (0x3 << 10) +# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT 10 +# define LC_CURRENT_DATA_RATE_MASK (0x3 << 13) /* 0/1/2 = gen1/2/3 */ +# define LC_CURRENT_DATA_RATE_SHIFT 13 +# define LC_CLR_FAILED_SPD_CHANGE_CNT (1 << 16) +# define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 18) +# define LC_OTHER_SIDE_SUPPORTS_GEN2 (1 << 19) +# define LC_OTHER_SIDE_EVER_SENT_GEN3 (1 << 20) +# define LC_OTHER_SIDE_SUPPORTS_GEN3 (1 << 21) + +#define PCIE_LC_CNTL2 0xb1 +# define LC_ALLOW_PDWN_IN_L1 (1 << 17) +# define LC_ALLOW_PDWN_IN_L23 (1 << 18) + +#define PCIE_LC_CNTL3 0xb5 /* PCIE_P */ +# define LC_GO_TO_RECOVERY (1 << 30) +#define PCIE_LC_CNTL4 0xb6 /* PCIE_P */ +# define LC_REDO_EQ (1 << 5) +# define LC_SET_QUIESCE (1 << 13) + +/* + * UVD + */ +#define UVD_UDEC_ADDR_CONFIG 0xEF4C +#define UVD_UDEC_DB_ADDR_CONFIG 0xEF50 +#define UVD_UDEC_DBW_ADDR_CONFIG 0xEF54 +#define UVD_RBC_RB_RPTR 0xF690 +#define UVD_RBC_RB_WPTR 0xF694 +#define UVD_STATUS 0xf6bc + +#define UVD_CGC_CTRL 0xF4B0 +# define DCM (1 << 0) +# define CG_DT(x) ((x) << 2) +# define CG_DT_MASK (0xf << 2) +# define CLK_OD(x) ((x) << 6) +# define CLK_OD_MASK (0x1f << 6) + + /* UVD CTX indirect */ +#define UVD_CGC_MEM_CTRL 0xC0 +#define UVD_CGC_CTRL2 0xC1 +# define DYN_OR_EN (1 << 0) +# define DYN_RR_EN (1 << 1) +# define G_DIV_ID(x) ((x) << 2) +# define G_DIV_ID_MASK (0x7 << 2) + /* * PM4 */ -#define PACKET_TYPE0 0 -#define PACKET_TYPE1 1 -#define PACKET_TYPE2 2 -#define PACKET_TYPE3 3 - -#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) -#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) -#define CP_PACKET0_GET_REG(h) (((h) & 0xFFFF) << 2) -#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) -#define PACKET0(reg, n) ((PACKET_TYPE0 << 30) | \ +#define PACKET0(reg, n) ((RADEON_PACKET_TYPE0 << 30) | \ (((reg) >> 2) & 0xFFFF) | \ ((n) & 0x3FFF) << 16) #define CP_PACKET2 0x80000000 @@ -808,7 +1590,7 @@ #define PACKET2(v) (CP_PACKET2 | REG_SET(PACKET2_PAD, (v))) -#define PACKET3(op, n) ((PACKET_TYPE3 << 30) | \ +#define PACKET3(op, n) ((RADEON_PACKET_TYPE3 << 30) | \ (((op) & 0xFF) << 8) | \ ((n) & 0x3FFF) << 16) @@ -870,6 +1652,23 @@ #define PACKET3_MPEG_INDEX 0x3A #define PACKET3_COPY_DW 0x3B #define PACKET3_WAIT_REG_MEM 0x3C +#define WAIT_REG_MEM_FUNCTION(x) ((x) << 0) + /* 0 - always + * 1 - < + * 2 - <= + * 3 - == + * 4 - != + * 5 - >= + * 6 - > + */ +#define WAIT_REG_MEM_MEM_SPACE(x) ((x) << 4) + /* 0 - reg + * 1 - mem + */ +#define WAIT_REG_MEM_ENGINE(x) ((x) << 8) + /* 0 - me + * 1 - pfp + */ #define PACKET3_MEM_WRITE 0x3D #define PACKET3_COPY_DATA 0x40 #define PACKET3_CP_DMA 0x41 @@ -1040,6 +1839,15 @@ # define DMA_IDLE (1 << 0) #define DMA_TILING_CONFIG 0xd0b8 +#define DMA_POWER_CNTL 0xd0bc +# define MEM_POWER_OVERRIDE (1 << 8) +#define DMA_CLK_CTRL 0xd0c0 + +#define DMA_PG 0xd0d4 +# define PG_CNTL_ENABLE (1 << 0) +#define DMA_PGFSM_CONFIG 0xd0d8 +#define DMA_PGFSM_WRITE 0xd0dc + #define DMA_PACKET(cmd, b, t, s, n) ((((cmd) & 0xF) << 28) | \ (((b) & 0x1) << 26) | \ (((t) & 0x1) << 23) | \ @@ -1064,6 +1872,83 @@ #define DMA_PACKET_TRAP 0x7 #define DMA_PACKET_SRBM_WRITE 0x9 #define DMA_PACKET_CONSTANT_FILL 0xd +#define DMA_PACKET_POLL_REG_MEM 0xe #define DMA_PACKET_NOP 0xf +#define VCE_STATUS 0x20004 +#define VCE_VCPU_CNTL 0x20014 +#define VCE_CLK_EN (1 << 0) +#define VCE_VCPU_CACHE_OFFSET0 0x20024 +#define VCE_VCPU_CACHE_SIZE0 0x20028 +#define VCE_VCPU_CACHE_OFFSET1 0x2002c +#define VCE_VCPU_CACHE_SIZE1 0x20030 +#define VCE_VCPU_CACHE_OFFSET2 0x20034 +#define VCE_VCPU_CACHE_SIZE2 0x20038 +#define VCE_VCPU_SCRATCH7 0x200dc +#define VCE_SOFT_RESET 0x20120 +#define VCE_ECPU_SOFT_RESET (1 << 0) +#define VCE_FME_SOFT_RESET (1 << 2) +#define VCE_RB_BASE_LO2 0x2016c +#define VCE_RB_BASE_HI2 0x20170 +#define VCE_RB_SIZE2 0x20174 +#define VCE_RB_RPTR2 0x20178 +#define VCE_RB_WPTR2 0x2017c +#define VCE_RB_BASE_LO 0x20180 +#define VCE_RB_BASE_HI 0x20184 +#define VCE_RB_SIZE 0x20188 +#define VCE_RB_RPTR 0x2018c +#define VCE_RB_WPTR 0x20190 +#define VCE_CLOCK_GATING_A 0x202f8 +# define CGC_DYN_CLOCK_MODE (1 << 16) +#define VCE_CLOCK_GATING_B 0x202fc +#define VCE_UENC_CLOCK_GATING 0x205bc +#define VCE_UENC_REG_CLOCK_GATING 0x205c0 +#define VCE_FW_REG_STATUS 0x20e10 +# define VCE_FW_REG_STATUS_BUSY (1 << 0) +# define VCE_FW_REG_STATUS_PASS (1 << 3) +# define VCE_FW_REG_STATUS_DONE (1 << 11) +#define VCE_LMI_FW_START_KEYSEL 0x20e18 +#define VCE_LMI_FW_PERIODIC_CTRL 0x20e20 +#define VCE_LMI_CTRL2 0x20e74 +#define VCE_LMI_CTRL 0x20e98 +#define VCE_LMI_VM_CTRL 0x20ea0 +#define VCE_LMI_SWAP_CNTL 0x20eb4 +#define VCE_LMI_SWAP_CNTL1 0x20eb8 +#define VCE_LMI_CACHE_CTRL 0x20ef4 + +#define VCE_CMD_NO_OP 0x00000000 +#define VCE_CMD_END 0x00000001 +#define VCE_CMD_IB 0x00000002 +#define VCE_CMD_FENCE 0x00000003 +#define VCE_CMD_TRAP 0x00000004 +#define VCE_CMD_IB_AUTO 0x00000005 +#define VCE_CMD_SEMAPHORE 0x00000006 + +/* discrete vce clocks */ +#define CG_VCEPLL_FUNC_CNTL 0xc0030600 +# define VCEPLL_RESET_MASK 0x00000001 +# define VCEPLL_SLEEP_MASK 0x00000002 +# define VCEPLL_BYPASS_EN_MASK 0x00000004 +# define VCEPLL_CTLREQ_MASK 0x00000008 +# define VCEPLL_VCO_MODE_MASK 0x00000600 +# define VCEPLL_REF_DIV_MASK 0x003F0000 +# define VCEPLL_CTLACK_MASK 0x40000000 +# define VCEPLL_CTLACK2_MASK 0x80000000 +#define CG_VCEPLL_FUNC_CNTL_2 0xc0030601 +# define VCEPLL_PDIV_A(x) ((x) << 0) +# define VCEPLL_PDIV_A_MASK 0x0000007F +# define VCEPLL_PDIV_B(x) ((x) << 8) +# define VCEPLL_PDIV_B_MASK 0x00007F00 +# define EVCLK_SRC_SEL(x) ((x) << 20) +# define EVCLK_SRC_SEL_MASK 0x01F00000 +# define ECCLK_SRC_SEL(x) ((x) << 25) +# define ECCLK_SRC_SEL_MASK 0x3E000000 +#define CG_VCEPLL_FUNC_CNTL_3 0xc0030602 +# define VCEPLL_FB_DIV(x) ((x) << 0) +# define VCEPLL_FB_DIV_MASK 0x01FFFFFF +#define CG_VCEPLL_FUNC_CNTL_4 0xc0030603 +#define CG_VCEPLL_FUNC_CNTL_5 0xc0030604 +#define CG_VCEPLL_SPREAD_SPECTRUM 0xc0030606 +# define VCEPLL_SSEN_MASK 0x00000001 + #endif diff --git a/sys/dev/pci/drm/radeon/sislands_smc.h b/sys/dev/pci/drm/radeon/sislands_smc.h new file mode 100644 index 00000000000..966e3a55601 --- /dev/null +++ b/sys/dev/pci/drm/radeon/sislands_smc.h @@ -0,0 +1,424 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef PP_SISLANDS_SMC_H +#define PP_SISLANDS_SMC_H + +#include "ppsmc.h" + +#pragma pack(push, 1) + +#define SISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE 16 + +struct PP_SIslands_Dpm2PerfLevel +{ + uint8_t MaxPS; + uint8_t TgtAct; + uint8_t MaxPS_StepInc; + uint8_t MaxPS_StepDec; + uint8_t PSSamplingTime; + uint8_t NearTDPDec; + uint8_t AboveSafeInc; + uint8_t BelowSafeInc; + uint8_t PSDeltaLimit; + uint8_t PSDeltaWin; + uint16_t PwrEfficiencyRatio; + uint8_t Reserved[4]; +}; + +typedef struct PP_SIslands_Dpm2PerfLevel PP_SIslands_Dpm2PerfLevel; + +struct PP_SIslands_DPM2Status +{ + uint32_t dpm2Flags; + uint8_t CurrPSkip; + uint8_t CurrPSkipPowerShift; + uint8_t CurrPSkipTDP; + uint8_t CurrPSkipOCP; + uint8_t MaxSPLLIndex; + uint8_t MinSPLLIndex; + uint8_t CurrSPLLIndex; + uint8_t InfSweepMode; + uint8_t InfSweepDir; + uint8_t TDPexceeded; + uint8_t reserved; + uint8_t SwitchDownThreshold; + uint32_t SwitchDownCounter; + uint32_t SysScalingFactor; +}; + +typedef struct PP_SIslands_DPM2Status PP_SIslands_DPM2Status; + +struct PP_SIslands_DPM2Parameters +{ + uint32_t TDPLimit; + uint32_t NearTDPLimit; + uint32_t SafePowerLimit; + uint32_t PowerBoostLimit; + uint32_t MinLimitDelta; +}; +typedef struct PP_SIslands_DPM2Parameters PP_SIslands_DPM2Parameters; + +struct PP_SIslands_PAPMStatus +{ + uint32_t EstimatedDGPU_T; + uint32_t EstimatedDGPU_P; + uint32_t EstimatedAPU_T; + uint32_t EstimatedAPU_P; + uint8_t dGPU_T_Limit_Exceeded; + uint8_t reserved[3]; +}; +typedef struct PP_SIslands_PAPMStatus PP_SIslands_PAPMStatus; + +struct PP_SIslands_PAPMParameters +{ + uint32_t NearTDPLimitTherm; + uint32_t NearTDPLimitPAPM; + uint32_t PlatformPowerLimit; + uint32_t dGPU_T_Limit; + uint32_t dGPU_T_Warning; + uint32_t dGPU_T_Hysteresis; +}; +typedef struct PP_SIslands_PAPMParameters PP_SIslands_PAPMParameters; + +struct SISLANDS_SMC_SCLK_VALUE +{ + uint32_t vCG_SPLL_FUNC_CNTL; + uint32_t vCG_SPLL_FUNC_CNTL_2; + uint32_t vCG_SPLL_FUNC_CNTL_3; + uint32_t vCG_SPLL_FUNC_CNTL_4; + uint32_t vCG_SPLL_SPREAD_SPECTRUM; + uint32_t vCG_SPLL_SPREAD_SPECTRUM_2; + uint32_t sclk_value; +}; + +typedef struct SISLANDS_SMC_SCLK_VALUE SISLANDS_SMC_SCLK_VALUE; + +struct SISLANDS_SMC_MCLK_VALUE +{ + uint32_t vMPLL_FUNC_CNTL; + uint32_t vMPLL_FUNC_CNTL_1; + uint32_t vMPLL_FUNC_CNTL_2; + uint32_t vMPLL_AD_FUNC_CNTL; + uint32_t vMPLL_DQ_FUNC_CNTL; + uint32_t vMCLK_PWRMGT_CNTL; + uint32_t vDLL_CNTL; + uint32_t vMPLL_SS; + uint32_t vMPLL_SS2; + uint32_t mclk_value; +}; + +typedef struct SISLANDS_SMC_MCLK_VALUE SISLANDS_SMC_MCLK_VALUE; + +struct SISLANDS_SMC_VOLTAGE_VALUE +{ + uint16_t value; + uint8_t index; + uint8_t phase_settings; +}; + +typedef struct SISLANDS_SMC_VOLTAGE_VALUE SISLANDS_SMC_VOLTAGE_VALUE; + +struct SISLANDS_SMC_HW_PERFORMANCE_LEVEL +{ + uint8_t ACIndex; + uint8_t displayWatermark; + uint8_t gen2PCIE; + uint8_t UVDWatermark; + uint8_t VCEWatermark; + uint8_t strobeMode; + uint8_t mcFlags; + uint8_t padding; + uint32_t aT; + uint32_t bSP; + SISLANDS_SMC_SCLK_VALUE sclk; + SISLANDS_SMC_MCLK_VALUE mclk; + SISLANDS_SMC_VOLTAGE_VALUE vddc; + SISLANDS_SMC_VOLTAGE_VALUE mvdd; + SISLANDS_SMC_VOLTAGE_VALUE vddci; + SISLANDS_SMC_VOLTAGE_VALUE std_vddc; + uint8_t hysteresisUp; + uint8_t hysteresisDown; + uint8_t stateFlags; + uint8_t arbRefreshState; + uint32_t SQPowerThrottle; + uint32_t SQPowerThrottle_2; + uint32_t MaxPoweredUpCU; + SISLANDS_SMC_VOLTAGE_VALUE high_temp_vddc; + SISLANDS_SMC_VOLTAGE_VALUE low_temp_vddc; + uint32_t reserved[2]; + PP_SIslands_Dpm2PerfLevel dpm2; +}; + +#define SISLANDS_SMC_STROBE_RATIO 0x0F +#define SISLANDS_SMC_STROBE_ENABLE 0x10 + +#define SISLANDS_SMC_MC_EDC_RD_FLAG 0x01 +#define SISLANDS_SMC_MC_EDC_WR_FLAG 0x02 +#define SISLANDS_SMC_MC_RTT_ENABLE 0x04 +#define SISLANDS_SMC_MC_STUTTER_EN 0x08 +#define SISLANDS_SMC_MC_PG_EN 0x10 + +typedef struct SISLANDS_SMC_HW_PERFORMANCE_LEVEL SISLANDS_SMC_HW_PERFORMANCE_LEVEL; + +struct SISLANDS_SMC_SWSTATE +{ + uint8_t flags; + uint8_t levelCount; + uint8_t padding2; + uint8_t padding3; + SISLANDS_SMC_HW_PERFORMANCE_LEVEL levels[1]; +}; + +typedef struct SISLANDS_SMC_SWSTATE SISLANDS_SMC_SWSTATE; + +#define SISLANDS_SMC_VOLTAGEMASK_VDDC 0 +#define SISLANDS_SMC_VOLTAGEMASK_MVDD 1 +#define SISLANDS_SMC_VOLTAGEMASK_VDDCI 2 +#define SISLANDS_SMC_VOLTAGEMASK_VDDC_PHASE_SHEDDING 3 +#define SISLANDS_SMC_VOLTAGEMASK_MAX 4 + +struct SISLANDS_SMC_VOLTAGEMASKTABLE +{ + uint32_t lowMask[SISLANDS_SMC_VOLTAGEMASK_MAX]; +}; + +typedef struct SISLANDS_SMC_VOLTAGEMASKTABLE SISLANDS_SMC_VOLTAGEMASKTABLE; + +#define SISLANDS_MAX_NO_VREG_STEPS 32 + +struct SISLANDS_SMC_STATETABLE +{ + uint8_t thermalProtectType; + uint8_t systemFlags; + uint8_t maxVDDCIndexInPPTable; + uint8_t extraFlags; + uint32_t lowSMIO[SISLANDS_MAX_NO_VREG_STEPS]; + SISLANDS_SMC_VOLTAGEMASKTABLE voltageMaskTable; + SISLANDS_SMC_VOLTAGEMASKTABLE phaseMaskTable; + PP_SIslands_DPM2Parameters dpm2Params; + SISLANDS_SMC_SWSTATE initialState; + SISLANDS_SMC_SWSTATE ACPIState; + SISLANDS_SMC_SWSTATE ULVState; + SISLANDS_SMC_SWSTATE driverState; + SISLANDS_SMC_HW_PERFORMANCE_LEVEL dpmLevels[SISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1]; +}; + +typedef struct SISLANDS_SMC_STATETABLE SISLANDS_SMC_STATETABLE; + +#define SI_SMC_SOFT_REGISTER_mclk_chg_timeout 0x0 +#define SI_SMC_SOFT_REGISTER_delay_vreg 0xC +#define SI_SMC_SOFT_REGISTER_delay_acpi 0x28 +#define SI_SMC_SOFT_REGISTER_seq_index 0x5C +#define SI_SMC_SOFT_REGISTER_mvdd_chg_time 0x60 +#define SI_SMC_SOFT_REGISTER_mclk_switch_lim 0x70 +#define SI_SMC_SOFT_REGISTER_watermark_threshold 0x78 +#define SI_SMC_SOFT_REGISTER_phase_shedding_delay 0x88 +#define SI_SMC_SOFT_REGISTER_ulv_volt_change_delay 0x8C +#define SI_SMC_SOFT_REGISTER_mc_block_delay 0x98 +#define SI_SMC_SOFT_REGISTER_ticks_per_us 0xA8 +#define SI_SMC_SOFT_REGISTER_crtc_index 0xC4 +#define SI_SMC_SOFT_REGISTER_mclk_change_block_cp_min 0xC8 +#define SI_SMC_SOFT_REGISTER_mclk_change_block_cp_max 0xCC +#define SI_SMC_SOFT_REGISTER_non_ulv_pcie_link_width 0xF4 +#define SI_SMC_SOFT_REGISTER_tdr_is_about_to_happen 0xFC +#define SI_SMC_SOFT_REGISTER_vr_hot_gpio 0x100 +#define SI_SMC_SOFT_REGISTER_svi_rework_plat_type 0x118 +#define SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svd 0x11c +#define SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svc 0x120 + +struct PP_SIslands_FanTable +{ + uint8_t fdo_mode; + uint8_t padding; + int16_t temp_min; + int16_t temp_med; + int16_t temp_max; + int16_t slope1; + int16_t slope2; + int16_t fdo_min; + int16_t hys_up; + int16_t hys_down; + int16_t hys_slope; + int16_t temp_resp_lim; + int16_t temp_curr; + int16_t slope_curr; + int16_t pwm_curr; + uint32_t refresh_period; + int16_t fdo_max; + uint8_t temp_src; + int8_t padding2; +}; + +typedef struct PP_SIslands_FanTable PP_SIslands_FanTable; + +#define SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16 +#define SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES 32 + +#define SMC_SISLANDS_SCALE_I 7 +#define SMC_SISLANDS_SCALE_R 12 + +struct PP_SIslands_CacConfig +{ + uint16_t cac_lkge_lut[SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES][SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES]; + uint32_t lkge_lut_V0; + uint32_t lkge_lut_Vstep; + uint32_t WinTime; + uint32_t R_LL; + uint32_t calculation_repeats; + uint32_t l2numWin_TDP; + uint32_t dc_cac; + uint8_t lts_truncate_n; + uint8_t SHIFT_N; + uint8_t log2_PG_LKG_SCALE; + uint8_t cac_temp; + uint32_t lkge_lut_T0; + uint32_t lkge_lut_Tstep; +}; + +typedef struct PP_SIslands_CacConfig PP_SIslands_CacConfig; + +#define SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE 16 +#define SMC_SISLANDS_MC_REGISTER_ARRAY_SET_COUNT 20 + +struct SMC_SIslands_MCRegisterAddress +{ + uint16_t s0; + uint16_t s1; +}; + +typedef struct SMC_SIslands_MCRegisterAddress SMC_SIslands_MCRegisterAddress; + +struct SMC_SIslands_MCRegisterSet +{ + uint32_t value[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE]; +}; + +typedef struct SMC_SIslands_MCRegisterSet SMC_SIslands_MCRegisterSet; + +struct SMC_SIslands_MCRegisters +{ + uint8_t last; + uint8_t reserved[3]; + SMC_SIslands_MCRegisterAddress address[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE]; + SMC_SIslands_MCRegisterSet data[SMC_SISLANDS_MC_REGISTER_ARRAY_SET_COUNT]; +}; + +typedef struct SMC_SIslands_MCRegisters SMC_SIslands_MCRegisters; + +struct SMC_SIslands_MCArbDramTimingRegisterSet +{ + uint32_t mc_arb_dram_timing; + uint32_t mc_arb_dram_timing2; + uint8_t mc_arb_rfsh_rate; + uint8_t mc_arb_burst_time; + uint8_t padding[2]; +}; + +typedef struct SMC_SIslands_MCArbDramTimingRegisterSet SMC_SIslands_MCArbDramTimingRegisterSet; + +struct SMC_SIslands_MCArbDramTimingRegisters +{ + uint8_t arb_current; + uint8_t reserved[3]; + SMC_SIslands_MCArbDramTimingRegisterSet data[16]; +}; + +typedef struct SMC_SIslands_MCArbDramTimingRegisters SMC_SIslands_MCArbDramTimingRegisters; + +struct SMC_SISLANDS_SPLL_DIV_TABLE +{ + uint32_t freq[256]; + uint32_t ss[256]; +}; + +#define SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_MASK 0x01ffffff +#define SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT 0 +#define SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_MASK 0xfe000000 +#define SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT 25 +#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_MASK 0x000fffff +#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT 0 +#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_MASK 0xfff00000 +#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT 20 + +typedef struct SMC_SISLANDS_SPLL_DIV_TABLE SMC_SISLANDS_SPLL_DIV_TABLE; + +#define SMC_SISLANDS_DTE_MAX_FILTER_STAGES 5 + +#define SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE 16 + +struct Smc_SIslands_DTE_Configuration +{ + uint32_t tau[SMC_SISLANDS_DTE_MAX_FILTER_STAGES]; + uint32_t R[SMC_SISLANDS_DTE_MAX_FILTER_STAGES]; + uint32_t K; + uint32_t T0; + uint32_t MaxT; + uint8_t WindowSize; + uint8_t Tdep_count; + uint8_t temp_select; + uint8_t DTE_mode; + uint8_t T_limits[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE]; + uint32_t Tdep_tau[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE]; + uint32_t Tdep_R[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE]; + uint32_t Tthreshold; +}; + +typedef struct Smc_SIslands_DTE_Configuration Smc_SIslands_DTE_Configuration; + +#define SMC_SISLANDS_DTE_STATUS_FLAG_DTE_ON 1 + +#define SISLANDS_SMC_FIRMWARE_HEADER_LOCATION 0x10000 + +#define SISLANDS_SMC_FIRMWARE_HEADER_version 0x0 +#define SISLANDS_SMC_FIRMWARE_HEADER_flags 0x4 +#define SISLANDS_SMC_FIRMWARE_HEADER_softRegisters 0xC +#define SISLANDS_SMC_FIRMWARE_HEADER_stateTable 0x10 +#define SISLANDS_SMC_FIRMWARE_HEADER_fanTable 0x14 +#define SISLANDS_SMC_FIRMWARE_HEADER_CacConfigTable 0x18 +#define SISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable 0x24 +#define SISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable 0x30 +#define SISLANDS_SMC_FIRMWARE_HEADER_spllTable 0x38 +#define SISLANDS_SMC_FIRMWARE_HEADER_DteConfiguration 0x40 +#define SISLANDS_SMC_FIRMWARE_HEADER_PAPMParameters 0x48 + +#pragma pack(pop) + +int si_copy_bytes_to_smc(struct radeon_device *rdev, + u32 smc_start_address, + const u8 *src, u32 byte_count, u32 limit); +void si_start_smc(struct radeon_device *rdev); +void si_reset_smc(struct radeon_device *rdev); +int si_program_jump_on_start(struct radeon_device *rdev); +void si_stop_smc_clock(struct radeon_device *rdev); +void si_start_smc_clock(struct radeon_device *rdev); +bool si_is_smc_running(struct radeon_device *rdev); +PPSMC_Result si_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg); +PPSMC_Result si_wait_for_smc_inactive(struct radeon_device *rdev); +int si_load_smc_ucode(struct radeon_device *rdev, u32 limit); +int si_read_smc_sram_dword(struct radeon_device *rdev, u32 smc_address, + u32 *value, u32 limit); +int si_write_smc_sram_dword(struct radeon_device *rdev, u32 smc_address, + u32 value, u32 limit); + +#endif + diff --git a/sys/dev/pci/drm/radeon/smu7.h b/sys/dev/pci/drm/radeon/smu7.h new file mode 100644 index 00000000000..75a380a1529 --- /dev/null +++ b/sys/dev/pci/drm/radeon/smu7.h @@ -0,0 +1,170 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef SMU7_H +#define SMU7_H + +#pragma pack(push, 1) + +#define SMU7_CONTEXT_ID_SMC 1 +#define SMU7_CONTEXT_ID_VBIOS 2 + + +#define SMU7_CONTEXT_ID_SMC 1 +#define SMU7_CONTEXT_ID_VBIOS 2 + +#define SMU7_MAX_LEVELS_VDDC 8 +#define SMU7_MAX_LEVELS_VDDCI 4 +#define SMU7_MAX_LEVELS_MVDD 4 +#define SMU7_MAX_LEVELS_VDDNB 8 + +#define SMU7_MAX_LEVELS_GRAPHICS SMU__NUM_SCLK_DPM_STATE // SCLK + SQ DPM + ULV +#define SMU7_MAX_LEVELS_MEMORY SMU__NUM_MCLK_DPM_LEVELS // MCLK Levels DPM +#define SMU7_MAX_LEVELS_GIO SMU__NUM_LCLK_DPM_LEVELS // LCLK Levels +#define SMU7_MAX_LEVELS_LINK SMU__NUM_PCIE_DPM_LEVELS // PCIe speed and number of lanes. +#define SMU7_MAX_LEVELS_UVD 8 // VCLK/DCLK levels for UVD. +#define SMU7_MAX_LEVELS_VCE 8 // ECLK levels for VCE. +#define SMU7_MAX_LEVELS_ACP 8 // ACLK levels for ACP. +#define SMU7_MAX_LEVELS_SAMU 8 // SAMCLK levels for SAMU. +#define SMU7_MAX_ENTRIES_SMIO 32 // Number of entries in SMIO table. + +#define DPM_NO_LIMIT 0 +#define DPM_NO_UP 1 +#define DPM_GO_DOWN 2 +#define DPM_GO_UP 3 + +#define SMU7_FIRST_DPM_GRAPHICS_LEVEL 0 +#define SMU7_FIRST_DPM_MEMORY_LEVEL 0 + +#define GPIO_CLAMP_MODE_VRHOT 1 +#define GPIO_CLAMP_MODE_THERM 2 +#define GPIO_CLAMP_MODE_DC 4 + +#define SCRATCH_B_TARG_PCIE_INDEX_SHIFT 0 +#define SCRATCH_B_TARG_PCIE_INDEX_MASK (0x7< +#include "radeon.h" +#include "radeon_asic.h" +#include "sumod.h" +#include "r600_dpm.h" +#include "cypress_dpm.h" +#include "sumo_dpm.h" + +#define SUMO_MAX_DEEPSLEEP_DIVIDER_ID 5 +#define SUMO_MINIMUM_ENGINE_CLOCK 800 +#define BOOST_DPM_LEVEL 7 + +static const u32 sumo_utc[SUMO_PM_NUMBER_OF_TC] = +{ + SUMO_UTC_DFLT_00, + SUMO_UTC_DFLT_01, + SUMO_UTC_DFLT_02, + SUMO_UTC_DFLT_03, + SUMO_UTC_DFLT_04, + SUMO_UTC_DFLT_05, + SUMO_UTC_DFLT_06, + SUMO_UTC_DFLT_07, + SUMO_UTC_DFLT_08, + SUMO_UTC_DFLT_09, + SUMO_UTC_DFLT_10, + SUMO_UTC_DFLT_11, + SUMO_UTC_DFLT_12, + SUMO_UTC_DFLT_13, + SUMO_UTC_DFLT_14, +}; + +static const u32 sumo_dtc[SUMO_PM_NUMBER_OF_TC] = +{ + SUMO_DTC_DFLT_00, + SUMO_DTC_DFLT_01, + SUMO_DTC_DFLT_02, + SUMO_DTC_DFLT_03, + SUMO_DTC_DFLT_04, + SUMO_DTC_DFLT_05, + SUMO_DTC_DFLT_06, + SUMO_DTC_DFLT_07, + SUMO_DTC_DFLT_08, + SUMO_DTC_DFLT_09, + SUMO_DTC_DFLT_10, + SUMO_DTC_DFLT_11, + SUMO_DTC_DFLT_12, + SUMO_DTC_DFLT_13, + SUMO_DTC_DFLT_14, +}; + +static struct sumo_ps *sumo_get_ps(struct radeon_ps *rps) +{ + struct sumo_ps *ps = rps->ps_priv; + + return ps; +} + +struct sumo_power_info *sumo_get_pi(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +static void sumo_gfx_clockgating_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN); + else { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN); + WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON); + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON); + RREG32(GB_ADDR_CONFIG); + } +} + +#define CGCG_CGTT_LOCAL0_MASK 0xE5BFFFFF +#define CGCG_CGTT_LOCAL1_MASK 0xEFFF07FF + +static void sumo_mg_clockgating_enable(struct radeon_device *rdev, bool enable) +{ + u32 local0; + u32 local1; + + local0 = RREG32(CG_CGTT_LOCAL_0); + local1 = RREG32(CG_CGTT_LOCAL_1); + + if (enable) { + WREG32(CG_CGTT_LOCAL_0, (0 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) ); + WREG32(CG_CGTT_LOCAL_1, (0 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) ); + } else { + WREG32(CG_CGTT_LOCAL_0, (0xFFFFFFFF & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) ); + WREG32(CG_CGTT_LOCAL_1, (0xFFFFCFFF & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) ); + } +} + +static void sumo_program_git(struct radeon_device *rdev) +{ + u32 p, u; + u32 xclk = radeon_get_xclk(rdev); + + r600_calculate_u_and_p(SUMO_GICST_DFLT, + xclk, 16, &p, &u); + + WREG32_P(CG_GIT, CG_GICST(p), ~CG_GICST_MASK); +} + +static void sumo_program_grsd(struct radeon_device *rdev) +{ + u32 p, u; + u32 xclk = radeon_get_xclk(rdev); + u32 grs = 256 * 25 / 100; + + r600_calculate_u_and_p(1, xclk, 14, &p, &u); + + WREG32(CG_GCOOR, PHC(grs) | SDC(p) | SU(u)); +} + +void sumo_gfx_clockgating_initialize(struct radeon_device *rdev) +{ + sumo_program_git(rdev); + sumo_program_grsd(rdev); +} + +static void sumo_gfx_powergating_initialize(struct radeon_device *rdev) +{ + u32 rcu_pwr_gating_cntl; + u32 p, u; + u32 p_c, p_p, d_p; + u32 r_t, i_t; + u32 xclk = radeon_get_xclk(rdev); + + if (rdev->family == CHIP_PALM) { + p_c = 4; + d_p = 10; + r_t = 10; + i_t = 4; + p_p = 50 + 1000/200 + 6 * 32; + } else { + p_c = 16; + d_p = 50; + r_t = 50; + i_t = 50; + p_p = 113; + } + + WREG32(CG_SCRATCH2, 0x01B60A17); + + r600_calculate_u_and_p(SUMO_GFXPOWERGATINGT_DFLT, + xclk, 16, &p, &u); + + WREG32_P(CG_PWR_GATING_CNTL, PGP(p) | PGU(u), + ~(PGP_MASK | PGU_MASK)); + + r600_calculate_u_and_p(SUMO_VOLTAGEDROPT_DFLT, + xclk, 16, &p, &u); + + WREG32_P(CG_CG_VOLTAGE_CNTL, PGP(p) | PGU(u), + ~(PGP_MASK | PGU_MASK)); + + if (rdev->family == CHIP_PALM) { + WREG32_RCU(RCU_PWR_GATING_SEQ0, 0x10103210); + WREG32_RCU(RCU_PWR_GATING_SEQ1, 0x10101010); + } else { + WREG32_RCU(RCU_PWR_GATING_SEQ0, 0x76543210); + WREG32_RCU(RCU_PWR_GATING_SEQ1, 0xFEDCBA98); + } + + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL); + rcu_pwr_gating_cntl &= + ~(RSVD_MASK | PCV_MASK | PGS_MASK); + rcu_pwr_gating_cntl |= PCV(p_c) | PGS(1) | PWR_GATING_EN; + if (rdev->family == CHIP_PALM) { + rcu_pwr_gating_cntl &= ~PCP_MASK; + rcu_pwr_gating_cntl |= PCP(0x77); + } + WREG32_RCU(RCU_PWR_GATING_CNTL, rcu_pwr_gating_cntl); + + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_2); + rcu_pwr_gating_cntl &= ~(MPPU_MASK | MPPD_MASK); + rcu_pwr_gating_cntl |= MPPU(p_p) | MPPD(50); + WREG32_RCU(RCU_PWR_GATING_CNTL_2, rcu_pwr_gating_cntl); + + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_3); + rcu_pwr_gating_cntl &= ~(DPPU_MASK | DPPD_MASK); + rcu_pwr_gating_cntl |= DPPU(d_p) | DPPD(50); + WREG32_RCU(RCU_PWR_GATING_CNTL_3, rcu_pwr_gating_cntl); + + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_4); + rcu_pwr_gating_cntl &= ~(RT_MASK | IT_MASK); + rcu_pwr_gating_cntl |= RT(r_t) | IT(i_t); + WREG32_RCU(RCU_PWR_GATING_CNTL_4, rcu_pwr_gating_cntl); + + if (rdev->family == CHIP_PALM) + WREG32_RCU(RCU_PWR_GATING_CNTL_5, 0xA02); + + sumo_smu_pg_init(rdev); + + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL); + rcu_pwr_gating_cntl &= + ~(RSVD_MASK | PCV_MASK | PGS_MASK); + rcu_pwr_gating_cntl |= PCV(p_c) | PGS(4) | PWR_GATING_EN; + if (rdev->family == CHIP_PALM) { + rcu_pwr_gating_cntl &= ~PCP_MASK; + rcu_pwr_gating_cntl |= PCP(0x77); + } + WREG32_RCU(RCU_PWR_GATING_CNTL, rcu_pwr_gating_cntl); + + if (rdev->family == CHIP_PALM) { + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_2); + rcu_pwr_gating_cntl &= ~(MPPU_MASK | MPPD_MASK); + rcu_pwr_gating_cntl |= MPPU(113) | MPPD(50); + WREG32_RCU(RCU_PWR_GATING_CNTL_2, rcu_pwr_gating_cntl); + + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_3); + rcu_pwr_gating_cntl &= ~(DPPU_MASK | DPPD_MASK); + rcu_pwr_gating_cntl |= DPPU(16) | DPPD(50); + WREG32_RCU(RCU_PWR_GATING_CNTL_3, rcu_pwr_gating_cntl); + } + + sumo_smu_pg_init(rdev); + + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL); + rcu_pwr_gating_cntl &= + ~(RSVD_MASK | PCV_MASK | PGS_MASK); + rcu_pwr_gating_cntl |= PGS(5) | PWR_GATING_EN; + + if (rdev->family == CHIP_PALM) { + rcu_pwr_gating_cntl |= PCV(4); + rcu_pwr_gating_cntl &= ~PCP_MASK; + rcu_pwr_gating_cntl |= PCP(0x77); + } else + rcu_pwr_gating_cntl |= PCV(11); + WREG32_RCU(RCU_PWR_GATING_CNTL, rcu_pwr_gating_cntl); + + if (rdev->family == CHIP_PALM) { + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_2); + rcu_pwr_gating_cntl &= ~(MPPU_MASK | MPPD_MASK); + rcu_pwr_gating_cntl |= MPPU(113) | MPPD(50); + WREG32_RCU(RCU_PWR_GATING_CNTL_2, rcu_pwr_gating_cntl); + + rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_3); + rcu_pwr_gating_cntl &= ~(DPPU_MASK | DPPD_MASK); + rcu_pwr_gating_cntl |= DPPU(22) | DPPD(50); + WREG32_RCU(RCU_PWR_GATING_CNTL_3, rcu_pwr_gating_cntl); + } + + sumo_smu_pg_init(rdev); +} + +static void sumo_gfx_powergating_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(CG_PWR_GATING_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN); + else { + WREG32_P(CG_PWR_GATING_CNTL, 0, ~DYN_PWR_DOWN_EN); + RREG32(GB_ADDR_CONFIG); + } +} + +static int sumo_enable_clock_power_gating(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + if (pi->enable_gfx_clock_gating) + sumo_gfx_clockgating_initialize(rdev); + if (pi->enable_gfx_power_gating) + sumo_gfx_powergating_initialize(rdev); + if (pi->enable_mg_clock_gating) + sumo_mg_clockgating_enable(rdev, true); + if (pi->enable_gfx_clock_gating) + sumo_gfx_clockgating_enable(rdev, true); + if (pi->enable_gfx_power_gating) + sumo_gfx_powergating_enable(rdev, true); + + return 0; +} + +static void sumo_disable_clock_power_gating(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + if (pi->enable_gfx_clock_gating) + sumo_gfx_clockgating_enable(rdev, false); + if (pi->enable_gfx_power_gating) + sumo_gfx_powergating_enable(rdev, false); + if (pi->enable_mg_clock_gating) + sumo_mg_clockgating_enable(rdev, false); +} + +static void sumo_calculate_bsp(struct radeon_device *rdev, + u32 high_clk) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 xclk = radeon_get_xclk(rdev); + + pi->pasi = 65535 * 100 / high_clk; + pi->asi = 65535 * 100 / high_clk; + + r600_calculate_u_and_p(pi->asi, + xclk, 16, &pi->bsp, &pi->bsu); + + r600_calculate_u_and_p(pi->pasi, + xclk, 16, &pi->pbsp, &pi->pbsu); + + pi->dsp = BSP(pi->bsp) | BSU(pi->bsu); + pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu); +} + +static void sumo_init_bsp(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + WREG32(CG_BSP_0, pi->psp); +} + + +static void sumo_program_bsp(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct sumo_ps *ps = sumo_get_ps(rps); + u32 i; + u32 highest_engine_clock = ps->levels[ps->num_levels - 1].sclk; + + if (ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE) + highest_engine_clock = pi->boost_pl.sclk; + + sumo_calculate_bsp(rdev, highest_engine_clock); + + for (i = 0; i < ps->num_levels - 1; i++) + WREG32(CG_BSP_0 + (i * 4), pi->dsp); + + WREG32(CG_BSP_0 + (i * 4), pi->psp); + + if (ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE) + WREG32(CG_BSP_0 + (BOOST_DPM_LEVEL * 4), pi->psp); +} + +static void sumo_write_at(struct radeon_device *rdev, + u32 index, u32 value) +{ + if (index == 0) + WREG32(CG_AT_0, value); + else if (index == 1) + WREG32(CG_AT_1, value); + else if (index == 2) + WREG32(CG_AT_2, value); + else if (index == 3) + WREG32(CG_AT_3, value); + else if (index == 4) + WREG32(CG_AT_4, value); + else if (index == 5) + WREG32(CG_AT_5, value); + else if (index == 6) + WREG32(CG_AT_6, value); + else if (index == 7) + WREG32(CG_AT_7, value); +} + +static void sumo_program_at(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct sumo_ps *ps = sumo_get_ps(rps); + u32 asi; + u32 i; + u32 m_a; + u32 a_t; + u32 r[SUMO_MAX_HARDWARE_POWERLEVELS]; + u32 l[SUMO_MAX_HARDWARE_POWERLEVELS]; + + r[0] = SUMO_R_DFLT0; + r[1] = SUMO_R_DFLT1; + r[2] = SUMO_R_DFLT2; + r[3] = SUMO_R_DFLT3; + r[4] = SUMO_R_DFLT4; + + l[0] = SUMO_L_DFLT0; + l[1] = SUMO_L_DFLT1; + l[2] = SUMO_L_DFLT2; + l[3] = SUMO_L_DFLT3; + l[4] = SUMO_L_DFLT4; + + for (i = 0; i < ps->num_levels; i++) { + asi = (i == ps->num_levels - 1) ? pi->pasi : pi->asi; + + m_a = asi * ps->levels[i].sclk / 100; + + a_t = CG_R(m_a * r[i] / 100) | CG_L(m_a * l[i] / 100); + + sumo_write_at(rdev, i, a_t); + } + + if (ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE) { + asi = pi->pasi; + + m_a = asi * pi->boost_pl.sclk / 100; + + a_t = CG_R(m_a * r[ps->num_levels - 1] / 100) | + CG_L(m_a * l[ps->num_levels - 1] / 100); + + sumo_write_at(rdev, BOOST_DPM_LEVEL, a_t); + } +} + +static void sumo_program_tp(struct radeon_device *rdev) +{ + int i; + enum r600_td td = R600_TD_DFLT; + + for (i = 0; i < SUMO_PM_NUMBER_OF_TC; i++) { + WREG32_P(CG_FFCT_0 + (i * 4), UTC_0(sumo_utc[i]), ~UTC_0_MASK); + WREG32_P(CG_FFCT_0 + (i * 4), DTC_0(sumo_dtc[i]), ~DTC_0_MASK); + } + + if (td == R600_TD_AUTO) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL); + else + WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL); + + if (td == R600_TD_UP) + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE); + + if (td == R600_TD_DOWN) + WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE); +} + +void sumo_program_vc(struct radeon_device *rdev, u32 vrc) +{ + WREG32(CG_FTV, vrc); +} + +void sumo_clear_vc(struct radeon_device *rdev) +{ + WREG32(CG_FTV, 0); +} + +void sumo_program_sstp(struct radeon_device *rdev) +{ + u32 p, u; + u32 xclk = radeon_get_xclk(rdev); + + r600_calculate_u_and_p(SUMO_SST_DFLT, + xclk, 16, &p, &u); + + WREG32(CG_SSP, SSTU(u) | SST(p)); +} + +static void sumo_set_divider_value(struct radeon_device *rdev, + u32 index, u32 divider) +{ + u32 reg_index = index / 4; + u32 field_index = index % 4; + + if (field_index == 0) + WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4), + SCLK_FSTATE_0_DIV(divider), ~SCLK_FSTATE_0_DIV_MASK); + else if (field_index == 1) + WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4), + SCLK_FSTATE_1_DIV(divider), ~SCLK_FSTATE_1_DIV_MASK); + else if (field_index == 2) + WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4), + SCLK_FSTATE_2_DIV(divider), ~SCLK_FSTATE_2_DIV_MASK); + else if (field_index == 3) + WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4), + SCLK_FSTATE_3_DIV(divider), ~SCLK_FSTATE_3_DIV_MASK); +} + +static void sumo_set_ds_dividers(struct radeon_device *rdev, + u32 index, u32 divider) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + if (pi->enable_sclk_ds) { + u32 dpm_ctrl = RREG32(CG_SCLK_DPM_CTRL_6); + + dpm_ctrl &= ~(0x7 << (index * 3)); + dpm_ctrl |= (divider << (index * 3)); + WREG32(CG_SCLK_DPM_CTRL_6, dpm_ctrl); + } +} + +static void sumo_set_ss_dividers(struct radeon_device *rdev, + u32 index, u32 divider) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + if (pi->enable_sclk_ds) { + u32 dpm_ctrl = RREG32(CG_SCLK_DPM_CTRL_11); + + dpm_ctrl &= ~(0x7 << (index * 3)); + dpm_ctrl |= (divider << (index * 3)); + WREG32(CG_SCLK_DPM_CTRL_11, dpm_ctrl); + } +} + +static void sumo_set_vid(struct radeon_device *rdev, u32 index, u32 vid) +{ + u32 voltage_cntl = RREG32(CG_DPM_VOLTAGE_CNTL); + + voltage_cntl &= ~(DPM_STATE0_LEVEL_MASK << (index * 2)); + voltage_cntl |= (vid << (DPM_STATE0_LEVEL_SHIFT + index * 2)); + WREG32(CG_DPM_VOLTAGE_CNTL, voltage_cntl); +} + +static void sumo_set_allos_gnb_slow(struct radeon_device *rdev, u32 index, u32 gnb_slow) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 temp = gnb_slow; + u32 cg_sclk_dpm_ctrl_3; + + if (pi->driver_nbps_policy_disable) + temp = 1; + + cg_sclk_dpm_ctrl_3 = RREG32(CG_SCLK_DPM_CTRL_3); + cg_sclk_dpm_ctrl_3 &= ~(GNB_SLOW_FSTATE_0_MASK << index); + cg_sclk_dpm_ctrl_3 |= (temp << (GNB_SLOW_FSTATE_0_SHIFT + index)); + + WREG32(CG_SCLK_DPM_CTRL_3, cg_sclk_dpm_ctrl_3); +} + +static void sumo_program_power_level(struct radeon_device *rdev, + struct sumo_pl *pl, u32 index) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + int ret; + struct atom_clock_dividers dividers; + u32 ds_en = RREG32(DEEP_SLEEP_CNTL) & ENABLE_DS; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + pl->sclk, false, ÷rs); + if (ret) + return; + + sumo_set_divider_value(rdev, index, dividers.post_div); + + sumo_set_vid(rdev, index, pl->vddc_index); + + if (pl->ss_divider_index == 0 || pl->ds_divider_index == 0) { + if (ds_en) + WREG32_P(DEEP_SLEEP_CNTL, 0, ~ENABLE_DS); + } else { + sumo_set_ss_dividers(rdev, index, pl->ss_divider_index); + sumo_set_ds_dividers(rdev, index, pl->ds_divider_index); + + if (!ds_en) + WREG32_P(DEEP_SLEEP_CNTL, ENABLE_DS, ~ENABLE_DS); + } + + sumo_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow); + + if (pi->enable_boost) + sumo_set_tdp_limit(rdev, index, pl->sclk_dpm_tdp_limit); +} + +static void sumo_power_level_enable(struct radeon_device *rdev, u32 index, bool enable) +{ + u32 reg_index = index / 4; + u32 field_index = index % 4; + + if (field_index == 0) + WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4), + enable ? SCLK_FSTATE_0_VLD : 0, ~SCLK_FSTATE_0_VLD); + else if (field_index == 1) + WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4), + enable ? SCLK_FSTATE_1_VLD : 0, ~SCLK_FSTATE_1_VLD); + else if (field_index == 2) + WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4), + enable ? SCLK_FSTATE_2_VLD : 0, ~SCLK_FSTATE_2_VLD); + else if (field_index == 3) + WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4), + enable ? SCLK_FSTATE_3_VLD : 0, ~SCLK_FSTATE_3_VLD); +} + +static bool sumo_dpm_enabled(struct radeon_device *rdev) +{ + if (RREG32(CG_SCLK_DPM_CTRL_3) & DPM_SCLK_ENABLE) + return true; + else + return false; +} + +static void sumo_start_dpm(struct radeon_device *rdev) +{ + WREG32_P(CG_SCLK_DPM_CTRL_3, DPM_SCLK_ENABLE, ~DPM_SCLK_ENABLE); +} + +static void sumo_stop_dpm(struct radeon_device *rdev) +{ + WREG32_P(CG_SCLK_DPM_CTRL_3, 0, ~DPM_SCLK_ENABLE); +} + +static void sumo_set_forced_mode(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_P(CG_SCLK_DPM_CTRL_3, FORCE_SCLK_STATE_EN, ~FORCE_SCLK_STATE_EN); + else + WREG32_P(CG_SCLK_DPM_CTRL_3, 0, ~FORCE_SCLK_STATE_EN); +} + +static void sumo_set_forced_mode_enabled(struct radeon_device *rdev) +{ + int i; + + sumo_set_forced_mode(rdev, true); + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(CG_SCLK_STATUS) & SCLK_OVERCLK_DETECT) + break; + udelay(1); + } +} + +static void sumo_wait_for_level_0(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURR_SCLK_INDEX_MASK) == 0) + break; + udelay(1); + } + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURR_INDEX_MASK) == 0) + break; + udelay(1); + } +} + +static void sumo_set_forced_mode_disabled(struct radeon_device *rdev) +{ + sumo_set_forced_mode(rdev, false); +} + +static void sumo_enable_power_level_0(struct radeon_device *rdev) +{ + sumo_power_level_enable(rdev, 0, true); +} + +static void sumo_patch_boost_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct sumo_ps *new_ps = sumo_get_ps(rps); + + if (new_ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE) { + pi->boost_pl = new_ps->levels[new_ps->num_levels - 1]; + pi->boost_pl.sclk = pi->sys_info.boost_sclk; + pi->boost_pl.vddc_index = pi->sys_info.boost_vid_2bit; + pi->boost_pl.sclk_dpm_tdp_limit = pi->sys_info.sclk_dpm_tdp_limit_boost; + } +} + +static void sumo_pre_notify_alt_vddnb_change(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct sumo_ps *new_ps = sumo_get_ps(new_rps); + struct sumo_ps *old_ps = sumo_get_ps(old_rps); + u32 nbps1_old = 0; + u32 nbps1_new = 0; + + if (old_ps != NULL) + nbps1_old = (old_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE) ? 1 : 0; + + nbps1_new = (new_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE) ? 1 : 0; + + if (nbps1_old == 1 && nbps1_new == 0) + sumo_smu_notify_alt_vddnb_change(rdev, 0, 0); +} + +static void sumo_post_notify_alt_vddnb_change(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct sumo_ps *new_ps = sumo_get_ps(new_rps); + struct sumo_ps *old_ps = sumo_get_ps(old_rps); + u32 nbps1_old = 0; + u32 nbps1_new = 0; + + if (old_ps != NULL) + nbps1_old = (old_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE)? 1 : 0; + + nbps1_new = (new_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE)? 1 : 0; + + if (nbps1_old == 0 && nbps1_new == 1) + sumo_smu_notify_alt_vddnb_change(rdev, 1, 1); +} + +static void sumo_enable_boost(struct radeon_device *rdev, + struct radeon_ps *rps, + bool enable) +{ + struct sumo_ps *new_ps = sumo_get_ps(rps); + + if (enable) { + if (new_ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE) + sumo_boost_state_enable(rdev, true); + } else + sumo_boost_state_enable(rdev, false); +} + +static void sumo_set_forced_level(struct radeon_device *rdev, u32 index) +{ + WREG32_P(CG_SCLK_DPM_CTRL_3, FORCE_SCLK_STATE(index), ~FORCE_SCLK_STATE_MASK); +} + +static void sumo_set_forced_level_0(struct radeon_device *rdev) +{ + sumo_set_forced_level(rdev, 0); +} + +static void sumo_program_wl(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct sumo_ps *new_ps = sumo_get_ps(rps); + u32 dpm_ctrl4 = RREG32(CG_SCLK_DPM_CTRL_4); + + dpm_ctrl4 &= 0xFFFFFF00; + dpm_ctrl4 |= (1 << (new_ps->num_levels - 1)); + + if (new_ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE) + dpm_ctrl4 |= (1 << BOOST_DPM_LEVEL); + + WREG32(CG_SCLK_DPM_CTRL_4, dpm_ctrl4); +} + +static void sumo_program_power_levels_0_to_n(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct sumo_ps *new_ps = sumo_get_ps(new_rps); + struct sumo_ps *old_ps = sumo_get_ps(old_rps); + u32 i; + u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels; + + for (i = 0; i < new_ps->num_levels; i++) { + sumo_program_power_level(rdev, &new_ps->levels[i], i); + sumo_power_level_enable(rdev, i, true); + } + + for (i = new_ps->num_levels; i < n_current_state_levels; i++) + sumo_power_level_enable(rdev, i, false); + + if (new_ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE) + sumo_program_power_level(rdev, &pi->boost_pl, BOOST_DPM_LEVEL); +} + +static void sumo_enable_acpi_pm(struct radeon_device *rdev) +{ + WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN); +} + +static void sumo_program_power_level_enter_state(struct radeon_device *rdev) +{ + WREG32_P(CG_SCLK_DPM_CTRL_5, SCLK_FSTATE_BOOTUP(0), ~SCLK_FSTATE_BOOTUP_MASK); +} + +static void sumo_program_acpi_power_level(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct atom_clock_dividers dividers; + int ret; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + pi->acpi_pl.sclk, + false, ÷rs); + if (ret) + return; + + WREG32_P(CG_ACPI_CNTL, SCLK_ACPI_DIV(dividers.post_div), ~SCLK_ACPI_DIV_MASK); + WREG32_P(CG_ACPI_VOLTAGE_CNTL, 0, ~ACPI_VOLTAGE_EN); +} + +static void sumo_program_bootup_state(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 dpm_ctrl4 = RREG32(CG_SCLK_DPM_CTRL_4); + u32 i; + + sumo_program_power_level(rdev, &pi->boot_pl, 0); + + dpm_ctrl4 &= 0xFFFFFF00; + WREG32(CG_SCLK_DPM_CTRL_4, dpm_ctrl4); + + for (i = 1; i < 8; i++) + sumo_power_level_enable(rdev, i, false); +} + +static void sumo_setup_uvd_clocks(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + if (pi->enable_gfx_power_gating) { + sumo_gfx_powergating_enable(rdev, false); + } + + radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk); + + if (pi->enable_gfx_power_gating) { + if (!pi->disable_gfx_power_gating_in_uvd || + !r600_is_uvd_state(new_rps->class, new_rps->class2)) + sumo_gfx_powergating_enable(rdev, true); + } +} + +static void sumo_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct sumo_ps *new_ps = sumo_get_ps(new_rps); + struct sumo_ps *current_ps = sumo_get_ps(old_rps); + + if ((new_rps->vclk == old_rps->vclk) && + (new_rps->dclk == old_rps->dclk)) + return; + + if (new_ps->levels[new_ps->num_levels - 1].sclk >= + current_ps->levels[current_ps->num_levels - 1].sclk) + return; + + sumo_setup_uvd_clocks(rdev, new_rps, old_rps); +} + +static void sumo_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct sumo_ps *new_ps = sumo_get_ps(new_rps); + struct sumo_ps *current_ps = sumo_get_ps(old_rps); + + if ((new_rps->vclk == old_rps->vclk) && + (new_rps->dclk == old_rps->dclk)) + return; + + if (new_ps->levels[new_ps->num_levels - 1].sclk < + current_ps->levels[current_ps->num_levels - 1].sclk) + return; + + sumo_setup_uvd_clocks(rdev, new_rps, old_rps); +} + +void sumo_take_smu_control(struct radeon_device *rdev, bool enable) +{ +/* This bit selects who handles display phy powergating. + * Clear the bit to let atom handle it. + * Set it to let the driver handle it. + * For now we just let atom handle it. + */ +#if 0 + u32 v = RREG32(DOUT_SCRATCH3); + + if (enable) + v |= 0x4; + else + v &= 0xFFFFFFFB; + + WREG32(DOUT_SCRATCH3, v); +#endif +} + +static void sumo_enable_sclk_ds(struct radeon_device *rdev, bool enable) +{ + if (enable) { + u32 deep_sleep_cntl = RREG32(DEEP_SLEEP_CNTL); + u32 deep_sleep_cntl2 = RREG32(DEEP_SLEEP_CNTL2); + u32 t = 1; + + deep_sleep_cntl &= ~R_DIS; + deep_sleep_cntl &= ~HS_MASK; + deep_sleep_cntl |= HS(t > 4095 ? 4095 : t); + + deep_sleep_cntl2 |= LB_UFP_EN; + deep_sleep_cntl2 &= INOUT_C_MASK; + deep_sleep_cntl2 |= INOUT_C(0xf); + + WREG32(DEEP_SLEEP_CNTL2, deep_sleep_cntl2); + WREG32(DEEP_SLEEP_CNTL, deep_sleep_cntl); + } else + WREG32_P(DEEP_SLEEP_CNTL, 0, ~ENABLE_DS); +} + +static void sumo_program_bootup_at(struct radeon_device *rdev) +{ + WREG32_P(CG_AT_0, CG_R(0xffff), ~CG_R_MASK); + WREG32_P(CG_AT_0, CG_L(0), ~CG_L_MASK); +} + +static void sumo_reset_am(struct radeon_device *rdev) +{ + WREG32_P(SCLK_PWRMGT_CNTL, FIR_RESET, ~FIR_RESET); +} + +static void sumo_start_am(struct radeon_device *rdev) +{ + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_RESET); +} + +static void sumo_program_ttp(struct radeon_device *rdev) +{ + u32 xclk = radeon_get_xclk(rdev); + u32 p, u; + u32 cg_sclk_dpm_ctrl_5 = RREG32(CG_SCLK_DPM_CTRL_5); + + r600_calculate_u_and_p(1000, + xclk, 16, &p, &u); + + cg_sclk_dpm_ctrl_5 &= ~(TT_TP_MASK | TT_TU_MASK); + cg_sclk_dpm_ctrl_5 |= TT_TP(p) | TT_TU(u); + + WREG32(CG_SCLK_DPM_CTRL_5, cg_sclk_dpm_ctrl_5); +} + +static void sumo_program_ttt(struct radeon_device *rdev) +{ + u32 cg_sclk_dpm_ctrl_3 = RREG32(CG_SCLK_DPM_CTRL_3); + struct sumo_power_info *pi = sumo_get_pi(rdev); + + cg_sclk_dpm_ctrl_3 &= ~(GNB_TT_MASK | GNB_THERMTHRO_MASK); + cg_sclk_dpm_ctrl_3 |= GNB_TT(pi->thermal_auto_throttling + 49); + + WREG32(CG_SCLK_DPM_CTRL_3, cg_sclk_dpm_ctrl_3); +} + + +static void sumo_enable_voltage_scaling(struct radeon_device *rdev, bool enable) +{ + if (enable) { + WREG32_P(CG_DPM_VOLTAGE_CNTL, DPM_VOLTAGE_EN, ~DPM_VOLTAGE_EN); + WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~CG_VOLTAGE_EN); + } else { + WREG32_P(CG_CG_VOLTAGE_CNTL, CG_VOLTAGE_EN, ~CG_VOLTAGE_EN); + WREG32_P(CG_DPM_VOLTAGE_CNTL, 0, ~DPM_VOLTAGE_EN); + } +} + +static void sumo_override_cnb_thermal_events(struct radeon_device *rdev) +{ + WREG32_P(CG_SCLK_DPM_CTRL_3, CNB_THERMTHRO_MASK_SCLK, + ~CNB_THERMTHRO_MASK_SCLK); +} + +static void sumo_program_dc_hto(struct radeon_device *rdev) +{ + u32 cg_sclk_dpm_ctrl_4 = RREG32(CG_SCLK_DPM_CTRL_4); + u32 p, u; + u32 xclk = radeon_get_xclk(rdev); + + r600_calculate_u_and_p(100000, + xclk, 14, &p, &u); + + cg_sclk_dpm_ctrl_4 &= ~(DC_HDC_MASK | DC_HU_MASK); + cg_sclk_dpm_ctrl_4 |= DC_HDC(p) | DC_HU(u); + + WREG32(CG_SCLK_DPM_CTRL_4, cg_sclk_dpm_ctrl_4); +} + +static void sumo_force_nbp_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct sumo_ps *new_ps = sumo_get_ps(rps); + + if (!pi->driver_nbps_policy_disable) { + if (new_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE) + WREG32_P(CG_SCLK_DPM_CTRL_3, FORCE_NB_PSTATE_1, ~FORCE_NB_PSTATE_1); + else + WREG32_P(CG_SCLK_DPM_CTRL_3, 0, ~FORCE_NB_PSTATE_1); + } +} + +u32 sumo_get_sleep_divider_from_id(u32 id) +{ + return 1 << id; +} + +u32 sumo_get_sleep_divider_id_from_clock(struct radeon_device *rdev, + u32 sclk, + u32 min_sclk_in_sr) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 i; + u32 temp; + u32 min = (min_sclk_in_sr > SUMO_MINIMUM_ENGINE_CLOCK) ? + min_sclk_in_sr : SUMO_MINIMUM_ENGINE_CLOCK; + + if (sclk < min) + return 0; + + if (!pi->enable_sclk_ds) + return 0; + + for (i = SUMO_MAX_DEEPSLEEP_DIVIDER_ID; ; i--) { + temp = sclk / sumo_get_sleep_divider_from_id(i); + + if (temp >= min || i == 0) + break; + } + return i; +} + +static u32 sumo_get_valid_engine_clock(struct radeon_device *rdev, + u32 lower_limit) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 i; + + for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) { + if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit) + return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency; + } + + return pi->sys_info.sclk_voltage_mapping_table.entries[pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries - 1].sclk_frequency; +} + +static void sumo_patch_thermal_state(struct radeon_device *rdev, + struct sumo_ps *ps, + struct sumo_ps *current_ps) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */ + u32 current_vddc; + u32 current_sclk; + u32 current_index = 0; + + if (current_ps) { + current_vddc = current_ps->levels[current_index].vddc_index; + current_sclk = current_ps->levels[current_index].sclk; + } else { + current_vddc = pi->boot_pl.vddc_index; + current_sclk = pi->boot_pl.sclk; + } + + ps->levels[0].vddc_index = current_vddc; + + if (ps->levels[0].sclk > current_sclk) + ps->levels[0].sclk = current_sclk; + + ps->levels[0].ss_divider_index = + sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr); + + ps->levels[0].ds_divider_index = + sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, SUMO_MINIMUM_ENGINE_CLOCK); + + if (ps->levels[0].ds_divider_index > ps->levels[0].ss_divider_index + 1) + ps->levels[0].ds_divider_index = ps->levels[0].ss_divider_index + 1; + + if (ps->levels[0].ss_divider_index == ps->levels[0].ds_divider_index) { + if (ps->levels[0].ss_divider_index > 1) + ps->levels[0].ss_divider_index = ps->levels[0].ss_divider_index - 1; + } + + if (ps->levels[0].ss_divider_index == 0) + ps->levels[0].ds_divider_index = 0; + + if (ps->levels[0].ds_divider_index == 0) + ps->levels[0].ss_divider_index = 0; +} + +static void sumo_apply_state_adjust_rules(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct sumo_ps *ps = sumo_get_ps(new_rps); + struct sumo_ps *current_ps = sumo_get_ps(old_rps); + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 min_voltage = 0; /* ??? */ + u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */ + u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */ + u32 i; + + if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL) + return sumo_patch_thermal_state(rdev, ps, current_ps); + + if (pi->enable_boost) { + if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) + ps->flags |= SUMO_POWERSTATE_FLAGS_BOOST_STATE; + } + + if ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) || + (new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) || + (new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE)) + ps->flags |= SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE; + + for (i = 0; i < ps->num_levels; i++) { + if (ps->levels[i].vddc_index < min_voltage) + ps->levels[i].vddc_index = min_voltage; + + if (ps->levels[i].sclk < min_sclk) + ps->levels[i].sclk = + sumo_get_valid_engine_clock(rdev, min_sclk); + + ps->levels[i].ss_divider_index = + sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr); + + ps->levels[i].ds_divider_index = + sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, SUMO_MINIMUM_ENGINE_CLOCK); + + if (ps->levels[i].ds_divider_index > ps->levels[i].ss_divider_index + 1) + ps->levels[i].ds_divider_index = ps->levels[i].ss_divider_index + 1; + + if (ps->levels[i].ss_divider_index == ps->levels[i].ds_divider_index) { + if (ps->levels[i].ss_divider_index > 1) + ps->levels[i].ss_divider_index = ps->levels[i].ss_divider_index - 1; + } + + if (ps->levels[i].ss_divider_index == 0) + ps->levels[i].ds_divider_index = 0; + + if (ps->levels[i].ds_divider_index == 0) + ps->levels[i].ss_divider_index = 0; + + if (ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE) + ps->levels[i].allow_gnb_slow = 1; + else if ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) || + (new_rps->class2 & ATOM_PPLIB_CLASSIFICATION2_MVC)) + ps->levels[i].allow_gnb_slow = 0; + else if (i == ps->num_levels - 1) + ps->levels[i].allow_gnb_slow = 0; + else + ps->levels[i].allow_gnb_slow = 1; + } +} + +static void sumo_cleanup_asic(struct radeon_device *rdev) +{ + sumo_take_smu_control(rdev, false); +} + +static int sumo_set_thermal_temperature_range(struct radeon_device *rdev, + int min_temp, int max_temp) +{ + int low_temp = 0 * 1000; + int high_temp = 255 * 1000; + + if (low_temp < min_temp) + low_temp = min_temp; + if (high_temp > max_temp) + high_temp = max_temp; + if (high_temp < low_temp) { + DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp); + return -EINVAL; + } + + WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK); + WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK); + + rdev->pm.dpm.thermal.min_temp = low_temp; + rdev->pm.dpm.thermal.max_temp = high_temp; + + return 0; +} + +static void sumo_update_current_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct sumo_ps *new_ps = sumo_get_ps(rps); + struct sumo_power_info *pi = sumo_get_pi(rdev); + + pi->current_rps = *rps; + pi->current_ps = *new_ps; + pi->current_rps.ps_priv = &pi->current_ps; +} + +static void sumo_update_requested_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct sumo_ps *new_ps = sumo_get_ps(rps); + struct sumo_power_info *pi = sumo_get_pi(rdev); + + pi->requested_rps = *rps; + pi->requested_ps = *new_ps; + pi->requested_rps.ps_priv = &pi->requested_ps; +} + +int sumo_dpm_enable(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + if (sumo_dpm_enabled(rdev)) + return -EINVAL; + + sumo_program_bootup_state(rdev); + sumo_init_bsp(rdev); + sumo_reset_am(rdev); + sumo_program_tp(rdev); + sumo_program_bootup_at(rdev); + sumo_start_am(rdev); + if (pi->enable_auto_thermal_throttling) { + sumo_program_ttp(rdev); + sumo_program_ttt(rdev); + } + sumo_program_dc_hto(rdev); + sumo_program_power_level_enter_state(rdev); + sumo_enable_voltage_scaling(rdev, true); + sumo_program_sstp(rdev); + sumo_program_vc(rdev, SUMO_VRC_DFLT); + sumo_override_cnb_thermal_events(rdev); + sumo_start_dpm(rdev); + sumo_wait_for_level_0(rdev); + if (pi->enable_sclk_ds) + sumo_enable_sclk_ds(rdev, true); + if (pi->enable_boost) + sumo_enable_boost_timer(rdev); + + sumo_update_current_ps(rdev, rdev->pm.dpm.boot_ps); + + return 0; +} + +int sumo_dpm_late_enable(struct radeon_device *rdev) +{ + int ret; + + ret = sumo_enable_clock_power_gating(rdev); + if (ret) + return ret; + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + ret = sumo_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) + return ret; + rdev->irq.dpm_thermal = true; + radeon_irq_set(rdev); + } + + return 0; +} + +void sumo_dpm_disable(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + if (!sumo_dpm_enabled(rdev)) + return; + sumo_disable_clock_power_gating(rdev); + if (pi->enable_sclk_ds) + sumo_enable_sclk_ds(rdev, false); + sumo_clear_vc(rdev); + sumo_wait_for_level_0(rdev); + sumo_stop_dpm(rdev); + sumo_enable_voltage_scaling(rdev, false); + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + rdev->irq.dpm_thermal = false; + radeon_irq_set(rdev); + } + + sumo_update_current_ps(rdev, rdev->pm.dpm.boot_ps); +} + +int sumo_dpm_pre_set_power_state(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps; + struct radeon_ps *new_ps = &requested_ps; + + sumo_update_requested_ps(rdev, new_ps); + + if (pi->enable_dynamic_patch_ps) + sumo_apply_state_adjust_rules(rdev, + &pi->requested_rps, + &pi->current_rps); + + return 0; +} + +int sumo_dpm_set_power_state(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct radeon_ps *new_ps = &pi->requested_rps; + struct radeon_ps *old_ps = &pi->current_rps; + + if (pi->enable_dpm) + sumo_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps); + if (pi->enable_boost) { + sumo_enable_boost(rdev, new_ps, false); + sumo_patch_boost_state(rdev, new_ps); + } + if (pi->enable_dpm) { + sumo_pre_notify_alt_vddnb_change(rdev, new_ps, old_ps); + sumo_enable_power_level_0(rdev); + sumo_set_forced_level_0(rdev); + sumo_set_forced_mode_enabled(rdev); + sumo_wait_for_level_0(rdev); + sumo_program_power_levels_0_to_n(rdev, new_ps, old_ps); + sumo_program_wl(rdev, new_ps); + sumo_program_bsp(rdev, new_ps); + sumo_program_at(rdev, new_ps); + sumo_force_nbp_state(rdev, new_ps); + sumo_set_forced_mode_disabled(rdev); + sumo_set_forced_mode_enabled(rdev); + sumo_set_forced_mode_disabled(rdev); + sumo_post_notify_alt_vddnb_change(rdev, new_ps, old_ps); + } + if (pi->enable_boost) + sumo_enable_boost(rdev, new_ps, true); + if (pi->enable_dpm) + sumo_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps); + + return 0; +} + +void sumo_dpm_post_set_power_state(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct radeon_ps *new_ps = &pi->requested_rps; + + sumo_update_current_ps(rdev, new_ps); +} + +#if 0 +void sumo_dpm_reset_asic(struct radeon_device *rdev) +{ + sumo_program_bootup_state(rdev); + sumo_enable_power_level_0(rdev); + sumo_set_forced_level_0(rdev); + sumo_set_forced_mode_enabled(rdev); + sumo_wait_for_level_0(rdev); + sumo_set_forced_mode_disabled(rdev); + sumo_set_forced_mode_enabled(rdev); + sumo_set_forced_mode_disabled(rdev); +} +#endif + +void sumo_dpm_setup_asic(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + sumo_initialize_m3_arb(rdev); + pi->fw_version = sumo_get_running_fw_version(rdev); + DRM_INFO("Found smc ucode version: 0x%08x\n", pi->fw_version); + sumo_program_acpi_power_level(rdev); + sumo_enable_acpi_pm(rdev); + sumo_take_smu_control(rdev, true); +} + +void sumo_dpm_display_configuration_changed(struct radeon_device *rdev) +{ + +} + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; +}; + +union pplib_clock_info { + struct _ATOM_PPLIB_R600_CLOCK_INFO r600; + struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; + struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; + struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; +}; + +union pplib_power_state { + struct _ATOM_PPLIB_STATE v1; + struct _ATOM_PPLIB_STATE_V2 v2; +}; + +static void sumo_patch_boot_state(struct radeon_device *rdev, + struct sumo_ps *ps) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + ps->num_levels = 1; + ps->flags = 0; + ps->levels[0] = pi->boot_pl; +} + +static void sumo_parse_pplib_non_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info, + u8 table_rev) +{ + struct sumo_ps *ps = sumo_get_ps(rps); + + rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); + rps->class = le16_to_cpu(non_clock_info->usClassification); + rps->class2 = le16_to_cpu(non_clock_info->usClassification2); + + if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) { + rps->vclk = le32_to_cpu(non_clock_info->ulVCLK); + rps->dclk = le32_to_cpu(non_clock_info->ulDCLK); + } else { + rps->vclk = 0; + rps->dclk = 0; + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { + rdev->pm.dpm.boot_ps = rps; + sumo_patch_boot_state(rdev, ps); + } + if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + rdev->pm.dpm.uvd_ps = rps; +} + +static void sumo_parse_pplib_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, int index, + union pplib_clock_info *clock_info) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct sumo_ps *ps = sumo_get_ps(rps); + struct sumo_pl *pl = &ps->levels[index]; + u32 sclk; + + sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow); + sclk |= clock_info->sumo.ucEngineClockHigh << 16; + pl->sclk = sclk; + pl->vddc_index = clock_info->sumo.vddcIndex; + pl->sclk_dpm_tdp_limit = clock_info->sumo.tdpLimit; + + ps->num_levels = index + 1; + + if (pi->enable_sclk_ds) { + pl->ds_divider_index = 5; + pl->ss_divider_index = 4; + } +} + +static int sumo_parse_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; + union pplib_power_state *power_state; + int i, j, k, non_clock_array_index, clock_array_index; + union pplib_clock_info *clock_info; + struct _StateArray *state_array; + struct _ClockInfoArray *clock_info_array; + struct _NonClockInfoArray *non_clock_info_array; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + u8 *power_state_offset; + struct sumo_ps *ps; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + state_array = (struct _StateArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usStateArrayOffset)); + clock_info_array = (struct _ClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usClockInfoArrayOffset)); + non_clock_info_array = (struct _NonClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset)); + + rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) * + state_array->ucNumEntries, GFP_KERNEL); + if (!rdev->pm.dpm.ps) + return -ENOMEM; + power_state_offset = (u8 *)state_array->states; + for (i = 0; i < state_array->ucNumEntries; i++) { + u8 *idx; + power_state = (union pplib_power_state *)power_state_offset; + non_clock_array_index = power_state->v2.nonClockInfoIndex; + non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) + &non_clock_info_array->nonClockInfo[non_clock_array_index]; + if (!rdev->pm.power_state[i].clock_info) + return -EINVAL; + ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL); + if (ps == NULL) { + kfree(rdev->pm.dpm.ps); + return -ENOMEM; + } + rdev->pm.dpm.ps[i].ps_priv = ps; + k = 0; + idx = (u8 *)&power_state->v2.clockInfoIndex[0]; + for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) { + clock_array_index = idx[j]; + if (k >= SUMO_MAX_HARDWARE_POWERLEVELS) + break; + + clock_info = (union pplib_clock_info *) + ((u8 *)&clock_info_array->clockInfo[0] + + (clock_array_index * clock_info_array->ucEntrySize)); + sumo_parse_pplib_clock_info(rdev, + &rdev->pm.dpm.ps[i], k, + clock_info); + k++; + } + sumo_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i], + non_clock_info, + non_clock_info_array->ucEntrySize); + power_state_offset += 2 + power_state->v2.ucNumDPMLevels; + } + rdev->pm.dpm.num_ps = state_array->ucNumEntries; + return 0; +} + +u32 sumo_convert_vid2_to_vid7(struct radeon_device *rdev, + struct sumo_vid_mapping_table *vid_mapping_table, + u32 vid_2bit) +{ + u32 i; + + for (i = 0; i < vid_mapping_table->num_entries; i++) { + if (vid_mapping_table->entries[i].vid_2bit == vid_2bit) + return vid_mapping_table->entries[i].vid_7bit; + } + + return vid_mapping_table->entries[vid_mapping_table->num_entries - 1].vid_7bit; +} + +#if 0 +u32 sumo_convert_vid7_to_vid2(struct radeon_device *rdev, + struct sumo_vid_mapping_table *vid_mapping_table, + u32 vid_7bit) +{ + u32 i; + + for (i = 0; i < vid_mapping_table->num_entries; i++) { + if (vid_mapping_table->entries[i].vid_7bit == vid_7bit) + return vid_mapping_table->entries[i].vid_2bit; + } + + return vid_mapping_table->entries[vid_mapping_table->num_entries - 1].vid_2bit; +} +#endif + +static u16 sumo_convert_voltage_index_to_value(struct radeon_device *rdev, + u32 vid_2bit) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit); + + if (vid_7bit > 0x7C) + return 0; + + return (15500 - vid_7bit * 125 + 5) / 10; +} + +static void sumo_construct_display_voltage_mapping_table(struct radeon_device *rdev, + struct sumo_disp_clock_voltage_mapping_table *disp_clk_voltage_mapping_table, + ATOM_CLK_VOLT_CAPABILITY *table) +{ + u32 i; + + for (i = 0; i < SUMO_MAX_NUMBER_VOLTAGES; i++) { + if (table[i].ulMaximumSupportedCLK == 0) + break; + + disp_clk_voltage_mapping_table->display_clock_frequency[i] = + table[i].ulMaximumSupportedCLK; + } + + disp_clk_voltage_mapping_table->num_max_voltage_levels = i; + + if (disp_clk_voltage_mapping_table->num_max_voltage_levels == 0) { + disp_clk_voltage_mapping_table->display_clock_frequency[0] = 80000; + disp_clk_voltage_mapping_table->num_max_voltage_levels = 1; + } +} + +void sumo_construct_sclk_voltage_mapping_table(struct radeon_device *rdev, + struct sumo_sclk_voltage_mapping_table *sclk_voltage_mapping_table, + ATOM_AVAILABLE_SCLK_LIST *table) +{ + u32 i; + u32 n = 0; + u32 prev_sclk = 0; + + for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++) { + if (table[i].ulSupportedSCLK > prev_sclk) { + sclk_voltage_mapping_table->entries[n].sclk_frequency = + table[i].ulSupportedSCLK; + sclk_voltage_mapping_table->entries[n].vid_2bit = + table[i].usVoltageIndex; + prev_sclk = table[i].ulSupportedSCLK; + n++; + } + } + + sclk_voltage_mapping_table->num_max_dpm_entries = n; +} + +void sumo_construct_vid_mapping_table(struct radeon_device *rdev, + struct sumo_vid_mapping_table *vid_mapping_table, + ATOM_AVAILABLE_SCLK_LIST *table) +{ + u32 i, j; + + for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++) { + if (table[i].ulSupportedSCLK != 0) { + vid_mapping_table->entries[table[i].usVoltageIndex].vid_7bit = + table[i].usVoltageID; + vid_mapping_table->entries[table[i].usVoltageIndex].vid_2bit = + table[i].usVoltageIndex; + } + } + + for (i = 0; i < SUMO_MAX_NUMBER_VOLTAGES; i++) { + if (vid_mapping_table->entries[i].vid_7bit == 0) { + for (j = i + 1; j < SUMO_MAX_NUMBER_VOLTAGES; j++) { + if (vid_mapping_table->entries[j].vid_7bit != 0) { + vid_mapping_table->entries[i] = + vid_mapping_table->entries[j]; + vid_mapping_table->entries[j].vid_7bit = 0; + break; + } + } + + if (j == SUMO_MAX_NUMBER_VOLTAGES) + break; + } + } + + vid_mapping_table->num_entries = i; +} + +union igp_info { + struct _ATOM_INTEGRATED_SYSTEM_INFO info; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6; +}; + +static int sumo_parse_sys_info_table(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct radeon_mode_info *mode_info = &rdev->mode_info; + int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo); + union igp_info *igp_info; + u8 frev, crev; + u16 data_offset; + int i; + + if (atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + igp_info = (union igp_info *)(mode_info->atom_context->bios + + data_offset); + + if (crev != 6) { + DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev); + return -EINVAL; + } + pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_6.ulBootUpEngineClock); + pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_6.ulMinEngineClock); + pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_6.ulBootUpUMAClock); + pi->sys_info.bootup_nb_voltage_index = + le16_to_cpu(igp_info->info_6.usBootUpNBVoltage); + if (igp_info->info_6.ucHtcTmpLmt == 0) + pi->sys_info.htc_tmp_lmt = 203; + else + pi->sys_info.htc_tmp_lmt = igp_info->info_6.ucHtcTmpLmt; + if (igp_info->info_6.ucHtcHystLmt == 0) + pi->sys_info.htc_hyst_lmt = 5; + else + pi->sys_info.htc_hyst_lmt = igp_info->info_6.ucHtcHystLmt; + if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) { + DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n"); + } + for (i = 0; i < NUMBER_OF_M3ARB_PARAM_SETS; i++) { + pi->sys_info.csr_m3_arb_cntl_default[i] = + le32_to_cpu(igp_info->info_6.ulCSR_M3_ARB_CNTL_DEFAULT[i]); + pi->sys_info.csr_m3_arb_cntl_uvd[i] = + le32_to_cpu(igp_info->info_6.ulCSR_M3_ARB_CNTL_UVD[i]); + pi->sys_info.csr_m3_arb_cntl_fs3d[i] = + le32_to_cpu(igp_info->info_6.ulCSR_M3_ARB_CNTL_FS3D[i]); + } + pi->sys_info.sclk_dpm_boost_margin = + le32_to_cpu(igp_info->info_6.SclkDpmBoostMargin); + pi->sys_info.sclk_dpm_throttle_margin = + le32_to_cpu(igp_info->info_6.SclkDpmThrottleMargin); + pi->sys_info.sclk_dpm_tdp_limit_pg = + le16_to_cpu(igp_info->info_6.SclkDpmTdpLimitPG); + pi->sys_info.gnb_tdp_limit = le16_to_cpu(igp_info->info_6.GnbTdpLimit); + pi->sys_info.sclk_dpm_tdp_limit_boost = + le16_to_cpu(igp_info->info_6.SclkDpmTdpLimitBoost); + pi->sys_info.boost_sclk = le32_to_cpu(igp_info->info_6.ulBoostEngineCLock); + pi->sys_info.boost_vid_2bit = igp_info->info_6.ulBoostVid_2bit; + if (igp_info->info_6.EnableBoost) + pi->sys_info.enable_boost = true; + else + pi->sys_info.enable_boost = false; + sumo_construct_display_voltage_mapping_table(rdev, + &pi->sys_info.disp_clk_voltage_mapping_table, + igp_info->info_6.sDISPCLK_Voltage); + sumo_construct_sclk_voltage_mapping_table(rdev, + &pi->sys_info.sclk_voltage_mapping_table, + igp_info->info_6.sAvail_SCLK); + sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table, + igp_info->info_6.sAvail_SCLK); + + } + return 0; +} + +static void sumo_construct_boot_and_acpi_state(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + pi->boot_pl.sclk = pi->sys_info.bootup_sclk; + pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index; + pi->boot_pl.ds_divider_index = 0; + pi->boot_pl.ss_divider_index = 0; + pi->boot_pl.allow_gnb_slow = 1; + pi->acpi_pl = pi->boot_pl; + pi->current_ps.num_levels = 1; + pi->current_ps.levels[0] = pi->boot_pl; +} + +int sumo_dpm_init(struct radeon_device *rdev) +{ + struct sumo_power_info *pi; + u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT; + int ret; + + pi = kzalloc(sizeof(struct sumo_power_info), GFP_KERNEL); + if (pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = pi; + + pi->driver_nbps_policy_disable = false; + if ((rdev->family == CHIP_PALM) && (hw_rev < 3)) + pi->disable_gfx_power_gating_in_uvd = true; + else + pi->disable_gfx_power_gating_in_uvd = false; + pi->enable_alt_vddnb = true; + pi->enable_sclk_ds = true; + pi->enable_dynamic_m3_arbiter = false; + pi->enable_dynamic_patch_ps = true; + /* Some PALM chips don't seem to properly ungate gfx when UVD is in use; + * for now just disable gfx PG. + */ + if (rdev->family == CHIP_PALM) + pi->enable_gfx_power_gating = false; + else + pi->enable_gfx_power_gating = true; + pi->enable_gfx_clock_gating = true; + pi->enable_mg_clock_gating = true; + pi->enable_auto_thermal_throttling = true; + + ret = sumo_parse_sys_info_table(rdev); + if (ret) + return ret; + + sumo_construct_boot_and_acpi_state(rdev); + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = sumo_parse_power_table(rdev); + if (ret) + return ret; + + pi->pasi = CYPRESS_HASI_DFLT; + pi->asi = RV770_ASI_DFLT; + pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt; + pi->enable_boost = pi->sys_info.enable_boost; + pi->enable_dpm = true; + + return 0; +} + +void sumo_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + int i; + struct sumo_ps *ps = sumo_get_ps(rps); + + r600_dpm_print_class_info(rps->class, rps->class2); + r600_dpm_print_cap_info(rps->caps); + printk("\tuvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + for (i = 0; i < ps->num_levels; i++) { + struct sumo_pl *pl = &ps->levels[i]; + printk("\t\tpower level %d sclk: %u vddc: %u\n", + i, pl->sclk, + sumo_convert_voltage_index_to_value(rdev, pl->vddc_index)); + } + r600_dpm_print_ps_status(rdev, rps); +} + +void sumo_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct radeon_ps *rps = &pi->current_rps; + struct sumo_ps *ps = sumo_get_ps(rps); + struct sumo_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURR_INDEX_MASK) >> + CURR_INDEX_SHIFT; + + if (current_index == BOOST_DPM_LEVEL) { + pl = &pi->boost_pl; + seq_printf(m, "uvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + seq_printf(m, "power level %d sclk: %u vddc: %u\n", + current_index, pl->sclk, + sumo_convert_voltage_index_to_value(rdev, pl->vddc_index)); + } else if (current_index >= ps->num_levels) { + seq_printf(m, "invalid dpm profile %d\n", current_index); + } else { + pl = &ps->levels[current_index]; + seq_printf(m, "uvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + seq_printf(m, "power level %d sclk: %u vddc: %u\n", + current_index, pl->sclk, + sumo_convert_voltage_index_to_value(rdev, pl->vddc_index)); + } +} + +u32 sumo_dpm_get_current_sclk(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct radeon_ps *rps = &pi->current_rps; + struct sumo_ps *ps = sumo_get_ps(rps); + struct sumo_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURR_INDEX_MASK) >> + CURR_INDEX_SHIFT; + + if (current_index == BOOST_DPM_LEVEL) { + pl = &pi->boost_pl; + return pl->sclk; + } else if (current_index >= ps->num_levels) { + return 0; + } else { + pl = &ps->levels[current_index]; + return pl->sclk; + } +} + +u32 sumo_dpm_get_current_mclk(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + return pi->sys_info.bootup_uma_clk; +} + +void sumo_dpm_fini(struct radeon_device *rdev) +{ + int i; + + sumo_cleanup_asic(rdev); /* ??? */ + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); +} + +u32 sumo_dpm_get_sclk(struct radeon_device *rdev, bool low) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct sumo_ps *requested_state = sumo_get_ps(&pi->requested_rps); + + if (low) + return requested_state->levels[0].sclk; + else + return requested_state->levels[requested_state->num_levels - 1].sclk; +} + +u32 sumo_dpm_get_mclk(struct radeon_device *rdev, bool low) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + + return pi->sys_info.bootup_uma_clk; +} + +int sumo_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + struct radeon_ps *rps = &pi->current_rps; + struct sumo_ps *ps = sumo_get_ps(rps); + int i; + + if (ps->num_levels <= 1) + return 0; + + if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + if (pi->enable_boost) + sumo_enable_boost(rdev, rps, false); + sumo_power_level_enable(rdev, ps->num_levels - 1, true); + sumo_set_forced_level(rdev, ps->num_levels - 1); + sumo_set_forced_mode_enabled(rdev); + for (i = 0; i < ps->num_levels - 1; i++) { + sumo_power_level_enable(rdev, i, false); + } + sumo_set_forced_mode(rdev, false); + sumo_set_forced_mode_enabled(rdev); + sumo_set_forced_mode(rdev, false); + } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { + if (pi->enable_boost) + sumo_enable_boost(rdev, rps, false); + sumo_power_level_enable(rdev, 0, true); + sumo_set_forced_level(rdev, 0); + sumo_set_forced_mode_enabled(rdev); + for (i = 1; i < ps->num_levels; i++) { + sumo_power_level_enable(rdev, i, false); + } + sumo_set_forced_mode(rdev, false); + sumo_set_forced_mode_enabled(rdev); + sumo_set_forced_mode(rdev, false); + } else { + for (i = 0; i < ps->num_levels; i++) { + sumo_power_level_enable(rdev, i, true); + } + if (pi->enable_boost) + sumo_enable_boost(rdev, rps, true); + } + + rdev->pm.dpm.forced_level = level; + + return 0; +} diff --git a/sys/dev/pci/drm/radeon/sumo_dpm.h b/sys/dev/pci/drm/radeon/sumo_dpm.h new file mode 100644 index 00000000000..07dda299c78 --- /dev/null +++ b/sys/dev/pci/drm/radeon/sumo_dpm.h @@ -0,0 +1,220 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __SUMO_DPM_H__ +#define __SUMO_DPM_H__ + +#include "atom.h" + +#define SUMO_MAX_HARDWARE_POWERLEVELS 5 +#define SUMO_PM_NUMBER_OF_TC 15 + +struct sumo_pl { + u32 sclk; + u32 vddc_index; + u32 ds_divider_index; + u32 ss_divider_index; + u32 allow_gnb_slow; + u32 sclk_dpm_tdp_limit; +}; + +/* used for the flags field */ +#define SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE (1 << 0) +#define SUMO_POWERSTATE_FLAGS_BOOST_STATE (1 << 1) + +struct sumo_ps { + struct sumo_pl levels[SUMO_MAX_HARDWARE_POWERLEVELS]; + u32 num_levels; + /* flags */ + u32 flags; +}; + +#define NUMBER_OF_M3ARB_PARAM_SETS 10 +#define SUMO_MAX_NUMBER_VOLTAGES 4 + +struct sumo_disp_clock_voltage_mapping_table { + u32 num_max_voltage_levels; + u32 display_clock_frequency[SUMO_MAX_NUMBER_VOLTAGES]; +}; + +struct sumo_vid_mapping_entry { + u16 vid_2bit; + u16 vid_7bit; +}; + +struct sumo_vid_mapping_table { + u32 num_entries; + struct sumo_vid_mapping_entry entries[SUMO_MAX_NUMBER_VOLTAGES]; +}; + +struct sumo_sclk_voltage_mapping_entry { + u32 sclk_frequency; + u16 vid_2bit; + u16 rsv; +}; + +struct sumo_sclk_voltage_mapping_table { + u32 num_max_dpm_entries; + struct sumo_sclk_voltage_mapping_entry entries[SUMO_MAX_HARDWARE_POWERLEVELS]; +}; + +struct sumo_sys_info { + u32 bootup_sclk; + u32 min_sclk; + u32 bootup_uma_clk; + u16 bootup_nb_voltage_index; + u8 htc_tmp_lmt; + u8 htc_hyst_lmt; + struct sumo_sclk_voltage_mapping_table sclk_voltage_mapping_table; + struct sumo_disp_clock_voltage_mapping_table disp_clk_voltage_mapping_table; + struct sumo_vid_mapping_table vid_mapping_table; + u32 csr_m3_arb_cntl_default[NUMBER_OF_M3ARB_PARAM_SETS]; + u32 csr_m3_arb_cntl_uvd[NUMBER_OF_M3ARB_PARAM_SETS]; + u32 csr_m3_arb_cntl_fs3d[NUMBER_OF_M3ARB_PARAM_SETS]; + u32 sclk_dpm_boost_margin; + u32 sclk_dpm_throttle_margin; + u32 sclk_dpm_tdp_limit_pg; + u32 gnb_tdp_limit; + u32 sclk_dpm_tdp_limit_boost; + u32 boost_sclk; + u32 boost_vid_2bit; + bool enable_boost; +}; + +struct sumo_power_info { + u32 asi; + u32 pasi; + u32 bsp; + u32 bsu; + u32 pbsp; + u32 pbsu; + u32 dsp; + u32 psp; + u32 thermal_auto_throttling; + u32 uvd_m3_arbiter; + u32 fw_version; + struct sumo_sys_info sys_info; + struct sumo_pl acpi_pl; + struct sumo_pl boot_pl; + struct sumo_pl boost_pl; + bool disable_gfx_power_gating_in_uvd; + bool driver_nbps_policy_disable; + bool enable_alt_vddnb; + bool enable_dynamic_m3_arbiter; + bool enable_gfx_clock_gating; + bool enable_gfx_power_gating; + bool enable_mg_clock_gating; + bool enable_sclk_ds; + bool enable_auto_thermal_throttling; + bool enable_dynamic_patch_ps; + bool enable_dpm; + bool enable_boost; + struct radeon_ps current_rps; + struct sumo_ps current_ps; + struct radeon_ps requested_rps; + struct sumo_ps requested_ps; +}; + +#define SUMO_UTC_DFLT_00 0x48 +#define SUMO_UTC_DFLT_01 0x44 +#define SUMO_UTC_DFLT_02 0x44 +#define SUMO_UTC_DFLT_03 0x44 +#define SUMO_UTC_DFLT_04 0x44 +#define SUMO_UTC_DFLT_05 0x44 +#define SUMO_UTC_DFLT_06 0x44 +#define SUMO_UTC_DFLT_07 0x44 +#define SUMO_UTC_DFLT_08 0x44 +#define SUMO_UTC_DFLT_09 0x44 +#define SUMO_UTC_DFLT_10 0x44 +#define SUMO_UTC_DFLT_11 0x44 +#define SUMO_UTC_DFLT_12 0x44 +#define SUMO_UTC_DFLT_13 0x44 +#define SUMO_UTC_DFLT_14 0x44 + +#define SUMO_DTC_DFLT_00 0x48 +#define SUMO_DTC_DFLT_01 0x44 +#define SUMO_DTC_DFLT_02 0x44 +#define SUMO_DTC_DFLT_03 0x44 +#define SUMO_DTC_DFLT_04 0x44 +#define SUMO_DTC_DFLT_05 0x44 +#define SUMO_DTC_DFLT_06 0x44 +#define SUMO_DTC_DFLT_07 0x44 +#define SUMO_DTC_DFLT_08 0x44 +#define SUMO_DTC_DFLT_09 0x44 +#define SUMO_DTC_DFLT_10 0x44 +#define SUMO_DTC_DFLT_11 0x44 +#define SUMO_DTC_DFLT_12 0x44 +#define SUMO_DTC_DFLT_13 0x44 +#define SUMO_DTC_DFLT_14 0x44 + +#define SUMO_AH_DFLT 5 + +#define SUMO_R_DFLT0 70 +#define SUMO_R_DFLT1 70 +#define SUMO_R_DFLT2 70 +#define SUMO_R_DFLT3 70 +#define SUMO_R_DFLT4 100 + +#define SUMO_L_DFLT0 0 +#define SUMO_L_DFLT1 20 +#define SUMO_L_DFLT2 20 +#define SUMO_L_DFLT3 20 +#define SUMO_L_DFLT4 20 +#define SUMO_VRC_DFLT 0x30033 +#define SUMO_MGCGTTLOCAL0_DFLT 0 +#define SUMO_MGCGTTLOCAL1_DFLT 0 +#define SUMO_GICST_DFLT 19 +#define SUMO_SST_DFLT 8 +#define SUMO_VOLTAGEDROPT_DFLT 1 +#define SUMO_GFXPOWERGATINGT_DFLT 100 + +/* sumo_dpm.c */ +void sumo_gfx_clockgating_initialize(struct radeon_device *rdev); +void sumo_program_vc(struct radeon_device *rdev, u32 vrc); +void sumo_clear_vc(struct radeon_device *rdev); +void sumo_program_sstp(struct radeon_device *rdev); +void sumo_take_smu_control(struct radeon_device *rdev, bool enable); +void sumo_construct_sclk_voltage_mapping_table(struct radeon_device *rdev, + struct sumo_sclk_voltage_mapping_table *sclk_voltage_mapping_table, + ATOM_AVAILABLE_SCLK_LIST *table); +void sumo_construct_vid_mapping_table(struct radeon_device *rdev, + struct sumo_vid_mapping_table *vid_mapping_table, + ATOM_AVAILABLE_SCLK_LIST *table); +u32 sumo_convert_vid2_to_vid7(struct radeon_device *rdev, + struct sumo_vid_mapping_table *vid_mapping_table, + u32 vid_2bit); +u32 sumo_get_sleep_divider_from_id(u32 id); +u32 sumo_get_sleep_divider_id_from_clock(struct radeon_device *rdev, + u32 sclk, + u32 min_sclk_in_sr); + +/* sumo_smc.c */ +void sumo_initialize_m3_arb(struct radeon_device *rdev); +void sumo_smu_pg_init(struct radeon_device *rdev); +void sumo_set_tdp_limit(struct radeon_device *rdev, u32 index, u32 tdp_limit); +void sumo_smu_notify_alt_vddnb_change(struct radeon_device *rdev, + bool powersaving, bool force_nbps1); +void sumo_boost_state_enable(struct radeon_device *rdev, bool enable); +void sumo_enable_boost_timer(struct radeon_device *rdev); +u32 sumo_get_running_fw_version(struct radeon_device *rdev); + +#endif diff --git a/sys/dev/pci/drm/radeon/sumo_smc.c b/sys/dev/pci/drm/radeon/sumo_smc.c new file mode 100644 index 00000000000..dcf9447e786 --- /dev/null +++ b/sys/dev/pci/drm/radeon/sumo_smc.c @@ -0,0 +1,221 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "radeon.h" +#include "sumod.h" +#include "sumo_dpm.h" +#include "ppsmc.h" + +#define SUMO_SMU_SERVICE_ROUTINE_PG_INIT 1 +#define SUMO_SMU_SERVICE_ROUTINE_ALTVDDNB_NOTIFY 27 +#define SUMO_SMU_SERVICE_ROUTINE_GFX_SRV_ID_20 20 + +struct sumo_power_info *sumo_get_pi(struct radeon_device *rdev); + +static void sumo_send_msg_to_smu(struct radeon_device *rdev, u32 id) +{ + u32 gfx_int_req; + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(GFX_INT_STATUS) & INT_DONE) + break; + udelay(1); + } + + gfx_int_req = SERV_INDEX(id) | INT_REQ; + WREG32(GFX_INT_REQ, gfx_int_req); + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(GFX_INT_REQ) & INT_REQ) + break; + udelay(1); + } + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(GFX_INT_STATUS) & INT_ACK) + break; + udelay(1); + } + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(GFX_INT_STATUS) & INT_DONE) + break; + udelay(1); + } + + gfx_int_req &= ~INT_REQ; + WREG32(GFX_INT_REQ, gfx_int_req); +} + +void sumo_initialize_m3_arb(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 i; + + if (!pi->enable_dynamic_m3_arbiter) + return; + + for (i = 0; i < NUMBER_OF_M3ARB_PARAM_SETS; i++) + WREG32_RCU(MCU_M3ARB_PARAMS + (i * 4), + pi->sys_info.csr_m3_arb_cntl_default[i]); + + for (; i < NUMBER_OF_M3ARB_PARAM_SETS * 2; i++) + WREG32_RCU(MCU_M3ARB_PARAMS + (i * 4), + pi->sys_info.csr_m3_arb_cntl_uvd[i % NUMBER_OF_M3ARB_PARAM_SETS]); + + for (; i < NUMBER_OF_M3ARB_PARAM_SETS * 3; i++) + WREG32_RCU(MCU_M3ARB_PARAMS + (i * 4), + pi->sys_info.csr_m3_arb_cntl_fs3d[i % NUMBER_OF_M3ARB_PARAM_SETS]); +} + +static bool sumo_is_alt_vddnb_supported(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + bool return_code = false; + + if (!pi->enable_alt_vddnb) + return return_code; + + if ((rdev->family == CHIP_SUMO) || (rdev->family == CHIP_SUMO2)) { + if (pi->fw_version >= 0x00010C00) + return_code = true; + } + + return return_code; +} + +void sumo_smu_notify_alt_vddnb_change(struct radeon_device *rdev, + bool powersaving, bool force_nbps1) +{ + u32 param = 0; + + if (!sumo_is_alt_vddnb_supported(rdev)) + return; + + if (powersaving) + param |= 1; + + if (force_nbps1) + param |= 2; + + WREG32_RCU(RCU_ALTVDDNB_NOTIFY, param); + + sumo_send_msg_to_smu(rdev, SUMO_SMU_SERVICE_ROUTINE_ALTVDDNB_NOTIFY); +} + +void sumo_smu_pg_init(struct radeon_device *rdev) +{ + sumo_send_msg_to_smu(rdev, SUMO_SMU_SERVICE_ROUTINE_PG_INIT); +} + +static u32 sumo_power_of_4(u32 unit) +{ + u32 ret = 1; + u32 i; + + for (i = 0; i < unit; i++) + ret *= 4; + + return ret; +} + +void sumo_enable_boost_timer(struct radeon_device *rdev) +{ + struct sumo_power_info *pi = sumo_get_pi(rdev); + u32 period, unit, timer_value; + u32 xclk = radeon_get_xclk(rdev); + + unit = (RREG32_RCU(RCU_LCLK_SCALING_CNTL) & LCLK_SCALING_TIMER_PRESCALER_MASK) + >> LCLK_SCALING_TIMER_PRESCALER_SHIFT; + + period = 100 * (xclk / 100 / sumo_power_of_4(unit)); + + timer_value = (period << 16) | (unit << 4); + + WREG32_RCU(RCU_GNB_PWR_REP_TIMER_CNTL, timer_value); + WREG32_RCU(RCU_BOOST_MARGIN, pi->sys_info.sclk_dpm_boost_margin); + WREG32_RCU(RCU_THROTTLE_MARGIN, pi->sys_info.sclk_dpm_throttle_margin); + WREG32_RCU(GNB_TDP_LIMIT, pi->sys_info.gnb_tdp_limit); + WREG32_RCU(RCU_SclkDpmTdpLimitPG, pi->sys_info.sclk_dpm_tdp_limit_pg); + + sumo_send_msg_to_smu(rdev, SUMO_SMU_SERVICE_ROUTINE_GFX_SRV_ID_20); +} + +void sumo_set_tdp_limit(struct radeon_device *rdev, u32 index, u32 tdp_limit) +{ + u32 regoffset = 0; + u32 shift = 0; + u32 mask = 0xFFF; + u32 sclk_dpm_tdp_limit; + + switch (index) { + case 0: + regoffset = RCU_SclkDpmTdpLimit01; + shift = 16; + break; + case 1: + regoffset = RCU_SclkDpmTdpLimit01; + shift = 0; + break; + case 2: + regoffset = RCU_SclkDpmTdpLimit23; + shift = 16; + break; + case 3: + regoffset = RCU_SclkDpmTdpLimit23; + shift = 0; + break; + case 4: + regoffset = RCU_SclkDpmTdpLimit47; + shift = 16; + break; + case 7: + regoffset = RCU_SclkDpmTdpLimit47; + shift = 0; + break; + default: + break; + } + + sclk_dpm_tdp_limit = RREG32_RCU(regoffset); + sclk_dpm_tdp_limit &= ~(mask << shift); + sclk_dpm_tdp_limit |= (tdp_limit << shift); + WREG32_RCU(regoffset, sclk_dpm_tdp_limit); +} + +void sumo_boost_state_enable(struct radeon_device *rdev, bool enable) +{ + u32 boost_disable = RREG32_RCU(RCU_GPU_BOOST_DISABLE); + + boost_disable &= 0xFFFFFFFE; + boost_disable |= (enable ? 0 : 1); + WREG32_RCU(RCU_GPU_BOOST_DISABLE, boost_disable); +} + +u32 sumo_get_running_fw_version(struct radeon_device *rdev) +{ + return RREG32_RCU(RCU_FW_VERSION); +} + diff --git a/sys/dev/pci/drm/radeon/sumod.h b/sys/dev/pci/drm/radeon/sumod.h new file mode 100644 index 00000000000..7c9c2d4b86c --- /dev/null +++ b/sys/dev/pci/drm/radeon/sumod.h @@ -0,0 +1,372 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#ifndef _SUMOD_H_ +#define _SUMOD_H_ + +/* pm registers */ + +/* rcu */ +#define RCU_FW_VERSION 0x30c + +#define RCU_PWR_GATING_SEQ0 0x408 +#define RCU_PWR_GATING_SEQ1 0x40c +#define RCU_PWR_GATING_CNTL 0x410 +# define PWR_GATING_EN (1 << 0) +# define RSVD_MASK (0x3 << 1) +# define PCV(x) ((x) << 3) +# define PCV_MASK (0x1f << 3) +# define PCV_SHIFT 3 +# define PCP(x) ((x) << 8) +# define PCP_MASK (0xf << 8) +# define PCP_SHIFT 8 +# define RPW(x) ((x) << 16) +# define RPW_MASK (0xf << 16) +# define RPW_SHIFT 16 +# define ID(x) ((x) << 24) +# define ID_MASK (0xf << 24) +# define ID_SHIFT 24 +# define PGS(x) ((x) << 28) +# define PGS_MASK (0xf << 28) +# define PGS_SHIFT 28 + +#define RCU_ALTVDDNB_NOTIFY 0x430 +#define RCU_LCLK_SCALING_CNTL 0x434 +# define LCLK_SCALING_EN (1 << 0) +# define LCLK_SCALING_TYPE (1 << 1) +# define LCLK_SCALING_TIMER_PRESCALER(x) ((x) << 4) +# define LCLK_SCALING_TIMER_PRESCALER_MASK (0xf << 4) +# define LCLK_SCALING_TIMER_PRESCALER_SHIFT 4 +# define LCLK_SCALING_TIMER_PERIOD(x) ((x) << 16) +# define LCLK_SCALING_TIMER_PERIOD_MASK (0xf << 16) +# define LCLK_SCALING_TIMER_PERIOD_SHIFT 16 + +#define RCU_PWR_GATING_CNTL_2 0x4a0 +# define MPPU(x) ((x) << 0) +# define MPPU_MASK (0xffff << 0) +# define MPPU_SHIFT 0 +# define MPPD(x) ((x) << 16) +# define MPPD_MASK (0xffff << 16) +# define MPPD_SHIFT 16 +#define RCU_PWR_GATING_CNTL_3 0x4a4 +# define DPPU(x) ((x) << 0) +# define DPPU_MASK (0xffff << 0) +# define DPPU_SHIFT 0 +# define DPPD(x) ((x) << 16) +# define DPPD_MASK (0xffff << 16) +# define DPPD_SHIFT 16 +#define RCU_PWR_GATING_CNTL_4 0x4a8 +# define RT(x) ((x) << 0) +# define RT_MASK (0xffff << 0) +# define RT_SHIFT 0 +# define IT(x) ((x) << 16) +# define IT_MASK (0xffff << 16) +# define IT_SHIFT 16 + +/* yes these two have the same address */ +#define RCU_PWR_GATING_CNTL_5 0x504 +#define RCU_GPU_BOOST_DISABLE 0x508 + +#define MCU_M3ARB_INDEX 0x504 +#define MCU_M3ARB_PARAMS 0x508 + +#define RCU_GNB_PWR_REP_TIMER_CNTL 0x50C + +#define RCU_SclkDpmTdpLimit01 0x514 +#define RCU_SclkDpmTdpLimit23 0x518 +#define RCU_SclkDpmTdpLimit47 0x51C +#define RCU_SclkDpmTdpLimitPG 0x520 + +#define GNB_TDP_LIMIT 0x540 +#define RCU_BOOST_MARGIN 0x544 +#define RCU_THROTTLE_MARGIN 0x548 + +#define SMU_PCIE_PG_ARGS 0x58C +#define SMU_PCIE_PG_ARGS_2 0x598 +#define SMU_PCIE_PG_ARGS_3 0x59C + +/* mmio */ +#define RCU_STATUS 0x11c +# define GMC_PWR_GATER_BUSY (1 << 8) +# define GFX_PWR_GATER_BUSY (1 << 9) +# define UVD_PWR_GATER_BUSY (1 << 10) +# define PCIE_PWR_GATER_BUSY (1 << 11) +# define GMC_PWR_GATER_STATE (1 << 12) +# define GFX_PWR_GATER_STATE (1 << 13) +# define UVD_PWR_GATER_STATE (1 << 14) +# define PCIE_PWR_GATER_STATE (1 << 15) +# define GFX1_PWR_GATER_BUSY (1 << 16) +# define GFX2_PWR_GATER_BUSY (1 << 17) +# define GFX1_PWR_GATER_STATE (1 << 18) +# define GFX2_PWR_GATER_STATE (1 << 19) + +#define GFX_INT_REQ 0x120 +# define INT_REQ (1 << 0) +# define SERV_INDEX(x) ((x) << 1) +# define SERV_INDEX_MASK (0xff << 1) +# define SERV_INDEX_SHIFT 1 +#define GFX_INT_STATUS 0x124 +# define INT_ACK (1 << 0) +# define INT_DONE (1 << 1) + +#define CG_SCLK_CNTL 0x600 +# define SCLK_DIVIDER(x) ((x) << 0) +# define SCLK_DIVIDER_MASK (0x7f << 0) +# define SCLK_DIVIDER_SHIFT 0 +#define CG_SCLK_STATUS 0x604 +# define SCLK_OVERCLK_DETECT (1 << 2) + +#define CG_DCLK_CNTL 0x610 +# define DCLK_DIVIDER_MASK 0x7f +# define DCLK_DIR_CNTL_EN (1 << 8) +#define CG_DCLK_STATUS 0x614 +# define DCLK_STATUS (1 << 0) +#define CG_VCLK_CNTL 0x618 +# define VCLK_DIVIDER_MASK 0x7f +# define VCLK_DIR_CNTL_EN (1 << 8) +#define CG_VCLK_STATUS 0x61c + +#define GENERAL_PWRMGT 0x63c +# define STATIC_PM_EN (1 << 1) + +#define SCLK_PWRMGT_CNTL 0x644 +# define SCLK_PWRMGT_OFF (1 << 0) +# define SCLK_LOW_D1 (1 << 1) +# define FIR_RESET (1 << 4) +# define FIR_FORCE_TREND_SEL (1 << 5) +# define FIR_TREND_MODE (1 << 6) +# define DYN_GFX_CLK_OFF_EN (1 << 7) +# define GFX_CLK_FORCE_ON (1 << 8) +# define GFX_CLK_REQUEST_OFF (1 << 9) +# define GFX_CLK_FORCE_OFF (1 << 10) +# define GFX_CLK_OFF_ACPI_D1 (1 << 11) +# define GFX_CLK_OFF_ACPI_D2 (1 << 12) +# define GFX_CLK_OFF_ACPI_D3 (1 << 13) +# define GFX_VOLTAGE_CHANGE_EN (1 << 16) +# define GFX_VOLTAGE_CHANGE_MODE (1 << 17) + +#define TARGET_AND_CURRENT_PROFILE_INDEX 0x66c +# define TARG_SCLK_INDEX(x) ((x) << 6) +# define TARG_SCLK_INDEX_MASK (0x7 << 6) +# define TARG_SCLK_INDEX_SHIFT 6 +# define CURR_SCLK_INDEX(x) ((x) << 9) +# define CURR_SCLK_INDEX_MASK (0x7 << 9) +# define CURR_SCLK_INDEX_SHIFT 9 +# define TARG_INDEX(x) ((x) << 12) +# define TARG_INDEX_MASK (0x7 << 12) +# define TARG_INDEX_SHIFT 12 +# define CURR_INDEX(x) ((x) << 15) +# define CURR_INDEX_MASK (0x7 << 15) +# define CURR_INDEX_SHIFT 15 + +#define CG_SCLK_DPM_CTRL 0x684 +# define SCLK_FSTATE_0_DIV(x) ((x) << 0) +# define SCLK_FSTATE_0_DIV_MASK (0x7f << 0) +# define SCLK_FSTATE_0_DIV_SHIFT 0 +# define SCLK_FSTATE_0_VLD (1 << 7) +# define SCLK_FSTATE_1_DIV(x) ((x) << 8) +# define SCLK_FSTATE_1_DIV_MASK (0x7f << 8) +# define SCLK_FSTATE_1_DIV_SHIFT 8 +# define SCLK_FSTATE_1_VLD (1 << 15) +# define SCLK_FSTATE_2_DIV(x) ((x) << 16) +# define SCLK_FSTATE_2_DIV_MASK (0x7f << 16) +# define SCLK_FSTATE_2_DIV_SHIFT 16 +# define SCLK_FSTATE_2_VLD (1 << 23) +# define SCLK_FSTATE_3_DIV(x) ((x) << 24) +# define SCLK_FSTATE_3_DIV_MASK (0x7f << 24) +# define SCLK_FSTATE_3_DIV_SHIFT 24 +# define SCLK_FSTATE_3_VLD (1 << 31) +#define CG_SCLK_DPM_CTRL_2 0x688 +#define CG_GCOOR 0x68c +# define PHC(x) ((x) << 0) +# define PHC_MASK (0x1f << 0) +# define PHC_SHIFT 0 +# define SDC(x) ((x) << 9) +# define SDC_MASK (0x3ff << 9) +# define SDC_SHIFT 9 +# define SU(x) ((x) << 23) +# define SU_MASK (0xf << 23) +# define SU_SHIFT 23 +# define DIV_ID(x) ((x) << 28) +# define DIV_ID_MASK (0x7 << 28) +# define DIV_ID_SHIFT 28 + +#define CG_FTV 0x690 +#define CG_FFCT_0 0x694 +# define UTC_0(x) ((x) << 0) +# define UTC_0_MASK (0x3ff << 0) +# define UTC_0_SHIFT 0 +# define DTC_0(x) ((x) << 10) +# define DTC_0_MASK (0x3ff << 10) +# define DTC_0_SHIFT 10 + +#define CG_GIT 0x6d8 +# define CG_GICST(x) ((x) << 0) +# define CG_GICST_MASK (0xffff << 0) +# define CG_GICST_SHIFT 0 +# define CG_GIPOT(x) ((x) << 16) +# define CG_GIPOT_MASK (0xffff << 16) +# define CG_GIPOT_SHIFT 16 + +#define CG_SCLK_DPM_CTRL_3 0x6e0 +# define FORCE_SCLK_STATE(x) ((x) << 0) +# define FORCE_SCLK_STATE_MASK (0x7 << 0) +# define FORCE_SCLK_STATE_SHIFT 0 +# define FORCE_SCLK_STATE_EN (1 << 3) +# define GNB_TT(x) ((x) << 8) +# define GNB_TT_MASK (0xff << 8) +# define GNB_TT_SHIFT 8 +# define GNB_THERMTHRO_MASK (1 << 16) +# define CNB_THERMTHRO_MASK_SCLK (1 << 17) +# define DPM_SCLK_ENABLE (1 << 18) +# define GNB_SLOW_FSTATE_0_MASK (1 << 23) +# define GNB_SLOW_FSTATE_0_SHIFT 23 +# define FORCE_NB_PSTATE_1 (1 << 31) + +#define CG_SSP 0x6e8 +# define SST(x) ((x) << 0) +# define SST_MASK (0xffff << 0) +# define SST_SHIFT 0 +# define SSTU(x) ((x) << 16) +# define SSTU_MASK (0xffff << 16) +# define SSTU_SHIFT 16 + +#define CG_ACPI_CNTL 0x70c +# define SCLK_ACPI_DIV(x) ((x) << 0) +# define SCLK_ACPI_DIV_MASK (0x7f << 0) +# define SCLK_ACPI_DIV_SHIFT 0 + +#define CG_SCLK_DPM_CTRL_4 0x71c +# define DC_HDC(x) ((x) << 14) +# define DC_HDC_MASK (0x3fff << 14) +# define DC_HDC_SHIFT 14 +# define DC_HU(x) ((x) << 28) +# define DC_HU_MASK (0xf << 28) +# define DC_HU_SHIFT 28 +#define CG_SCLK_DPM_CTRL_5 0x720 +# define SCLK_FSTATE_BOOTUP(x) ((x) << 0) +# define SCLK_FSTATE_BOOTUP_MASK (0x7 << 0) +# define SCLK_FSTATE_BOOTUP_SHIFT 0 +# define TT_TP(x) ((x) << 3) +# define TT_TP_MASK (0xffff << 3) +# define TT_TP_SHIFT 3 +# define TT_TU(x) ((x) << 19) +# define TT_TU_MASK (0xff << 19) +# define TT_TU_SHIFT 19 +#define CG_SCLK_DPM_CTRL_6 0x724 +#define CG_AT_0 0x728 +# define CG_R(x) ((x) << 0) +# define CG_R_MASK (0xffff << 0) +# define CG_R_SHIFT 0 +# define CG_L(x) ((x) << 16) +# define CG_L_MASK (0xffff << 16) +# define CG_L_SHIFT 16 +#define CG_AT_1 0x72c +#define CG_AT_2 0x730 +#define CG_THERMAL_INT 0x734 +#define DIG_THERM_INTH(x) ((x) << 8) +#define DIG_THERM_INTH_MASK 0x0000FF00 +#define DIG_THERM_INTH_SHIFT 8 +#define DIG_THERM_INTL(x) ((x) << 16) +#define DIG_THERM_INTL_MASK 0x00FF0000 +#define DIG_THERM_INTL_SHIFT 16 +#define THERM_INT_MASK_HIGH (1 << 24) +#define THERM_INT_MASK_LOW (1 << 25) +#define CG_AT_3 0x738 +#define CG_AT_4 0x73c +#define CG_AT_5 0x740 +#define CG_AT_6 0x744 +#define CG_AT_7 0x748 + +#define CG_BSP_0 0x750 +# define BSP(x) ((x) << 0) +# define BSP_MASK (0xffff << 0) +# define BSP_SHIFT 0 +# define BSU(x) ((x) << 16) +# define BSU_MASK (0xf << 16) +# define BSU_SHIFT 16 + +#define CG_CG_VOLTAGE_CNTL 0x770 +# define REQ (1 << 0) +# define LEVEL(x) ((x) << 1) +# define LEVEL_MASK (0x3 << 1) +# define LEVEL_SHIFT 1 +# define CG_VOLTAGE_EN (1 << 3) +# define FORCE (1 << 4) +# define PERIOD(x) ((x) << 8) +# define PERIOD_MASK (0xffff << 8) +# define PERIOD_SHIFT 8 +# define UNIT(x) ((x) << 24) +# define UNIT_MASK (0xf << 24) +# define UNIT_SHIFT 24 + +#define CG_ACPI_VOLTAGE_CNTL 0x780 +# define ACPI_VOLTAGE_EN (1 << 8) + +#define CG_DPM_VOLTAGE_CNTL 0x788 +# define DPM_STATE0_LEVEL_MASK (0x3 << 0) +# define DPM_STATE0_LEVEL_SHIFT 0 +# define DPM_VOLTAGE_EN (1 << 16) + +#define CG_PWR_GATING_CNTL 0x7ac +# define DYN_PWR_DOWN_EN (1 << 0) +# define ACPI_PWR_DOWN_EN (1 << 1) +# define GFX_CLK_OFF_PWR_DOWN_EN (1 << 2) +# define IOC_DISGPU_PWR_DOWN_EN (1 << 3) +# define FORCE_POWR_ON (1 << 4) +# define PGP(x) ((x) << 8) +# define PGP_MASK (0xffff << 8) +# define PGP_SHIFT 8 +# define PGU(x) ((x) << 24) +# define PGU_MASK (0xf << 24) +# define PGU_SHIFT 24 + +#define CG_CGTT_LOCAL_0 0x7d0 +#define CG_CGTT_LOCAL_1 0x7d4 + +#define DEEP_SLEEP_CNTL 0x818 +# define R_DIS (1 << 3) +# define HS(x) ((x) << 4) +# define HS_MASK (0xfff << 4) +# define HS_SHIFT 4 +# define ENABLE_DS (1 << 31) +#define DEEP_SLEEP_CNTL2 0x81c +# define LB_UFP_EN (1 << 0) +# define INOUT_C(x) ((x) << 4) +# define INOUT_C_MASK (0xff << 4) +# define INOUT_C_SHIFT 4 + +#define CG_SCRATCH2 0x824 + +#define CG_SCLK_DPM_CTRL_11 0x830 + +#define HW_REV 0x5564 +# define ATI_REV_ID_MASK (0xf << 28) +# define ATI_REV_ID_SHIFT 28 +/* 0 = A0, 1 = A1, 2 = B0, 3 = C0, etc. */ + +#define DOUT_SCRATCH3 0x611c + +#define GB_ADDR_CONFIG 0x98f8 + +#endif diff --git a/sys/dev/pci/drm/radeon/trinity_dpm.c b/sys/dev/pci/drm/radeon/trinity_dpm.c new file mode 100644 index 00000000000..93b869e7eb7 --- /dev/null +++ b/sys/dev/pci/drm/radeon/trinity_dpm.c @@ -0,0 +1,2103 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "trinityd.h" +#include "r600_dpm.h" +#include "trinity_dpm.h" + +#define TRINITY_MAX_DEEPSLEEP_DIVIDER_ID 5 +#define TRINITY_MINIMUM_ENGINE_CLOCK 800 +#define SCLK_MIN_DIV_INTV_SHIFT 12 +#define TRINITY_DISPCLK_BYPASS_THRESHOLD 10000 + +#ifndef TRINITY_MGCG_SEQUENCE +#define TRINITY_MGCG_SEQUENCE 100 + +static const u32 trinity_mgcg_shls_default[] = +{ + /* Register, Value, Mask */ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x00003fc4, 0xc0000000, 0xffffffff, + 0x00005448, 0x00000100, 0xffffffff, + 0x000055e4, 0x00000100, 0xffffffff, + 0x0000160c, 0x00000100, 0xffffffff, + 0x00008984, 0x06000100, 0xffffffff, + 0x0000c164, 0x00000100, 0xffffffff, + 0x00008a18, 0x00000100, 0xffffffff, + 0x0000897c, 0x06000100, 0xffffffff, + 0x00008b28, 0x00000100, 0xffffffff, + 0x00009144, 0x00800200, 0xffffffff, + 0x00009a60, 0x00000100, 0xffffffff, + 0x00009868, 0x00000100, 0xffffffff, + 0x00008d58, 0x00000100, 0xffffffff, + 0x00009510, 0x00000100, 0xffffffff, + 0x0000949c, 0x00000100, 0xffffffff, + 0x00009654, 0x00000100, 0xffffffff, + 0x00009030, 0x00000100, 0xffffffff, + 0x00009034, 0x00000100, 0xffffffff, + 0x00009038, 0x00000100, 0xffffffff, + 0x0000903c, 0x00000100, 0xffffffff, + 0x00009040, 0x00000100, 0xffffffff, + 0x0000a200, 0x00000100, 0xffffffff, + 0x0000a204, 0x00000100, 0xffffffff, + 0x0000a208, 0x00000100, 0xffffffff, + 0x0000a20c, 0x00000100, 0xffffffff, + 0x00009744, 0x00000100, 0xffffffff, + 0x00003f80, 0x00000100, 0xffffffff, + 0x0000a210, 0x00000100, 0xffffffff, + 0x0000a214, 0x00000100, 0xffffffff, + 0x000004d8, 0x00000100, 0xffffffff, + 0x00009664, 0x00000100, 0xffffffff, + 0x00009698, 0x00000100, 0xffffffff, + 0x000004d4, 0x00000200, 0xffffffff, + 0x000004d0, 0x00000000, 0xffffffff, + 0x000030cc, 0x00000104, 0xffffffff, + 0x0000d0c0, 0x00000100, 0xffffffff, + 0x0000d8c0, 0x00000100, 0xffffffff, + 0x0000951c, 0x00010000, 0xffffffff, + 0x00009160, 0x00030002, 0xffffffff, + 0x00009164, 0x00050004, 0xffffffff, + 0x00009168, 0x00070006, 0xffffffff, + 0x00009178, 0x00070000, 0xffffffff, + 0x0000917c, 0x00030002, 0xffffffff, + 0x00009180, 0x00050004, 0xffffffff, + 0x0000918c, 0x00010006, 0xffffffff, + 0x00009190, 0x00090008, 0xffffffff, + 0x00009194, 0x00070000, 0xffffffff, + 0x00009198, 0x00030002, 0xffffffff, + 0x0000919c, 0x00050004, 0xffffffff, + 0x000091a8, 0x00010006, 0xffffffff, + 0x000091ac, 0x00090008, 0xffffffff, + 0x000091b0, 0x00070000, 0xffffffff, + 0x000091b4, 0x00030002, 0xffffffff, + 0x000091b8, 0x00050004, 0xffffffff, + 0x000091c4, 0x00010006, 0xffffffff, + 0x000091c8, 0x00090008, 0xffffffff, + 0x000091cc, 0x00070000, 0xffffffff, + 0x000091d0, 0x00030002, 0xffffffff, + 0x000091d4, 0x00050004, 0xffffffff, + 0x000091e0, 0x00010006, 0xffffffff, + 0x000091e4, 0x00090008, 0xffffffff, + 0x000091e8, 0x00000000, 0xffffffff, + 0x000091ec, 0x00070000, 0xffffffff, + 0x000091f0, 0x00030002, 0xffffffff, + 0x000091f4, 0x00050004, 0xffffffff, + 0x00009200, 0x00010006, 0xffffffff, + 0x00009204, 0x00090008, 0xffffffff, + 0x00009208, 0x00070000, 0xffffffff, + 0x0000920c, 0x00030002, 0xffffffff, + 0x00009210, 0x00050004, 0xffffffff, + 0x0000921c, 0x00010006, 0xffffffff, + 0x00009220, 0x00090008, 0xffffffff, + 0x00009294, 0x00000000, 0xffffffff +}; + +static const u32 trinity_mgcg_shls_enable[] = +{ + /* Register, Value, Mask */ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0x00000000, 0x000133FF, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0x00000000, 0xE00B03FC, + 0x00009150, 0x96944200, 0xffffffff +}; + +static const u32 trinity_mgcg_shls_disable[] = +{ + /* Register, Value, Mask */ + 0x0000802c, 0xc0000000, 0xffffffff, + 0x00009150, 0x00600000, 0xffffffff, + 0x000008f8, 0x00000000, 0xffffffff, + 0x000008fc, 0xffffffff, 0x000133FF, + 0x000008f8, 0x00000001, 0xffffffff, + 0x000008fc, 0xffffffff, 0xE00B03FC +}; +#endif + +#ifndef TRINITY_SYSLS_SEQUENCE +#define TRINITY_SYSLS_SEQUENCE 100 + +static const u32 trinity_sysls_default[] = +{ + /* Register, Value, Mask */ + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x0000d8bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x000020c0, 0x000c0c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x00002f50, 0x00000404, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x0000641c, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00006dfc, 0x00000000, 0xffffffff +}; + +static const u32 trinity_sysls_disable[] = +{ + /* Register, Value, Mask */ + 0x0000d0c0, 0x00000000, 0xffffffff, + 0x0000d8c0, 0x00000000, 0xffffffff, + 0x000055e8, 0x00000000, 0xffffffff, + 0x0000d0bc, 0x00000000, 0xffffffff, + 0x0000d8bc, 0x00000000, 0xffffffff, + 0x000015c0, 0x00041401, 0xffffffff, + 0x0000264c, 0x00040400, 0xffffffff, + 0x00002648, 0x00040400, 0xffffffff, + 0x00002650, 0x00040400, 0xffffffff, + 0x000020b8, 0x00040400, 0xffffffff, + 0x000020bc, 0x00040400, 0xffffffff, + 0x000020c0, 0x00040c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680000, 0xffffffff, + 0x00002f50, 0x00000404, 0xffffffff, + 0x000004c8, 0x00000001, 0xffffffff, + 0x0000641c, 0x00007ffd, 0xffffffff, + 0x00000c7c, 0x0000ff00, 0xffffffff, + 0x00006dfc, 0x0000007f, 0xffffffff +}; + +static const u32 trinity_sysls_enable[] = +{ + /* Register, Value, Mask */ + 0x000055e8, 0x00000001, 0xffffffff, + 0x0000d0bc, 0x00000100, 0xffffffff, + 0x0000d8bc, 0x00000100, 0xffffffff, + 0x000015c0, 0x000c1401, 0xffffffff, + 0x0000264c, 0x000c0400, 0xffffffff, + 0x00002648, 0x000c0400, 0xffffffff, + 0x00002650, 0x000c0400, 0xffffffff, + 0x000020b8, 0x000c0400, 0xffffffff, + 0x000020bc, 0x000c0400, 0xffffffff, + 0x000020c0, 0x000c0c80, 0xffffffff, + 0x0000f4a0, 0x000000c0, 0xffffffff, + 0x0000f4a4, 0x00680fff, 0xffffffff, + 0x00002f50, 0x00000903, 0xffffffff, + 0x000004c8, 0x00000000, 0xffffffff, + 0x0000641c, 0x00000000, 0xffffffff, + 0x00000c7c, 0x00000000, 0xffffffff, + 0x00006dfc, 0x00000000, 0xffffffff +}; +#endif + +static const u32 trinity_override_mgpg_sequences[] = +{ + /* Register, Value */ + 0x00000200, 0xE030032C, + 0x00000204, 0x00000FFF, + 0x00000200, 0xE0300058, + 0x00000204, 0x00030301, + 0x00000200, 0xE0300054, + 0x00000204, 0x500010FF, + 0x00000200, 0xE0300074, + 0x00000204, 0x00030301, + 0x00000200, 0xE0300070, + 0x00000204, 0x500010FF, + 0x00000200, 0xE0300090, + 0x00000204, 0x00030301, + 0x00000200, 0xE030008C, + 0x00000204, 0x500010FF, + 0x00000200, 0xE03000AC, + 0x00000204, 0x00030301, + 0x00000200, 0xE03000A8, + 0x00000204, 0x500010FF, + 0x00000200, 0xE03000C8, + 0x00000204, 0x00030301, + 0x00000200, 0xE03000C4, + 0x00000204, 0x500010FF, + 0x00000200, 0xE03000E4, + 0x00000204, 0x00030301, + 0x00000200, 0xE03000E0, + 0x00000204, 0x500010FF, + 0x00000200, 0xE0300100, + 0x00000204, 0x00030301, + 0x00000200, 0xE03000FC, + 0x00000204, 0x500010FF, + 0x00000200, 0xE0300058, + 0x00000204, 0x00030303, + 0x00000200, 0xE0300054, + 0x00000204, 0x600010FF, + 0x00000200, 0xE0300074, + 0x00000204, 0x00030303, + 0x00000200, 0xE0300070, + 0x00000204, 0x600010FF, + 0x00000200, 0xE0300090, + 0x00000204, 0x00030303, + 0x00000200, 0xE030008C, + 0x00000204, 0x600010FF, + 0x00000200, 0xE03000AC, + 0x00000204, 0x00030303, + 0x00000200, 0xE03000A8, + 0x00000204, 0x600010FF, + 0x00000200, 0xE03000C8, + 0x00000204, 0x00030303, + 0x00000200, 0xE03000C4, + 0x00000204, 0x600010FF, + 0x00000200, 0xE03000E4, + 0x00000204, 0x00030303, + 0x00000200, 0xE03000E0, + 0x00000204, 0x600010FF, + 0x00000200, 0xE0300100, + 0x00000204, 0x00030303, + 0x00000200, 0xE03000FC, + 0x00000204, 0x600010FF, + 0x00000200, 0xE0300058, + 0x00000204, 0x00030303, + 0x00000200, 0xE0300054, + 0x00000204, 0x700010FF, + 0x00000200, 0xE0300074, + 0x00000204, 0x00030303, + 0x00000200, 0xE0300070, + 0x00000204, 0x700010FF, + 0x00000200, 0xE0300090, + 0x00000204, 0x00030303, + 0x00000200, 0xE030008C, + 0x00000204, 0x700010FF, + 0x00000200, 0xE03000AC, + 0x00000204, 0x00030303, + 0x00000200, 0xE03000A8, + 0x00000204, 0x700010FF, + 0x00000200, 0xE03000C8, + 0x00000204, 0x00030303, + 0x00000200, 0xE03000C4, + 0x00000204, 0x700010FF, + 0x00000200, 0xE03000E4, + 0x00000204, 0x00030303, + 0x00000200, 0xE03000E0, + 0x00000204, 0x700010FF, + 0x00000200, 0xE0300100, + 0x00000204, 0x00030303, + 0x00000200, 0xE03000FC, + 0x00000204, 0x700010FF, + 0x00000200, 0xE0300058, + 0x00000204, 0x00010303, + 0x00000200, 0xE0300054, + 0x00000204, 0x800010FF, + 0x00000200, 0xE0300074, + 0x00000204, 0x00010303, + 0x00000200, 0xE0300070, + 0x00000204, 0x800010FF, + 0x00000200, 0xE0300090, + 0x00000204, 0x00010303, + 0x00000200, 0xE030008C, + 0x00000204, 0x800010FF, + 0x00000200, 0xE03000AC, + 0x00000204, 0x00010303, + 0x00000200, 0xE03000A8, + 0x00000204, 0x800010FF, + 0x00000200, 0xE03000C4, + 0x00000204, 0x800010FF, + 0x00000200, 0xE03000C8, + 0x00000204, 0x00010303, + 0x00000200, 0xE03000E4, + 0x00000204, 0x00010303, + 0x00000200, 0xE03000E0, + 0x00000204, 0x800010FF, + 0x00000200, 0xE0300100, + 0x00000204, 0x00010303, + 0x00000200, 0xE03000FC, + 0x00000204, 0x800010FF, + 0x00000200, 0x0001f198, + 0x00000204, 0x0003ffff, + 0x00000200, 0x0001f19C, + 0x00000204, 0x3fffffff, + 0x00000200, 0xE030032C, + 0x00000204, 0x00000000, +}; + +extern void vce_v1_0_enable_mgcg(struct radeon_device *rdev, bool enable); +static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev, + const u32 *seq, u32 count); +static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev); +static void trinity_apply_state_adjust_rules(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps); + +static struct trinity_ps *trinity_get_ps(struct radeon_ps *rps) +{ + struct trinity_ps *ps = rps->ps_priv; + + return ps; +} + +static struct trinity_power_info *trinity_get_pi(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = rdev->pm.dpm.priv; + + return pi; +} + +static void trinity_gfx_powergating_initialize(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 p, u; + u32 value; + struct atom_clock_dividers dividers; + u32 xclk = radeon_get_xclk(rdev); + u32 sssd = 1; + int ret; + u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + 25000, false, ÷rs); + if (ret) + return; + + value = RREG32_SMC(GFX_POWER_GATING_CNTL); + value &= ~(SSSD_MASK | PDS_DIV_MASK); + if (sssd) + value |= SSSD(1); + value |= PDS_DIV(dividers.post_div); + WREG32_SMC(GFX_POWER_GATING_CNTL, value); + + r600_calculate_u_and_p(500, xclk, 16, &p, &u); + + WREG32(CG_PG_CTRL, SP(p) | SU(u)); + + WREG32_P(CG_GIPOTS, CG_GIPOT(p), ~CG_GIPOT_MASK); + + /* XXX double check hw_rev */ + if (pi->override_dynamic_mgpg && (hw_rev == 0)) + trinity_override_dynamic_mg_powergating(rdev); + +} + +#define CGCG_CGTT_LOCAL0_MASK 0xFFFF33FF +#define CGCG_CGTT_LOCAL1_MASK 0xFFFB0FFE +#define CGTS_SM_CTRL_REG_DISABLE 0x00600000 +#define CGTS_SM_CTRL_REG_ENABLE 0x96944200 + +static void trinity_mg_clockgating_enable(struct radeon_device *rdev, + bool enable) +{ + u32 local0; + u32 local1; + + if (enable) { + local0 = RREG32_CG(CG_CGTT_LOCAL_0); + local1 = RREG32_CG(CG_CGTT_LOCAL_1); + + WREG32_CG(CG_CGTT_LOCAL_0, + (0x00380000 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) ); + WREG32_CG(CG_CGTT_LOCAL_1, + (0x0E000000 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) ); + + WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_ENABLE); + } else { + WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_DISABLE); + + local0 = RREG32_CG(CG_CGTT_LOCAL_0); + local1 = RREG32_CG(CG_CGTT_LOCAL_1); + + WREG32_CG(CG_CGTT_LOCAL_0, + CGCG_CGTT_LOCAL0_MASK | (local0 & ~CGCG_CGTT_LOCAL0_MASK) ); + WREG32_CG(CG_CGTT_LOCAL_1, + CGCG_CGTT_LOCAL1_MASK | (local1 & ~CGCG_CGTT_LOCAL1_MASK) ); + } +} + +static void trinity_mg_clockgating_initialize(struct radeon_device *rdev) +{ + u32 count; + const u32 *seq = NULL; + + seq = &trinity_mgcg_shls_default[0]; + count = sizeof(trinity_mgcg_shls_default) / (3 * sizeof(u32)); + + trinity_program_clk_gating_hw_sequence(rdev, seq, count); +} + +static void trinity_gfx_clockgating_enable(struct radeon_device *rdev, + bool enable) +{ + if (enable) { + WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN); + } else { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN); + WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON); + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON); + RREG32(GB_ADDR_CONFIG); + } +} + +static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev, + const u32 *seq, u32 count) +{ + u32 i, length = count * 3; + + for (i = 0; i < length; i += 3) + WREG32_P(seq[i], seq[i+1], ~seq[i+2]); +} + +static void trinity_program_override_mgpg_sequences(struct radeon_device *rdev, + const u32 *seq, u32 count) +{ + u32 i, length = count * 2; + + for (i = 0; i < length; i += 2) + WREG32(seq[i], seq[i+1]); + +} + +static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev) +{ + u32 count; + const u32 *seq = NULL; + + seq = &trinity_override_mgpg_sequences[0]; + count = sizeof(trinity_override_mgpg_sequences) / (2 * sizeof(u32)); + + trinity_program_override_mgpg_sequences(rdev, seq, count); +} + +static void trinity_ls_clockgating_enable(struct radeon_device *rdev, + bool enable) +{ + u32 count; + const u32 *seq = NULL; + + if (enable) { + seq = &trinity_sysls_enable[0]; + count = sizeof(trinity_sysls_enable) / (3 * sizeof(u32)); + } else { + seq = &trinity_sysls_disable[0]; + count = sizeof(trinity_sysls_disable) / (3 * sizeof(u32)); + } + + trinity_program_clk_gating_hw_sequence(rdev, seq, count); +} + +static void trinity_gfx_powergating_enable(struct radeon_device *rdev, + bool enable) +{ + if (enable) { + if (RREG32_SMC(CC_SMU_TST_EFUSE1_MISC) & RB_BACKEND_DISABLE_MASK) + WREG32_SMC(SMU_SCRATCH_A, (RREG32_SMC(SMU_SCRATCH_A) | 0x01)); + + WREG32_P(SCLK_PWRMGT_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN); + } else { + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_PWR_DOWN_EN); + RREG32(GB_ADDR_CONFIG); + } +} + +static void trinity_gfx_dynamic_mgpg_enable(struct radeon_device *rdev, + bool enable) +{ + u32 value; + + if (enable) { + value = RREG32_SMC(PM_I_CNTL_1); + value &= ~DS_PG_CNTL_MASK; + value |= DS_PG_CNTL(1); + WREG32_SMC(PM_I_CNTL_1, value); + + value = RREG32_SMC(SMU_S_PG_CNTL); + value &= ~DS_PG_EN_MASK; + value |= DS_PG_EN(1); + WREG32_SMC(SMU_S_PG_CNTL, value); + } else { + value = RREG32_SMC(SMU_S_PG_CNTL); + value &= ~DS_PG_EN_MASK; + WREG32_SMC(SMU_S_PG_CNTL, value); + + value = RREG32_SMC(PM_I_CNTL_1); + value &= ~DS_PG_CNTL_MASK; + WREG32_SMC(PM_I_CNTL_1, value); + } + + trinity_gfx_dynamic_mgpg_config(rdev); + +} + +static void trinity_enable_clock_power_gating(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + if (pi->enable_gfx_clock_gating) + sumo_gfx_clockgating_initialize(rdev); + if (pi->enable_mg_clock_gating) + trinity_mg_clockgating_initialize(rdev); + if (pi->enable_gfx_power_gating) + trinity_gfx_powergating_initialize(rdev); + if (pi->enable_mg_clock_gating) { + trinity_ls_clockgating_enable(rdev, true); + trinity_mg_clockgating_enable(rdev, true); + } + if (pi->enable_gfx_clock_gating) + trinity_gfx_clockgating_enable(rdev, true); + if (pi->enable_gfx_dynamic_mgpg) + trinity_gfx_dynamic_mgpg_enable(rdev, true); + if (pi->enable_gfx_power_gating) + trinity_gfx_powergating_enable(rdev, true); +} + +static void trinity_disable_clock_power_gating(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + if (pi->enable_gfx_power_gating) + trinity_gfx_powergating_enable(rdev, false); + if (pi->enable_gfx_dynamic_mgpg) + trinity_gfx_dynamic_mgpg_enable(rdev, false); + if (pi->enable_gfx_clock_gating) + trinity_gfx_clockgating_enable(rdev, false); + if (pi->enable_mg_clock_gating) { + trinity_mg_clockgating_enable(rdev, false); + trinity_ls_clockgating_enable(rdev, false); + } +} + +static void trinity_set_divider_value(struct radeon_device *rdev, + u32 index, u32 sclk) +{ + struct atom_clock_dividers dividers; + int ret; + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + sclk, false, ÷rs); + if (ret) + return; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix); + value &= ~CLK_DIVIDER_MASK; + value |= CLK_DIVIDER(dividers.post_div); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value); + + ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, + sclk/2, false, ÷rs); + if (ret) + return; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix); + value &= ~PD_SCLK_DIVIDER_MASK; + value |= PD_SCLK_DIVIDER(dividers.post_div); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix, value); +} + +static void trinity_set_ds_dividers(struct radeon_device *rdev, + u32 index, u32 divider) +{ + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix); + value &= ~DS_DIV_MASK; + value |= DS_DIV(divider); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value); +} + +static void trinity_set_ss_dividers(struct radeon_device *rdev, + u32 index, u32 divider) +{ + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix); + value &= ~DS_SH_DIV_MASK; + value |= DS_SH_DIV(divider); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value); +} + +static void trinity_set_vid(struct radeon_device *rdev, u32 index, u32 vid) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid); + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix); + value &= ~VID_MASK; + value |= VID(vid_7bit); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value); + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix); + value &= ~LVRT_MASK; + value |= LVRT(0); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value); +} + +static void trinity_set_allos_gnb_slow(struct radeon_device *rdev, + u32 index, u32 gnb_slow) +{ + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix); + value &= ~GNB_SLOW_MASK; + value |= GNB_SLOW(gnb_slow); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value); +} + +static void trinity_set_force_nbp_state(struct radeon_device *rdev, + u32 index, u32 force_nbp_state) +{ + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix); + value &= ~FORCE_NBPS1_MASK; + value |= FORCE_NBPS1(force_nbp_state); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value); +} + +static void trinity_set_display_wm(struct radeon_device *rdev, + u32 index, u32 wm) +{ + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix); + value &= ~DISPLAY_WM_MASK; + value |= DISPLAY_WM(wm); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value); +} + +static void trinity_set_vce_wm(struct radeon_device *rdev, + u32 index, u32 wm) +{ + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix); + value &= ~VCE_WM_MASK; + value |= VCE_WM(wm); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value); +} + +static void trinity_set_at(struct radeon_device *rdev, + u32 index, u32 at) +{ + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix); + value &= ~AT_MASK; + value |= AT(at); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix, value); +} + +static void trinity_program_power_level(struct radeon_device *rdev, + struct trinity_pl *pl, u32 index) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + if (index >= SUMO_MAX_HARDWARE_POWERLEVELS) + return; + + trinity_set_divider_value(rdev, index, pl->sclk); + trinity_set_vid(rdev, index, pl->vddc_index); + trinity_set_ss_dividers(rdev, index, pl->ss_divider_index); + trinity_set_ds_dividers(rdev, index, pl->ds_divider_index); + trinity_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow); + trinity_set_force_nbp_state(rdev, index, pl->force_nbp_state); + trinity_set_display_wm(rdev, index, pl->display_wm); + trinity_set_vce_wm(rdev, index, pl->vce_wm); + trinity_set_at(rdev, index, pi->at[index]); +} + +static void trinity_power_level_enable_disable(struct radeon_device *rdev, + u32 index, bool enable) +{ + u32 value; + u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE; + + value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix); + value &= ~STATE_VALID_MASK; + if (enable) + value |= STATE_VALID(1); + WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value); +} + +static bool trinity_dpm_enabled(struct radeon_device *rdev) +{ + if (RREG32_SMC(SMU_SCLK_DPM_CNTL) & SCLK_DPM_EN(1)) + return true; + else + return false; +} + +static void trinity_start_dpm(struct radeon_device *rdev) +{ + u32 value = RREG32_SMC(SMU_SCLK_DPM_CNTL); + + value &= ~(SCLK_DPM_EN_MASK | SCLK_DPM_BOOT_STATE_MASK | VOLTAGE_CHG_EN_MASK); + value |= SCLK_DPM_EN(1) | SCLK_DPM_BOOT_STATE(0) | VOLTAGE_CHG_EN(1); + WREG32_SMC(SMU_SCLK_DPM_CNTL, value); + + WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN); + WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~EN); + + trinity_dpm_config(rdev, true); +} + +static void trinity_wait_for_dpm_enabled(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(SCLK_PWRMGT_CNTL) & DYNAMIC_PM_EN) + break; + udelay(1); + } + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_STATE_MASK) == 0) + break; + udelay(1); + } + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0) + break; + udelay(1); + } +} + +static void trinity_stop_dpm(struct radeon_device *rdev) +{ + u32 sclk_dpm_cntl; + + WREG32_P(CG_CG_VOLTAGE_CNTL, EN, ~EN); + + sclk_dpm_cntl = RREG32_SMC(SMU_SCLK_DPM_CNTL); + sclk_dpm_cntl &= ~(SCLK_DPM_EN_MASK | VOLTAGE_CHG_EN_MASK); + WREG32_SMC(SMU_SCLK_DPM_CNTL, sclk_dpm_cntl); + + trinity_dpm_config(rdev, false); +} + +static void trinity_start_am(struct radeon_device *rdev) +{ + WREG32_P(SCLK_PWRMGT_CNTL, 0, ~(RESET_SCLK_CNT | RESET_BUSY_CNT)); +} + +static void trinity_reset_am(struct radeon_device *rdev) +{ + WREG32_P(SCLK_PWRMGT_CNTL, RESET_SCLK_CNT | RESET_BUSY_CNT, + ~(RESET_SCLK_CNT | RESET_BUSY_CNT)); +} + +static void trinity_wait_for_level_0(struct radeon_device *rdev) +{ + int i; + + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0) + break; + udelay(1); + } +} + +static void trinity_enable_power_level_0(struct radeon_device *rdev) +{ + trinity_power_level_enable_disable(rdev, 0, true); +} + +static void trinity_force_level_0(struct radeon_device *rdev) +{ + trinity_dpm_force_state(rdev, 0); +} + +static void trinity_unforce_levels(struct radeon_device *rdev) +{ + trinity_dpm_no_forced_level(rdev); +} + +static void trinity_program_power_levels_0_to_n(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct trinity_ps *new_ps = trinity_get_ps(new_rps); + struct trinity_ps *old_ps = trinity_get_ps(old_rps); + u32 i; + u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels; + + for (i = 0; i < new_ps->num_levels; i++) { + trinity_program_power_level(rdev, &new_ps->levels[i], i); + trinity_power_level_enable_disable(rdev, i, true); + } + + for (i = new_ps->num_levels; i < n_current_state_levels; i++) + trinity_power_level_enable_disable(rdev, i, false); +} + +static void trinity_program_bootup_state(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 i; + + trinity_program_power_level(rdev, &pi->boot_pl, 0); + trinity_power_level_enable_disable(rdev, 0, true); + + for (i = 1; i < 8; i++) + trinity_power_level_enable_disable(rdev, i, false); +} + +static void trinity_setup_uvd_clock_table(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct trinity_ps *ps = trinity_get_ps(rps); + u32 uvdstates = (ps->vclk_low_divider | + ps->vclk_high_divider << 8 | + ps->dclk_low_divider << 16 | + ps->dclk_high_divider << 24); + + WREG32_SMC(SMU_UVD_DPM_STATES, uvdstates); +} + +static void trinity_setup_uvd_dpm_interval(struct radeon_device *rdev, + u32 interval) +{ + u32 p, u; + u32 tp = RREG32_SMC(PM_TP); + u32 val; + u32 xclk = radeon_get_xclk(rdev); + + r600_calculate_u_and_p(interval, xclk, 16, &p, &u); + + val = (p + tp - 1) / tp; + + WREG32_SMC(SMU_UVD_DPM_CNTL, val); +} + +static bool trinity_uvd_clocks_zero(struct radeon_ps *rps) +{ + if ((rps->vclk == 0) && (rps->dclk == 0)) + return true; + else + return false; +} + +static bool trinity_uvd_clocks_equal(struct radeon_ps *rps1, + struct radeon_ps *rps2) +{ + struct trinity_ps *ps1 = trinity_get_ps(rps1); + struct trinity_ps *ps2 = trinity_get_ps(rps2); + + if ((rps1->vclk == rps2->vclk) && + (rps1->dclk == rps2->dclk) && + (ps1->vclk_low_divider == ps2->vclk_low_divider) && + (ps1->vclk_high_divider == ps2->vclk_high_divider) && + (ps1->dclk_low_divider == ps2->dclk_low_divider) && + (ps1->dclk_high_divider == ps2->dclk_high_divider)) + return true; + else + return false; +} + +static void trinity_setup_uvd_clocks(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + if (pi->enable_gfx_power_gating) { + trinity_gfx_powergating_enable(rdev, false); + } + + if (pi->uvd_dpm) { + if (trinity_uvd_clocks_zero(new_rps) && + !trinity_uvd_clocks_zero(old_rps)) { + trinity_setup_uvd_dpm_interval(rdev, 0); + } else if (!trinity_uvd_clocks_zero(new_rps)) { + trinity_setup_uvd_clock_table(rdev, new_rps); + + if (trinity_uvd_clocks_zero(old_rps)) { + u32 tmp = RREG32(CG_MISC_REG); + tmp &= 0xfffffffd; + WREG32(CG_MISC_REG, tmp); + + radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk); + + trinity_setup_uvd_dpm_interval(rdev, 3000); + } + } + trinity_uvd_dpm_config(rdev); + } else { + if (trinity_uvd_clocks_zero(new_rps) || + trinity_uvd_clocks_equal(new_rps, old_rps)) + return; + + radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk); + } + + if (pi->enable_gfx_power_gating) { + trinity_gfx_powergating_enable(rdev, true); + } +} + +static void trinity_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct trinity_ps *new_ps = trinity_get_ps(new_rps); + struct trinity_ps *current_ps = trinity_get_ps(new_rps); + + if (new_ps->levels[new_ps->num_levels - 1].sclk >= + current_ps->levels[current_ps->num_levels - 1].sclk) + return; + + trinity_setup_uvd_clocks(rdev, new_rps, old_rps); +} + +static void trinity_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct trinity_ps *new_ps = trinity_get_ps(new_rps); + struct trinity_ps *current_ps = trinity_get_ps(old_rps); + + if (new_ps->levels[new_ps->num_levels - 1].sclk < + current_ps->levels[current_ps->num_levels - 1].sclk) + return; + + trinity_setup_uvd_clocks(rdev, new_rps, old_rps); +} + +static void trinity_set_vce_clock(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + if ((old_rps->evclk != new_rps->evclk) || + (old_rps->ecclk != new_rps->ecclk)) { + /* turn the clocks on when encoding, off otherwise */ + if (new_rps->evclk || new_rps->ecclk) + vce_v1_0_enable_mgcg(rdev, false); + else + vce_v1_0_enable_mgcg(rdev, true); + radeon_set_vce_clocks(rdev, new_rps->evclk, new_rps->ecclk); + } +} + +static void trinity_program_ttt(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 value = RREG32_SMC(SMU_SCLK_DPM_TTT); + + value &= ~(HT_MASK | LT_MASK); + value |= HT((pi->thermal_auto_throttling + 49) * 8); + value |= LT((pi->thermal_auto_throttling + 49 - pi->sys_info.htc_hyst_lmt) * 8); + WREG32_SMC(SMU_SCLK_DPM_TTT, value); +} + +static void trinity_enable_att(struct radeon_device *rdev) +{ + u32 value = RREG32_SMC(SMU_SCLK_DPM_TT_CNTL); + + value &= ~SCLK_TT_EN_MASK; + value |= SCLK_TT_EN(1); + WREG32_SMC(SMU_SCLK_DPM_TT_CNTL, value); +} + +static void trinity_program_sclk_dpm(struct radeon_device *rdev) +{ + u32 p, u; + u32 tp = RREG32_SMC(PM_TP); + u32 ni; + u32 xclk = radeon_get_xclk(rdev); + u32 value; + + r600_calculate_u_and_p(400, xclk, 16, &p, &u); + + ni = (p + tp - 1) / tp; + + value = RREG32_SMC(PM_I_CNTL_1); + value &= ~SCLK_DPM_MASK; + value |= SCLK_DPM(ni); + WREG32_SMC(PM_I_CNTL_1, value); +} + +static int trinity_set_thermal_temperature_range(struct radeon_device *rdev, + int min_temp, int max_temp) +{ + int low_temp = 0 * 1000; + int high_temp = 255 * 1000; + + if (low_temp < min_temp) + low_temp = min_temp; + if (high_temp > max_temp) + high_temp = max_temp; + if (high_temp < low_temp) { + DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp); + return -EINVAL; + } + + WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK); + WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK); + + rdev->pm.dpm.thermal.min_temp = low_temp; + rdev->pm.dpm.thermal.max_temp = high_temp; + + return 0; +} + +static void trinity_update_current_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct trinity_ps *new_ps = trinity_get_ps(rps); + struct trinity_power_info *pi = trinity_get_pi(rdev); + + pi->current_rps = *rps; + pi->current_ps = *new_ps; + pi->current_rps.ps_priv = &pi->current_ps; +} + +static void trinity_update_requested_ps(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct trinity_ps *new_ps = trinity_get_ps(rps); + struct trinity_power_info *pi = trinity_get_pi(rdev); + + pi->requested_rps = *rps; + pi->requested_ps = *new_ps; + pi->requested_rps.ps_priv = &pi->requested_ps; +} + +void trinity_dpm_enable_bapm(struct radeon_device *rdev, bool enable) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + if (pi->enable_bapm) { + trinity_acquire_mutex(rdev); + trinity_dpm_bapm_enable(rdev, enable); + trinity_release_mutex(rdev); + } +} + +int trinity_dpm_enable(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + trinity_acquire_mutex(rdev); + + if (trinity_dpm_enabled(rdev)) { + trinity_release_mutex(rdev); + return -EINVAL; + } + + trinity_program_bootup_state(rdev); + sumo_program_vc(rdev, 0x00C00033); + trinity_start_am(rdev); + if (pi->enable_auto_thermal_throttling) { + trinity_program_ttt(rdev); + trinity_enable_att(rdev); + } + trinity_program_sclk_dpm(rdev); + trinity_start_dpm(rdev); + trinity_wait_for_dpm_enabled(rdev); + trinity_dpm_bapm_enable(rdev, false); + trinity_release_mutex(rdev); + + trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps); + + return 0; +} + +int trinity_dpm_late_enable(struct radeon_device *rdev) +{ + int ret; + + trinity_acquire_mutex(rdev); + trinity_enable_clock_power_gating(rdev); + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + ret = trinity_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) { + trinity_release_mutex(rdev); + return ret; + } + rdev->irq.dpm_thermal = true; + radeon_irq_set(rdev); + } + trinity_release_mutex(rdev); + + return 0; +} + +void trinity_dpm_disable(struct radeon_device *rdev) +{ + trinity_acquire_mutex(rdev); + if (!trinity_dpm_enabled(rdev)) { + trinity_release_mutex(rdev); + return; + } + trinity_dpm_bapm_enable(rdev, false); + trinity_disable_clock_power_gating(rdev); + sumo_clear_vc(rdev); + trinity_wait_for_level_0(rdev); + trinity_stop_dpm(rdev); + trinity_reset_am(rdev); + trinity_release_mutex(rdev); + + if (rdev->irq.installed && + r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) { + rdev->irq.dpm_thermal = false; + radeon_irq_set(rdev); + } + + trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps); +} + +static void trinity_get_min_sclk_divider(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + pi->min_sclk_did = + (RREG32_SMC(CC_SMU_MISC_FUSES) & MinSClkDid_MASK) >> MinSClkDid_SHIFT; +} + +static void trinity_setup_nbp_sim(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct trinity_ps *new_ps = trinity_get_ps(rps); + u32 nbpsconfig; + + if (pi->sys_info.nb_dpm_enable) { + nbpsconfig = RREG32_SMC(NB_PSTATE_CONFIG); + nbpsconfig &= ~(Dpm0PgNbPsLo_MASK | Dpm0PgNbPsHi_MASK | DpmXNbPsLo_MASK | DpmXNbPsHi_MASK); + nbpsconfig |= (Dpm0PgNbPsLo(new_ps->Dpm0PgNbPsLo) | + Dpm0PgNbPsHi(new_ps->Dpm0PgNbPsHi) | + DpmXNbPsLo(new_ps->DpmXNbPsLo) | + DpmXNbPsHi(new_ps->DpmXNbPsHi)); + WREG32_SMC(NB_PSTATE_CONFIG, nbpsconfig); + } +} + +int trinity_dpm_force_performance_level(struct radeon_device *rdev, + enum radeon_dpm_forced_level level) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct radeon_ps *rps = &pi->current_rps; + struct trinity_ps *ps = trinity_get_ps(rps); + int i, ret; + + if (ps->num_levels <= 1) + return 0; + + if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + /* not supported by the hw */ + return -EINVAL; + } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { + ret = trinity_dpm_n_levels_disabled(rdev, ps->num_levels - 1); + if (ret) + return ret; + } else { + for (i = 0; i < ps->num_levels; i++) { + ret = trinity_dpm_n_levels_disabled(rdev, 0); + if (ret) + return ret; + } + } + + rdev->pm.dpm.forced_level = level; + + return 0; +} + +int trinity_dpm_pre_set_power_state(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps; + struct radeon_ps *new_ps = &requested_ps; + + trinity_update_requested_ps(rdev, new_ps); + + trinity_apply_state_adjust_rules(rdev, + &pi->requested_rps, + &pi->current_rps); + + return 0; +} + +int trinity_dpm_set_power_state(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct radeon_ps *new_ps = &pi->requested_rps; + struct radeon_ps *old_ps = &pi->current_rps; + + trinity_acquire_mutex(rdev); + if (pi->enable_dpm) { + if (pi->enable_bapm) + trinity_dpm_bapm_enable(rdev, rdev->pm.dpm.ac_power); + trinity_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps); + trinity_enable_power_level_0(rdev); + trinity_force_level_0(rdev); + trinity_wait_for_level_0(rdev); + trinity_setup_nbp_sim(rdev, new_ps); + trinity_program_power_levels_0_to_n(rdev, new_ps, old_ps); + trinity_force_level_0(rdev); + trinity_unforce_levels(rdev); + trinity_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps); + trinity_set_vce_clock(rdev, new_ps, old_ps); + } + trinity_release_mutex(rdev); + + return 0; +} + +void trinity_dpm_post_set_power_state(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct radeon_ps *new_ps = &pi->requested_rps; + + trinity_update_current_ps(rdev, new_ps); +} + +void trinity_dpm_setup_asic(struct radeon_device *rdev) +{ + trinity_acquire_mutex(rdev); + sumo_program_sstp(rdev); + sumo_take_smu_control(rdev, true); + trinity_get_min_sclk_divider(rdev); + trinity_release_mutex(rdev); +} + +#if 0 +void trinity_dpm_reset_asic(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + trinity_acquire_mutex(rdev); + if (pi->enable_dpm) { + trinity_enable_power_level_0(rdev); + trinity_force_level_0(rdev); + trinity_wait_for_level_0(rdev); + trinity_program_bootup_state(rdev); + trinity_force_level_0(rdev); + trinity_unforce_levels(rdev); + } + trinity_release_mutex(rdev); +} +#endif + +static u16 trinity_convert_voltage_index_to_value(struct radeon_device *rdev, + u32 vid_2bit) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit); + u32 svi_mode = (RREG32_SMC(PM_CONFIG) & SVI_Mode) ? 1 : 0; + u32 step = (svi_mode == 0) ? 1250 : 625; + u32 delta = vid_7bit * step + 50; + + if (delta > 155000) + return 0; + + return (155000 - delta) / 100; +} + +static void trinity_patch_boot_state(struct radeon_device *rdev, + struct trinity_ps *ps) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + ps->num_levels = 1; + ps->nbps_flags = 0; + ps->bapm_flags = 0; + ps->levels[0] = pi->boot_pl; +} + +static u8 trinity_calculate_vce_wm(struct radeon_device *rdev, u32 sclk) +{ + if (sclk < 20000) + return 1; + return 0; +} + +static void trinity_construct_boot_state(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + pi->boot_pl.sclk = pi->sys_info.bootup_sclk; + pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index; + pi->boot_pl.ds_divider_index = 0; + pi->boot_pl.ss_divider_index = 0; + pi->boot_pl.allow_gnb_slow = 1; + pi->boot_pl.force_nbp_state = 0; + pi->boot_pl.display_wm = 0; + pi->boot_pl.vce_wm = 0; + pi->current_ps.num_levels = 1; + pi->current_ps.levels[0] = pi->boot_pl; +} + +static u8 trinity_get_sleep_divider_id_from_clock(struct radeon_device *rdev, + u32 sclk, u32 min_sclk_in_sr) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 i; + u32 temp; + u32 min = (min_sclk_in_sr > TRINITY_MINIMUM_ENGINE_CLOCK) ? + min_sclk_in_sr : TRINITY_MINIMUM_ENGINE_CLOCK; + + if (sclk < min) + return 0; + + if (!pi->enable_sclk_ds) + return 0; + + for (i = TRINITY_MAX_DEEPSLEEP_DIVIDER_ID; ; i--) { + temp = sclk / sumo_get_sleep_divider_from_id(i); + if (temp >= min || i == 0) + break; + } + + return (u8)i; +} + +static u32 trinity_get_valid_engine_clock(struct radeon_device *rdev, + u32 lower_limit) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 i; + + for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) { + if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit) + return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency; + } + + if (i == pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries) + DRM_ERROR("engine clock out of range!"); + + return 0; +} + +static void trinity_patch_thermal_state(struct radeon_device *rdev, + struct trinity_ps *ps, + struct trinity_ps *current_ps) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */ + u32 current_vddc; + u32 current_sclk; + u32 current_index = 0; + + if (current_ps) { + current_vddc = current_ps->levels[current_index].vddc_index; + current_sclk = current_ps->levels[current_index].sclk; + } else { + current_vddc = pi->boot_pl.vddc_index; + current_sclk = pi->boot_pl.sclk; + } + + ps->levels[0].vddc_index = current_vddc; + + if (ps->levels[0].sclk > current_sclk) + ps->levels[0].sclk = current_sclk; + + ps->levels[0].ds_divider_index = + trinity_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr); + ps->levels[0].ss_divider_index = ps->levels[0].ds_divider_index; + ps->levels[0].allow_gnb_slow = 1; + ps->levels[0].force_nbp_state = 0; + ps->levels[0].display_wm = 0; + ps->levels[0].vce_wm = + trinity_calculate_vce_wm(rdev, ps->levels[0].sclk); +} + +static u8 trinity_calculate_display_wm(struct radeon_device *rdev, + struct trinity_ps *ps, u32 index) +{ + if (ps == NULL || ps->num_levels <= 1) + return 0; + else if (ps->num_levels == 2) { + if (index == 0) + return 0; + else + return 1; + } else { + if (index == 0) + return 0; + else if (ps->levels[index].sclk < 30000) + return 0; + else + return 1; + } +} + +static u32 trinity_get_uvd_clock_index(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 i = 0; + + for (i = 0; i < 4; i++) { + if ((rps->vclk == pi->sys_info.uvd_clock_table_entries[i].vclk) && + (rps->dclk == pi->sys_info.uvd_clock_table_entries[i].dclk)) + break; + } + + if (i >= 4) { + DRM_ERROR("UVD clock index not found!\n"); + i = 3; + } + return i; +} + +static void trinity_adjust_uvd_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + struct trinity_ps *ps = trinity_get_ps(rps); + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 high_index = 0; + u32 low_index = 0; + + if (pi->uvd_dpm && r600_is_uvd_state(rps->class, rps->class2)) { + high_index = trinity_get_uvd_clock_index(rdev, rps); + + switch(high_index) { + case 3: + case 2: + low_index = 1; + break; + case 1: + case 0: + default: + low_index = 0; + break; + } + + ps->vclk_low_divider = + pi->sys_info.uvd_clock_table_entries[high_index].vclk_did; + ps->dclk_low_divider = + pi->sys_info.uvd_clock_table_entries[high_index].dclk_did; + ps->vclk_high_divider = + pi->sys_info.uvd_clock_table_entries[low_index].vclk_did; + ps->dclk_high_divider = + pi->sys_info.uvd_clock_table_entries[low_index].dclk_did; + } +} + +static int trinity_get_vce_clock_voltage(struct radeon_device *rdev, + u32 evclk, u32 ecclk, u16 *voltage) +{ + u32 i; + int ret = -EINVAL; + struct radeon_vce_clock_voltage_dependency_table *table = + &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table; + + if (((evclk == 0) && (ecclk == 0)) || + (table && (table->count == 0))) { + *voltage = 0; + return 0; + } + + for (i = 0; i < table->count; i++) { + if ((evclk <= table->entries[i].evclk) && + (ecclk <= table->entries[i].ecclk)) { + *voltage = table->entries[i].v; + ret = 0; + break; + } + } + + /* if no match return the highest voltage */ + if (ret) + *voltage = table->entries[table->count - 1].v; + + return ret; +} + +static void trinity_apply_state_adjust_rules(struct radeon_device *rdev, + struct radeon_ps *new_rps, + struct radeon_ps *old_rps) +{ + struct trinity_ps *ps = trinity_get_ps(new_rps); + struct trinity_ps *current_ps = trinity_get_ps(old_rps); + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 min_voltage = 0; /* ??? */ + u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */ + u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */ + u32 i; + u16 min_vce_voltage; + bool force_high; + u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count; + + if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL) + return trinity_patch_thermal_state(rdev, ps, current_ps); + + trinity_adjust_uvd_state(rdev, new_rps); + + if (new_rps->vce_active) { + new_rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk; + new_rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk; + } else { + new_rps->evclk = 0; + new_rps->ecclk = 0; + } + + for (i = 0; i < ps->num_levels; i++) { + if (ps->levels[i].vddc_index < min_voltage) + ps->levels[i].vddc_index = min_voltage; + + if (ps->levels[i].sclk < min_sclk) + ps->levels[i].sclk = + trinity_get_valid_engine_clock(rdev, min_sclk); + + /* patch in vce limits */ + if (new_rps->vce_active) { + /* sclk */ + if (ps->levels[i].sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk) + ps->levels[i].sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk; + /* vddc */ + trinity_get_vce_clock_voltage(rdev, new_rps->evclk, new_rps->ecclk, &min_vce_voltage); + if (ps->levels[i].vddc_index < min_vce_voltage) + ps->levels[i].vddc_index = min_vce_voltage; + } + + ps->levels[i].ds_divider_index = + sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr); + + ps->levels[i].ss_divider_index = ps->levels[i].ds_divider_index; + + ps->levels[i].allow_gnb_slow = 1; + ps->levels[i].force_nbp_state = 0; + ps->levels[i].display_wm = + trinity_calculate_display_wm(rdev, ps, i); + ps->levels[i].vce_wm = + trinity_calculate_vce_wm(rdev, ps->levels[0].sclk); + } + + if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) || + ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)) + ps->bapm_flags |= TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE; + + if (pi->sys_info.nb_dpm_enable) { + ps->Dpm0PgNbPsLo = 0x1; + ps->Dpm0PgNbPsHi = 0x0; + ps->DpmXNbPsLo = 0x2; + ps->DpmXNbPsHi = 0x1; + + if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) || + ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)) { + force_high = ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) || + ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) && + (pi->sys_info.uma_channel_number == 1))); + force_high = (num_active_displays >= 3) || force_high; + ps->Dpm0PgNbPsLo = force_high ? 0x2 : 0x3; + ps->Dpm0PgNbPsHi = 0x1; + ps->DpmXNbPsLo = force_high ? 0x2 : 0x3; + ps->DpmXNbPsHi = 0x2; + ps->levels[ps->num_levels - 1].allow_gnb_slow = 0; + } + } +} + +static void trinity_cleanup_asic(struct radeon_device *rdev) +{ + sumo_take_smu_control(rdev, false); +} + +#if 0 +static void trinity_pre_display_configuration_change(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + if (pi->voltage_drop_in_dce) + trinity_dce_enable_voltage_adjustment(rdev, false); +} +#endif + +static void trinity_add_dccac_value(struct radeon_device *rdev) +{ + u32 gpu_cac_avrg_cntl_window_size; + u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count; + u64 disp_clk = rdev->clock.default_dispclk / 100; + u32 dc_cac_value; + + gpu_cac_avrg_cntl_window_size = + (RREG32_SMC(GPU_CAC_AVRG_CNTL) & WINDOW_SIZE_MASK) >> WINDOW_SIZE_SHIFT; + + dc_cac_value = (u32)((14213 * disp_clk * disp_clk * (u64)num_active_displays) >> + (32 - gpu_cac_avrg_cntl_window_size)); + + WREG32_SMC(DC_CAC_VALUE, dc_cac_value); +} + +void trinity_dpm_display_configuration_changed(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + if (pi->voltage_drop_in_dce) + trinity_dce_enable_voltage_adjustment(rdev, true); + trinity_add_dccac_value(rdev); +} + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; +}; + +union pplib_clock_info { + struct _ATOM_PPLIB_R600_CLOCK_INFO r600; + struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780; + struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen; + struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo; +}; + +union pplib_power_state { + struct _ATOM_PPLIB_STATE v1; + struct _ATOM_PPLIB_STATE_V2 v2; +}; + +static void trinity_parse_pplib_non_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info, + u8 table_rev) +{ + struct trinity_ps *ps = trinity_get_ps(rps); + + rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings); + rps->class = le16_to_cpu(non_clock_info->usClassification); + rps->class2 = le16_to_cpu(non_clock_info->usClassification2); + + if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) { + rps->vclk = le32_to_cpu(non_clock_info->ulVCLK); + rps->dclk = le32_to_cpu(non_clock_info->ulDCLK); + } else { + rps->vclk = 0; + rps->dclk = 0; + } + + if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) { + rdev->pm.dpm.boot_ps = rps; + trinity_patch_boot_state(rdev, ps); + } + if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + rdev->pm.dpm.uvd_ps = rps; +} + +static void trinity_parse_pplib_clock_info(struct radeon_device *rdev, + struct radeon_ps *rps, int index, + union pplib_clock_info *clock_info) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct trinity_ps *ps = trinity_get_ps(rps); + struct trinity_pl *pl = &ps->levels[index]; + u32 sclk; + + sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow); + sclk |= clock_info->sumo.ucEngineClockHigh << 16; + pl->sclk = sclk; + pl->vddc_index = clock_info->sumo.vddcIndex; + + ps->num_levels = index + 1; + + if (pi->enable_sclk_ds) { + pl->ds_divider_index = 5; + pl->ss_divider_index = 5; + } +} + +static int trinity_parse_power_table(struct radeon_device *rdev) +{ + struct radeon_mode_info *mode_info = &rdev->mode_info; + struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info; + union pplib_power_state *power_state; + int i, j, k, non_clock_array_index, clock_array_index; + union pplib_clock_info *clock_info; + struct _StateArray *state_array; + struct _ClockInfoArray *clock_info_array; + struct _NonClockInfoArray *non_clock_info_array; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + u8 *power_state_offset; + struct sumo_ps *ps; + + if (!atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + state_array = (struct _StateArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usStateArrayOffset)); + clock_info_array = (struct _ClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usClockInfoArrayOffset)); + non_clock_info_array = (struct _NonClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset)); + + rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) * + state_array->ucNumEntries, GFP_KERNEL); + if (!rdev->pm.dpm.ps) + return -ENOMEM; + power_state_offset = (u8 *)state_array->states; + for (i = 0; i < state_array->ucNumEntries; i++) { + u8 *idx; + power_state = (union pplib_power_state *)power_state_offset; + non_clock_array_index = power_state->v2.nonClockInfoIndex; + non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) + &non_clock_info_array->nonClockInfo[non_clock_array_index]; + if (!rdev->pm.power_state[i].clock_info) + return -EINVAL; + ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL); + if (ps == NULL) { + kfree(rdev->pm.dpm.ps); + return -ENOMEM; + } + rdev->pm.dpm.ps[i].ps_priv = ps; + k = 0; + idx = (u8 *)&power_state->v2.clockInfoIndex[0]; + for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) { + clock_array_index = idx[j]; + if (clock_array_index >= clock_info_array->ucNumEntries) + continue; + if (k >= SUMO_MAX_HARDWARE_POWERLEVELS) + break; + clock_info = (union pplib_clock_info *) + ((u8 *)&clock_info_array->clockInfo[0] + + (clock_array_index * clock_info_array->ucEntrySize)); + trinity_parse_pplib_clock_info(rdev, + &rdev->pm.dpm.ps[i], k, + clock_info); + k++; + } + trinity_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i], + non_clock_info, + non_clock_info_array->ucEntrySize); + power_state_offset += 2 + power_state->v2.ucNumDPMLevels; + } + rdev->pm.dpm.num_ps = state_array->ucNumEntries; + + /* fill in the vce power states */ + for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) { + u32 sclk; + clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx; + clock_info = (union pplib_clock_info *) + &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize]; + sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow); + sclk |= clock_info->sumo.ucEngineClockHigh << 16; + rdev->pm.dpm.vce_states[i].sclk = sclk; + rdev->pm.dpm.vce_states[i].mclk = 0; + } + + return 0; +} + +union igp_info { + struct _ATOM_INTEGRATED_SYSTEM_INFO info; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7; +}; + +static u32 trinity_convert_did_to_freq(struct radeon_device *rdev, u8 did) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + u32 divider; + + if (did >= 8 && did <= 0x3f) + divider = did * 25; + else if (did > 0x3f && did <= 0x5f) + divider = (did - 64) * 50 + 1600; + else if (did > 0x5f && did <= 0x7e) + divider = (did - 96) * 100 + 3200; + else if (did == 0x7f) + divider = 128 * 100; + else + return 10000; + + return ((pi->sys_info.dentist_vco_freq * 100) + (divider - 1)) / divider; +} + +static int trinity_parse_sys_info_table(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct radeon_mode_info *mode_info = &rdev->mode_info; + int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo); + union igp_info *igp_info; + u8 frev, crev; + u16 data_offset; + int i; + + if (atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + igp_info = (union igp_info *)(mode_info->atom_context->bios + + data_offset); + + if (crev != 7) { + DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev); + return -EINVAL; + } + pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_7.ulBootUpEngineClock); + pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_7.ulMinEngineClock); + pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_7.ulBootUpUMAClock); + pi->sys_info.dentist_vco_freq = le32_to_cpu(igp_info->info_7.ulDentistVCOFreq); + pi->sys_info.bootup_nb_voltage_index = + le16_to_cpu(igp_info->info_7.usBootUpNBVoltage); + if (igp_info->info_7.ucHtcTmpLmt == 0) + pi->sys_info.htc_tmp_lmt = 203; + else + pi->sys_info.htc_tmp_lmt = igp_info->info_7.ucHtcTmpLmt; + if (igp_info->info_7.ucHtcHystLmt == 0) + pi->sys_info.htc_hyst_lmt = 5; + else + pi->sys_info.htc_hyst_lmt = igp_info->info_7.ucHtcHystLmt; + if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) { + DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n"); + } + + if (pi->enable_nbps_policy) + pi->sys_info.nb_dpm_enable = igp_info->info_7.ucNBDPMEnable; + else + pi->sys_info.nb_dpm_enable = 0; + + for (i = 0; i < TRINITY_NUM_NBPSTATES; i++) { + pi->sys_info.nbp_mclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateMemclkFreq[i]); + pi->sys_info.nbp_nclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateNClkFreq[i]); + } + + pi->sys_info.nbp_voltage_index[0] = le16_to_cpu(igp_info->info_7.usNBP0Voltage); + pi->sys_info.nbp_voltage_index[1] = le16_to_cpu(igp_info->info_7.usNBP1Voltage); + pi->sys_info.nbp_voltage_index[2] = le16_to_cpu(igp_info->info_7.usNBP2Voltage); + pi->sys_info.nbp_voltage_index[3] = le16_to_cpu(igp_info->info_7.usNBP3Voltage); + + if (!pi->sys_info.nb_dpm_enable) { + for (i = 1; i < TRINITY_NUM_NBPSTATES; i++) { + pi->sys_info.nbp_mclk[i] = pi->sys_info.nbp_mclk[0]; + pi->sys_info.nbp_nclk[i] = pi->sys_info.nbp_nclk[0]; + pi->sys_info.nbp_voltage_index[i] = pi->sys_info.nbp_voltage_index[0]; + } + } + + pi->sys_info.uma_channel_number = igp_info->info_7.ucUMAChannelNumber; + + sumo_construct_sclk_voltage_mapping_table(rdev, + &pi->sys_info.sclk_voltage_mapping_table, + igp_info->info_7.sAvail_SCLK); + sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table, + igp_info->info_7.sAvail_SCLK); + + pi->sys_info.uvd_clock_table_entries[0].vclk_did = + igp_info->info_7.ucDPMState0VclkFid; + pi->sys_info.uvd_clock_table_entries[1].vclk_did = + igp_info->info_7.ucDPMState1VclkFid; + pi->sys_info.uvd_clock_table_entries[2].vclk_did = + igp_info->info_7.ucDPMState2VclkFid; + pi->sys_info.uvd_clock_table_entries[3].vclk_did = + igp_info->info_7.ucDPMState3VclkFid; + + pi->sys_info.uvd_clock_table_entries[0].dclk_did = + igp_info->info_7.ucDPMState0DclkFid; + pi->sys_info.uvd_clock_table_entries[1].dclk_did = + igp_info->info_7.ucDPMState1DclkFid; + pi->sys_info.uvd_clock_table_entries[2].dclk_did = + igp_info->info_7.ucDPMState2DclkFid; + pi->sys_info.uvd_clock_table_entries[3].dclk_did = + igp_info->info_7.ucDPMState3DclkFid; + + for (i = 0; i < 4; i++) { + pi->sys_info.uvd_clock_table_entries[i].vclk = + trinity_convert_did_to_freq(rdev, + pi->sys_info.uvd_clock_table_entries[i].vclk_did); + pi->sys_info.uvd_clock_table_entries[i].dclk = + trinity_convert_did_to_freq(rdev, + pi->sys_info.uvd_clock_table_entries[i].dclk_did); + } + + + + } + return 0; +} + +int trinity_dpm_init(struct radeon_device *rdev) +{ + struct trinity_power_info *pi; + int ret, i; + + pi = kzalloc(sizeof(struct trinity_power_info), GFP_KERNEL); + if (pi == NULL) + return -ENOMEM; + rdev->pm.dpm.priv = pi; + + for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++) + pi->at[i] = TRINITY_AT_DFLT; + + if (radeon_bapm == -1) { + /* There are stability issues reported on with + * bapm enabled when switching between AC and battery + * power. At the same time, some MSI boards hang + * if it's not enabled and dpm is enabled. Just enable + * it for MSI boards right now. + */ + if (rdev->pdev->subsystem_vendor == 0x1462) + pi->enable_bapm = true; + else + pi->enable_bapm = false; + } else if (radeon_bapm == 0) { + pi->enable_bapm = false; + } else { + pi->enable_bapm = true; + } + pi->enable_nbps_policy = true; + pi->enable_sclk_ds = true; + pi->enable_gfx_power_gating = true; + pi->enable_gfx_clock_gating = true; + pi->enable_mg_clock_gating = false; + pi->enable_gfx_dynamic_mgpg = false; + pi->override_dynamic_mgpg = false; + pi->enable_auto_thermal_throttling = true; + pi->voltage_drop_in_dce = false; /* need to restructure dpm/modeset interaction */ + pi->uvd_dpm = true; /* ??? */ + + ret = trinity_parse_sys_info_table(rdev); + if (ret) + return ret; + + trinity_construct_boot_state(rdev); + + ret = r600_get_platform_caps(rdev); + if (ret) + return ret; + + ret = r600_parse_extended_power_table(rdev); + if (ret) + return ret; + + ret = trinity_parse_power_table(rdev); + if (ret) + return ret; + + pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt; + pi->enable_dpm = true; + + return 0; +} + +void trinity_dpm_print_power_state(struct radeon_device *rdev, + struct radeon_ps *rps) +{ + int i; + struct trinity_ps *ps = trinity_get_ps(rps); + + r600_dpm_print_class_info(rps->class, rps->class2); + r600_dpm_print_cap_info(rps->caps); + printk("\tuvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + for (i = 0; i < ps->num_levels; i++) { + struct trinity_pl *pl = &ps->levels[i]; + printk("\t\tpower level %d sclk: %u vddc: %u\n", + i, pl->sclk, + trinity_convert_voltage_index_to_value(rdev, pl->vddc_index)); + } + r600_dpm_print_ps_status(rdev, rps); +} + +void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev, + struct seq_file *m) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct radeon_ps *rps = &pi->current_rps; + struct trinity_ps *ps = trinity_get_ps(rps); + struct trinity_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >> + CURRENT_STATE_SHIFT; + + if (current_index >= ps->num_levels) { + seq_printf(m, "invalid dpm profile %d\n", current_index); + } else { + pl = &ps->levels[current_index]; + seq_printf(m, "uvd vclk: %d dclk: %d\n", rps->vclk, rps->dclk); + seq_printf(m, "power level %d sclk: %u vddc: %u\n", + current_index, pl->sclk, + trinity_convert_voltage_index_to_value(rdev, pl->vddc_index)); + } +} + +u32 trinity_dpm_get_current_sclk(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct radeon_ps *rps = &pi->current_rps; + struct trinity_ps *ps = trinity_get_ps(rps); + struct trinity_pl *pl; + u32 current_index = + (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >> + CURRENT_STATE_SHIFT; + + if (current_index >= ps->num_levels) { + return 0; + } else { + pl = &ps->levels[current_index]; + return pl->sclk; + } +} + +u32 trinity_dpm_get_current_mclk(struct radeon_device *rdev) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + return pi->sys_info.bootup_uma_clk; +} + +void trinity_dpm_fini(struct radeon_device *rdev) +{ + int i; + + trinity_cleanup_asic(rdev); /* ??? */ + + for (i = 0; i < rdev->pm.dpm.num_ps; i++) { + kfree(rdev->pm.dpm.ps[i].ps_priv); + } + kfree(rdev->pm.dpm.ps); + kfree(rdev->pm.dpm.priv); + r600_free_extended_power_table(rdev); +} + +u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + struct trinity_ps *requested_state = trinity_get_ps(&pi->requested_rps); + + if (low) + return requested_state->levels[0].sclk; + else + return requested_state->levels[requested_state->num_levels - 1].sclk; +} + +u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low) +{ + struct trinity_power_info *pi = trinity_get_pi(rdev); + + return pi->sys_info.bootup_uma_clk; +} diff --git a/sys/dev/pci/drm/radeon/trinity_dpm.h b/sys/dev/pci/drm/radeon/trinity_dpm.h new file mode 100644 index 00000000000..c261657750c --- /dev/null +++ b/sys/dev/pci/drm/radeon/trinity_dpm.h @@ -0,0 +1,134 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __TRINITY_DPM_H__ +#define __TRINITY_DPM_H__ + +#include "sumo_dpm.h" + +#define TRINITY_SIZEOF_DPM_STATE_TABLE (SMU_SCLK_DPM_STATE_1_CNTL_0 - SMU_SCLK_DPM_STATE_0_CNTL_0) + +struct trinity_pl { + u32 sclk; + u8 vddc_index; + u8 ds_divider_index; + u8 ss_divider_index; + u8 allow_gnb_slow; + u8 force_nbp_state; + u8 display_wm; + u8 vce_wm; +}; + +#define TRINITY_POWERSTATE_FLAGS_NBPS_FORCEHIGH (1 << 0) +#define TRINITY_POWERSTATE_FLAGS_NBPS_LOCKTOHIGH (1 << 1) +#define TRINITY_POWERSTATE_FLAGS_NBPS_LOCKTOLOW (1 << 2) + +#define TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE (1 << 0) + +struct trinity_ps { + u32 num_levels; + struct trinity_pl levels[SUMO_MAX_HARDWARE_POWERLEVELS]; + + u32 nbps_flags; + u32 bapm_flags; + + u8 Dpm0PgNbPsLo; + u8 Dpm0PgNbPsHi; + u8 DpmXNbPsLo; + u8 DpmXNbPsHi; + + u32 vclk_low_divider; + u32 vclk_high_divider; + u32 dclk_low_divider; + u32 dclk_high_divider; +}; + +#define TRINITY_NUM_NBPSTATES 4 + +struct trinity_uvd_clock_table_entry +{ + u32 vclk; + u32 dclk; + u8 vclk_did; + u8 dclk_did; + u8 rsv[2]; +}; + +struct trinity_sys_info { + u32 bootup_uma_clk; + u32 bootup_sclk; + u32 min_sclk; + u32 dentist_vco_freq; + u32 nb_dpm_enable; + u32 nbp_mclk[TRINITY_NUM_NBPSTATES]; + u32 nbp_nclk[TRINITY_NUM_NBPSTATES]; + u16 nbp_voltage_index[TRINITY_NUM_NBPSTATES]; + u16 bootup_nb_voltage_index; + u8 htc_tmp_lmt; + u8 htc_hyst_lmt; + struct sumo_sclk_voltage_mapping_table sclk_voltage_mapping_table; + struct sumo_vid_mapping_table vid_mapping_table; + u32 uma_channel_number; + struct trinity_uvd_clock_table_entry uvd_clock_table_entries[4]; +}; + +struct trinity_power_info { + u32 at[SUMO_MAX_HARDWARE_POWERLEVELS]; + u32 dpm_interval; + u32 thermal_auto_throttling; + struct trinity_sys_info sys_info; + struct trinity_pl boot_pl; + u32 min_sclk_did; + bool enable_nbps_policy; + bool voltage_drop_in_dce; + bool override_dynamic_mgpg; + bool enable_gfx_clock_gating; + bool enable_gfx_power_gating; + bool enable_mg_clock_gating; + bool enable_gfx_dynamic_mgpg; + bool enable_auto_thermal_throttling; + bool enable_dpm; + bool enable_sclk_ds; + bool enable_bapm; + bool uvd_dpm; + struct radeon_ps current_rps; + struct trinity_ps current_ps; + struct radeon_ps requested_rps; + struct trinity_ps requested_ps; +}; + +#define TRINITY_AT_DFLT 30 + +/* trinity_smc.c */ +int trinity_dpm_bapm_enable(struct radeon_device *rdev, bool enable); +int trinity_dpm_config(struct radeon_device *rdev, bool enable); +int trinity_uvd_dpm_config(struct radeon_device *rdev); +int trinity_dpm_force_state(struct radeon_device *rdev, u32 n); +int trinity_dpm_n_levels_disabled(struct radeon_device *rdev, u32 n); +int trinity_dpm_no_forced_level(struct radeon_device *rdev); +int trinity_dce_enable_voltage_adjustment(struct radeon_device *rdev, + bool enable); +int trinity_gfx_dynamic_mgpg_config(struct radeon_device *rdev); +void trinity_acquire_mutex(struct radeon_device *rdev); +void trinity_release_mutex(struct radeon_device *rdev); + +#endif diff --git a/sys/dev/pci/drm/radeon/trinity_smc.c b/sys/dev/pci/drm/radeon/trinity_smc.c new file mode 100644 index 00000000000..86013436d48 --- /dev/null +++ b/sys/dev/pci/drm/radeon/trinity_smc.c @@ -0,0 +1,127 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "radeon.h" +#include "trinityd.h" +#include "trinity_dpm.h" +#include "ppsmc.h" + +static int trinity_notify_message_to_smu(struct radeon_device *rdev, u32 id) +{ + int i; + u32 v = 0; + + WREG32(SMC_MESSAGE_0, id); + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(SMC_RESP_0) != 0) + break; + udelay(1); + } + v = RREG32(SMC_RESP_0); + + if (v != 1) { + if (v == 0xFF) { + DRM_ERROR("SMC failed to handle the message!\n"); + return -EINVAL; + } else if (v == 0xFE) { + DRM_ERROR("Unknown SMC message!\n"); + return -EINVAL; + } + } + + return 0; +} + +int trinity_dpm_bapm_enable(struct radeon_device *rdev, bool enable) +{ + if (enable) + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_EnableBAPM); + else + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DisableBAPM); +} + +int trinity_dpm_config(struct radeon_device *rdev, bool enable) +{ + if (enable) + WREG32_SMC(SMU_SCRATCH0, 1); + else + WREG32_SMC(SMU_SCRATCH0, 0); + + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DPM_Config); +} + +int trinity_dpm_force_state(struct radeon_device *rdev, u32 n) +{ + WREG32_SMC(SMU_SCRATCH0, n); + + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DPM_ForceState); +} + +int trinity_dpm_n_levels_disabled(struct radeon_device *rdev, u32 n) +{ + WREG32_SMC(SMU_SCRATCH0, n); + + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DPM_N_LevelsDisabled); +} + +int trinity_uvd_dpm_config(struct radeon_device *rdev) +{ + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_UVD_DPM_Config); +} + +int trinity_dpm_no_forced_level(struct radeon_device *rdev) +{ + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_NoForcedLevel); +} + +int trinity_dce_enable_voltage_adjustment(struct radeon_device *rdev, + bool enable) +{ + if (enable) + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DCE_AllowVoltageAdjustment); + else + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DCE_RemoveVoltageAdjustment); +} + +int trinity_gfx_dynamic_mgpg_config(struct radeon_device *rdev) +{ + return trinity_notify_message_to_smu(rdev, PPSMC_MSG_PG_SIMD_Config); +} + +void trinity_acquire_mutex(struct radeon_device *rdev) +{ + int i; + + WREG32(SMC_INT_REQ, 1); + for (i = 0; i < rdev->usec_timeout; i++) { + if ((RREG32(SMC_INT_REQ) & 0xffff) == 1) + break; + udelay(1); + } +} + +void trinity_release_mutex(struct radeon_device *rdev) +{ + WREG32(SMC_INT_REQ, 0); +} diff --git a/sys/dev/pci/drm/radeon/trinityd.h b/sys/dev/pci/drm/radeon/trinityd.h new file mode 100644 index 00000000000..fd32e277175 --- /dev/null +++ b/sys/dev/pci/drm/radeon/trinityd.h @@ -0,0 +1,228 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#ifndef _TRINITYD_H_ +#define _TRINITYD_H_ + +/* pm registers */ + +/* cg */ +#define CG_CGTT_LOCAL_0 0x0 +#define CG_CGTT_LOCAL_1 0x1 + +/* smc */ +#define SMU_SCLK_DPM_STATE_0_CNTL_0 0x1f000 +# define STATE_VALID(x) ((x) << 0) +# define STATE_VALID_MASK (0xff << 0) +# define STATE_VALID_SHIFT 0 +# define CLK_DIVIDER(x) ((x) << 8) +# define CLK_DIVIDER_MASK (0xff << 8) +# define CLK_DIVIDER_SHIFT 8 +# define VID(x) ((x) << 16) +# define VID_MASK (0xff << 16) +# define VID_SHIFT 16 +# define LVRT(x) ((x) << 24) +# define LVRT_MASK (0xff << 24) +# define LVRT_SHIFT 24 +#define SMU_SCLK_DPM_STATE_0_CNTL_1 0x1f004 +# define DS_DIV(x) ((x) << 0) +# define DS_DIV_MASK (0xff << 0) +# define DS_DIV_SHIFT 0 +# define DS_SH_DIV(x) ((x) << 8) +# define DS_SH_DIV_MASK (0xff << 8) +# define DS_SH_DIV_SHIFT 8 +# define DISPLAY_WM(x) ((x) << 16) +# define DISPLAY_WM_MASK (0xff << 16) +# define DISPLAY_WM_SHIFT 16 +# define VCE_WM(x) ((x) << 24) +# define VCE_WM_MASK (0xff << 24) +# define VCE_WM_SHIFT 24 + +#define SMU_SCLK_DPM_STATE_0_CNTL_3 0x1f00c +# define GNB_SLOW(x) ((x) << 0) +# define GNB_SLOW_MASK (0xff << 0) +# define GNB_SLOW_SHIFT 0 +# define FORCE_NBPS1(x) ((x) << 8) +# define FORCE_NBPS1_MASK (0xff << 8) +# define FORCE_NBPS1_SHIFT 8 +#define SMU_SCLK_DPM_STATE_0_AT 0x1f010 +# define AT(x) ((x) << 0) +# define AT_MASK (0xff << 0) +# define AT_SHIFT 0 + +#define SMU_SCLK_DPM_STATE_0_PG_CNTL 0x1f014 +# define PD_SCLK_DIVIDER(x) ((x) << 16) +# define PD_SCLK_DIVIDER_MASK (0xff << 16) +# define PD_SCLK_DIVIDER_SHIFT 16 + +#define SMU_SCLK_DPM_STATE_1_CNTL_0 0x1f020 + +#define SMU_SCLK_DPM_CNTL 0x1f100 +# define SCLK_DPM_EN(x) ((x) << 0) +# define SCLK_DPM_EN_MASK (0xff << 0) +# define SCLK_DPM_EN_SHIFT 0 +# define SCLK_DPM_BOOT_STATE(x) ((x) << 16) +# define SCLK_DPM_BOOT_STATE_MASK (0xff << 16) +# define SCLK_DPM_BOOT_STATE_SHIFT 16 +# define VOLTAGE_CHG_EN(x) ((x) << 24) +# define VOLTAGE_CHG_EN_MASK (0xff << 24) +# define VOLTAGE_CHG_EN_SHIFT 24 + +#define SMU_SCLK_DPM_TT_CNTL 0x1f108 +# define SCLK_TT_EN(x) ((x) << 0) +# define SCLK_TT_EN_MASK (0xff << 0) +# define SCLK_TT_EN_SHIFT 0 +#define SMU_SCLK_DPM_TTT 0x1f10c +# define LT(x) ((x) << 0) +# define LT_MASK (0xffff << 0) +# define LT_SHIFT 0 +# define HT(x) ((x) << 16) +# define HT_MASK (0xffff << 16) +# define HT_SHIFT 16 + +#define SMU_UVD_DPM_STATES 0x1f1a0 +#define SMU_UVD_DPM_CNTL 0x1f1a4 + +#define SMU_S_PG_CNTL 0x1f118 +# define DS_PG_EN(x) ((x) << 16) +# define DS_PG_EN_MASK (0xff << 16) +# define DS_PG_EN_SHIFT 16 + +#define GFX_POWER_GATING_CNTL 0x1f38c +# define PDS_DIV(x) ((x) << 0) +# define PDS_DIV_MASK (0xff << 0) +# define PDS_DIV_SHIFT 0 +# define SSSD(x) ((x) << 8) +# define SSSD_MASK (0xff << 8) +# define SSSD_SHIFT 8 + +#define PM_CONFIG 0x1f428 +# define SVI_Mode (1 << 29) + +#define PM_I_CNTL_1 0x1f464 +# define SCLK_DPM(x) ((x) << 0) +# define SCLK_DPM_MASK (0xff << 0) +# define SCLK_DPM_SHIFT 0 +# define DS_PG_CNTL(x) ((x) << 16) +# define DS_PG_CNTL_MASK (0xff << 16) +# define DS_PG_CNTL_SHIFT 16 +#define PM_TP 0x1f468 + +#define NB_PSTATE_CONFIG 0x1f5f8 +# define Dpm0PgNbPsLo(x) ((x) << 0) +# define Dpm0PgNbPsLo_MASK (3 << 0) +# define Dpm0PgNbPsLo_SHIFT 0 +# define Dpm0PgNbPsHi(x) ((x) << 2) +# define Dpm0PgNbPsHi_MASK (3 << 2) +# define Dpm0PgNbPsHi_SHIFT 2 +# define DpmXNbPsLo(x) ((x) << 4) +# define DpmXNbPsLo_MASK (3 << 4) +# define DpmXNbPsLo_SHIFT 4 +# define DpmXNbPsHi(x) ((x) << 6) +# define DpmXNbPsHi_MASK (3 << 6) +# define DpmXNbPsHi_SHIFT 6 + +#define DC_CAC_VALUE 0x1f908 + +#define GPU_CAC_AVRG_CNTL 0x1f920 +# define WINDOW_SIZE(x) ((x) << 0) +# define WINDOW_SIZE_MASK (0xff << 0) +# define WINDOW_SIZE_SHIFT 0 + +#define CC_SMU_MISC_FUSES 0xe0001004 +# define MinSClkDid(x) ((x) << 2) +# define MinSClkDid_MASK (0x7f << 2) +# define MinSClkDid_SHIFT 2 + +#define CC_SMU_TST_EFUSE1_MISC 0xe000101c +# define RB_BACKEND_DISABLE(x) ((x) << 16) +# define RB_BACKEND_DISABLE_MASK (3 << 16) +# define RB_BACKEND_DISABLE_SHIFT 16 + +#define SMU_SCRATCH_A 0xe0003024 + +#define SMU_SCRATCH0 0xe0003040 + +/* mmio */ +#define SMC_INT_REQ 0x220 + +#define SMC_MESSAGE_0 0x22c +#define SMC_RESP_0 0x230 + +#define GENERAL_PWRMGT 0x670 +# define GLOBAL_PWRMGT_EN (1 << 0) + +#define SCLK_PWRMGT_CNTL 0x678 +# define DYN_PWR_DOWN_EN (1 << 2) +# define RESET_BUSY_CNT (1 << 4) +# define RESET_SCLK_CNT (1 << 5) +# define DYN_GFX_CLK_OFF_EN (1 << 7) +# define GFX_CLK_FORCE_ON (1 << 8) +# define DYNAMIC_PM_EN (1 << 21) + +#define TARGET_AND_CURRENT_PROFILE_INDEX 0x684 +# define TARGET_STATE(x) ((x) << 0) +# define TARGET_STATE_MASK (0xf << 0) +# define TARGET_STATE_SHIFT 0 +# define CURRENT_STATE(x) ((x) << 4) +# define CURRENT_STATE_MASK (0xf << 4) +# define CURRENT_STATE_SHIFT 4 + +#define CG_GIPOTS 0x6d8 +# define CG_GIPOT(x) ((x) << 16) +# define CG_GIPOT_MASK (0xffff << 16) +# define CG_GIPOT_SHIFT 16 + +#define CG_PG_CTRL 0x6e0 +# define SP(x) ((x) << 0) +# define SP_MASK (0xffff << 0) +# define SP_SHIFT 0 +# define SU(x) ((x) << 16) +# define SU_MASK (0xffff << 16) +# define SU_SHIFT 16 + +#define CG_MISC_REG 0x708 + +#define CG_THERMAL_INT_CTRL 0x738 +# define DIG_THERM_INTH(x) ((x) << 0) +# define DIG_THERM_INTH_MASK (0xff << 0) +# define DIG_THERM_INTH_SHIFT 0 +# define DIG_THERM_INTL(x) ((x) << 8) +# define DIG_THERM_INTL_MASK (0xff << 8) +# define DIG_THERM_INTL_SHIFT 8 +# define THERM_INTH_MASK (1 << 24) +# define THERM_INTL_MASK (1 << 25) + +#define CG_CG_VOLTAGE_CNTL 0x770 +# define EN (1 << 9) + +#define HW_REV 0x5564 +# define ATI_REV_ID_MASK (0xf << 28) +# define ATI_REV_ID_SHIFT 28 +/* 0 = A0, 1 = A1, 2 = B0, 3 = C0, etc. */ + +#define CGTS_SM_CTRL_REG 0x9150 + +#define GB_ADDR_CONFIG 0x98f8 + +#endif diff --git a/sys/dev/pci/drm/radeon/uvd_v1_0.c b/sys/dev/pci/drm/radeon/uvd_v1_0.c new file mode 100644 index 00000000000..1f22d1ea8b1 --- /dev/null +++ b/sys/dev/pci/drm/radeon/uvd_v1_0.c @@ -0,0 +1,534 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "r600d.h" + +/** + * uvd_v1_0_get_rptr - get read pointer + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * + * Returns the current hardware read pointer + */ +uint32_t uvd_v1_0_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + return RREG32(UVD_RBC_RB_RPTR); +} + +/** + * uvd_v1_0_get_wptr - get write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * + * Returns the current hardware write pointer + */ +uint32_t uvd_v1_0_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + return RREG32(UVD_RBC_RB_WPTR); +} + +/** + * uvd_v1_0_set_wptr - set write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * + * Commits the write pointer to the hardware + */ +void uvd_v1_0_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + WREG32(UVD_RBC_RB_WPTR, ring->wptr); +} + +/** + * uvd_v1_0_fence_emit - emit an fence & trap command + * + * @rdev: radeon_device pointer + * @fence: fence to emit + * + * Write a fence and a trap command to the ring. + */ +void uvd_v1_0_fence_emit(struct radeon_device *rdev, + struct radeon_fence *fence) +{ + struct radeon_ring *ring = &rdev->ring[fence->ring]; + uint64_t addr = rdev->fence_drv[fence->ring].gpu_addr; + + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0)); + radeon_ring_write(ring, addr & 0xffffffff); + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0)); + radeon_ring_write(ring, fence->seq); + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0)); + radeon_ring_write(ring, 0); + + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0)); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0)); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0)); + radeon_ring_write(ring, 2); + return; +} + +/** + * uvd_v1_0_resume - memory controller programming + * + * @rdev: radeon_device pointer + * + * Let the UVD memory controller know it's offsets + */ +int uvd_v1_0_resume(struct radeon_device *rdev) +{ + uint64_t addr; + uint32_t size; + int r; + + r = radeon_uvd_resume(rdev); + if (r) + return r; + + /* programm the VCPU memory controller bits 0-27 */ + addr = (rdev->uvd.gpu_addr >> 3) + 16; + size = RADEON_GPU_PAGE_ALIGN(rdev->uvd_fw->size) >> 3; + WREG32(UVD_VCPU_CACHE_OFFSET0, addr); + WREG32(UVD_VCPU_CACHE_SIZE0, size); + + addr += size; + size = RADEON_UVD_STACK_SIZE >> 3; + WREG32(UVD_VCPU_CACHE_OFFSET1, addr); + WREG32(UVD_VCPU_CACHE_SIZE1, size); + + addr += size; + size = RADEON_UVD_HEAP_SIZE >> 3; + WREG32(UVD_VCPU_CACHE_OFFSET2, addr); + WREG32(UVD_VCPU_CACHE_SIZE2, size); + + /* bits 28-31 */ + addr = (rdev->uvd.gpu_addr >> 28) & 0xF; + WREG32(UVD_LMI_ADDR_EXT, (addr << 12) | (addr << 0)); + + /* bits 32-39 */ + addr = (rdev->uvd.gpu_addr >> 32) & 0xFF; + WREG32(UVD_LMI_EXT40_ADDR, addr | (0x9 << 16) | (0x1 << 31)); + + WREG32(UVD_FW_START, *((uint32_t*)rdev->uvd.cpu_addr)); + + return 0; +} + +/** + * uvd_v1_0_init - start and test UVD block + * + * @rdev: radeon_device pointer + * + * Initialize the hardware, boot up the VCPU and do some testing + */ +int uvd_v1_0_init(struct radeon_device *rdev) +{ + struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + uint32_t tmp; + int r; + + /* raise clocks while booting up the VCPU */ + if (rdev->family < CHIP_RV740) + radeon_set_uvd_clocks(rdev, 10000, 10000); + else + radeon_set_uvd_clocks(rdev, 53300, 40000); + + r = uvd_v1_0_start(rdev); + if (r) + goto done; + + ring->ready = true; + r = radeon_ring_test(rdev, R600_RING_TYPE_UVD_INDEX, ring); + if (r) { + ring->ready = false; + goto done; + } + + r = radeon_ring_lock(rdev, ring, 10); + if (r) { + DRM_ERROR("radeon: ring failed to lock UVD ring (%d).\n", r); + goto done; + } + + tmp = PACKET0(UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL, 0); + radeon_ring_write(ring, tmp); + radeon_ring_write(ring, 0xFFFFF); + + tmp = PACKET0(UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL, 0); + radeon_ring_write(ring, tmp); + radeon_ring_write(ring, 0xFFFFF); + + tmp = PACKET0(UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL, 0); + radeon_ring_write(ring, tmp); + radeon_ring_write(ring, 0xFFFFF); + + /* Clear timeout status bits */ + radeon_ring_write(ring, PACKET0(UVD_SEMA_TIMEOUT_STATUS, 0)); + radeon_ring_write(ring, 0x8); + + radeon_ring_write(ring, PACKET0(UVD_SEMA_CNTL, 0)); + radeon_ring_write(ring, 3); + + radeon_ring_unlock_commit(rdev, ring, false); + +done: + /* lower clocks again */ + radeon_set_uvd_clocks(rdev, 0, 0); + + if (!r) { + switch (rdev->family) { + case CHIP_RV610: + case CHIP_RV630: + case CHIP_RV620: + /* 64byte granularity workaround */ + WREG32(MC_CONFIG, 0); + WREG32(MC_CONFIG, 1 << 4); + WREG32(RS_DQ_RD_RET_CONF, 0x3f); + WREG32(MC_CONFIG, 0x1f); + + /* fall through */ + case CHIP_RV670: + case CHIP_RV635: + + /* write clean workaround */ + WREG32_P(UVD_VCPU_CNTL, 0x10, ~0x10); + break; + + default: + /* TODO: Do we need more? */ + break; + } + + DRM_INFO("UVD initialized successfully.\n"); + } + + return r; +} + +/** + * uvd_v1_0_fini - stop the hardware block + * + * @rdev: radeon_device pointer + * + * Stop the UVD block, mark ring as not ready any more + */ +void uvd_v1_0_fini(struct radeon_device *rdev) +{ + struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + + uvd_v1_0_stop(rdev); + ring->ready = false; +} + +/** + * uvd_v1_0_start - start UVD block + * + * @rdev: radeon_device pointer + * + * Setup and start the UVD block + */ +int uvd_v1_0_start(struct radeon_device *rdev) +{ + struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; + uint32_t rb_bufsz; + int i, j, r; + + /* disable byte swapping */ + u32 lmi_swap_cntl = 0; + u32 mp_swap_cntl = 0; + + /* disable clock gating */ + WREG32(UVD_CGC_GATE, 0); + + /* disable interupt */ + WREG32_P(UVD_MASTINT_EN, 0, ~(1 << 1)); + + /* Stall UMC and register bus before resetting VCPU */ + WREG32_P(UVD_LMI_CTRL2, 1 << 8, ~(1 << 8)); + WREG32_P(UVD_RB_ARB_CTRL, 1 << 3, ~(1 << 3)); + mdelay(1); + + /* put LMI, VCPU, RBC etc... into reset */ + WREG32(UVD_SOFT_RESET, LMI_SOFT_RESET | VCPU_SOFT_RESET | + LBSI_SOFT_RESET | RBC_SOFT_RESET | CSM_SOFT_RESET | + CXW_SOFT_RESET | TAP_SOFT_RESET | LMI_UMC_SOFT_RESET); + mdelay(5); + + /* take UVD block out of reset */ + WREG32_P(SRBM_SOFT_RESET, 0, ~SOFT_RESET_UVD); + mdelay(5); + + /* initialize UVD memory controller */ + WREG32(UVD_LMI_CTRL, 0x40 | (1 << 8) | (1 << 13) | + (1 << 21) | (1 << 9) | (1 << 20)); + +#ifdef __BIG_ENDIAN + /* swap (8 in 32) RB and IB */ + lmi_swap_cntl = 0xa; + mp_swap_cntl = 0; +#endif + WREG32(UVD_LMI_SWAP_CNTL, lmi_swap_cntl); + WREG32(UVD_MP_SWAP_CNTL, mp_swap_cntl); + + WREG32(UVD_MPC_SET_MUXA0, 0x40c2040); + WREG32(UVD_MPC_SET_MUXA1, 0x0); + WREG32(UVD_MPC_SET_MUXB0, 0x40c2040); + WREG32(UVD_MPC_SET_MUXB1, 0x0); + WREG32(UVD_MPC_SET_ALU, 0); + WREG32(UVD_MPC_SET_MUX, 0x88); + + /* take all subblocks out of reset, except VCPU */ + WREG32(UVD_SOFT_RESET, VCPU_SOFT_RESET); + mdelay(5); + + /* enable VCPU clock */ + WREG32(UVD_VCPU_CNTL, 1 << 9); + + /* enable UMC */ + WREG32_P(UVD_LMI_CTRL2, 0, ~(1 << 8)); + + WREG32_P(UVD_RB_ARB_CTRL, 0, ~(1 << 3)); + + /* boot up the VCPU */ + WREG32(UVD_SOFT_RESET, 0); + mdelay(10); + + for (i = 0; i < 10; ++i) { + uint32_t status; + for (j = 0; j < 100; ++j) { + status = RREG32(UVD_STATUS); + if (status & 2) + break; + mdelay(10); + } + r = 0; + if (status & 2) + break; + + DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n"); + WREG32_P(UVD_SOFT_RESET, VCPU_SOFT_RESET, ~VCPU_SOFT_RESET); + mdelay(10); + WREG32_P(UVD_SOFT_RESET, 0, ~VCPU_SOFT_RESET); + mdelay(10); + r = -1; + } + + if (r) { + DRM_ERROR("UVD not responding, giving up!!!\n"); + return r; + } + + /* enable interupt */ + WREG32_P(UVD_MASTINT_EN, 3<<1, ~(3 << 1)); + + /* force RBC into idle state */ + WREG32(UVD_RBC_RB_CNTL, 0x11010101); + + /* Set the write pointer delay */ + WREG32(UVD_RBC_RB_WPTR_CNTL, 0); + + /* programm the 4GB memory segment for rptr and ring buffer */ + WREG32(UVD_LMI_EXT40_ADDR, upper_32_bits(ring->gpu_addr) | + (0x7 << 16) | (0x1 << 31)); + + /* Initialize the ring buffer's read and write pointers */ + WREG32(UVD_RBC_RB_RPTR, 0x0); + + ring->wptr = RREG32(UVD_RBC_RB_RPTR); + WREG32(UVD_RBC_RB_WPTR, ring->wptr); + + /* set the ring address */ + WREG32(UVD_RBC_RB_BASE, ring->gpu_addr); + + /* Set ring buffer size */ + rb_bufsz = order_base_2(ring->ring_size); + rb_bufsz = (0x1 << 8) | rb_bufsz; + WREG32_P(UVD_RBC_RB_CNTL, rb_bufsz, ~0x11f1f); + + return 0; +} + +/** + * uvd_v1_0_stop - stop UVD block + * + * @rdev: radeon_device pointer + * + * stop the UVD block + */ +void uvd_v1_0_stop(struct radeon_device *rdev) +{ + /* force RBC into idle state */ + WREG32(UVD_RBC_RB_CNTL, 0x11010101); + + /* Stall UMC and register bus before resetting VCPU */ + WREG32_P(UVD_LMI_CTRL2, 1 << 8, ~(1 << 8)); + WREG32_P(UVD_RB_ARB_CTRL, 1 << 3, ~(1 << 3)); + mdelay(1); + + /* put VCPU into reset */ + WREG32(UVD_SOFT_RESET, VCPU_SOFT_RESET); + mdelay(5); + + /* disable VCPU clock */ + WREG32(UVD_VCPU_CNTL, 0x0); + + /* Unstall UMC and register bus */ + WREG32_P(UVD_LMI_CTRL2, 0, ~(1 << 8)); + WREG32_P(UVD_RB_ARB_CTRL, 0, ~(1 << 3)); +} + +/** + * uvd_v1_0_ring_test - register write test + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * + * Test if we can successfully write to the context register + */ +int uvd_v1_0_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) +{ + uint32_t tmp = 0; + unsigned i; + int r; + + WREG32(UVD_CONTEXT_ID, 0xCAFEDEAD); + r = radeon_ring_lock(rdev, ring, 3); + if (r) { + DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", + ring->idx, r); + return r; + } + radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0)); + radeon_ring_write(ring, 0xDEADBEEF); + radeon_ring_unlock_commit(rdev, ring, false); + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = RREG32(UVD_CONTEXT_ID); + if (tmp == 0xDEADBEEF) + break; + DRM_UDELAY(1); + } + + if (i < rdev->usec_timeout) { + DRM_INFO("ring test on %d succeeded in %d usecs\n", + ring->idx, i); + } else { + DRM_ERROR("radeon: ring %d test failed (0x%08X)\n", + ring->idx, tmp); + r = -EINVAL; + } + return r; +} + +/** + * uvd_v1_0_semaphore_emit - emit semaphore command + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * @semaphore: semaphore to emit commands for + * @emit_wait: true if we should emit a wait command + * + * Emit a semaphore command (either wait or signal) to the UVD ring. + */ +bool uvd_v1_0_semaphore_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait) +{ + /* disable semaphores for UVD V1 hardware */ + return false; +} + +/** + * uvd_v1_0_ib_execute - execute indirect buffer + * + * @rdev: radeon_device pointer + * @ib: indirect buffer to execute + * + * Write ring commands to execute the indirect buffer + */ +void uvd_v1_0_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) +{ + struct radeon_ring *ring = &rdev->ring[ib->ring]; + + radeon_ring_write(ring, PACKET0(UVD_RBC_IB_BASE, 0)); + radeon_ring_write(ring, ib->gpu_addr); + radeon_ring_write(ring, PACKET0(UVD_RBC_IB_SIZE, 0)); + radeon_ring_write(ring, ib->length_dw); +} + +/** + * uvd_v1_0_ib_test - test ib execution + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * + * Test if we can successfully execute an IB + */ +int uvd_v1_0_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) +{ + struct radeon_fence *fence = NULL; + int r; + + if (rdev->family < CHIP_RV740) + r = radeon_set_uvd_clocks(rdev, 10000, 10000); + else + r = radeon_set_uvd_clocks(rdev, 53300, 40000); + if (r) { + DRM_ERROR("radeon: failed to raise UVD clocks (%d).\n", r); + return r; + } + + r = radeon_uvd_get_create_msg(rdev, ring->idx, 1, NULL); + if (r) { + DRM_ERROR("radeon: failed to get create msg (%d).\n", r); + goto error; + } + + r = radeon_uvd_get_destroy_msg(rdev, ring->idx, 1, &fence); + if (r) { + DRM_ERROR("radeon: failed to get destroy ib (%d).\n", r); + goto error; + } + + r = radeon_fence_wait(fence, false); + if (r) { + DRM_ERROR("radeon: fence wait failed (%d).\n", r); + goto error; + } + DRM_INFO("ib test on ring %d succeeded\n", ring->idx); +error: + radeon_fence_unref(&fence); + radeon_set_uvd_clocks(rdev, 0, 0); + return r; +} diff --git a/sys/dev/pci/drm/radeon/uvd_v2_2.c b/sys/dev/pci/drm/radeon/uvd_v2_2.c new file mode 100644 index 00000000000..8eb919a87d9 --- /dev/null +++ b/sys/dev/pci/drm/radeon/uvd_v2_2.c @@ -0,0 +1,197 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "rv770d.h" + +/** + * uvd_v2_2_fence_emit - emit an fence & trap command + * + * @rdev: radeon_device pointer + * @fence: fence to emit + * + * Write a fence and a trap command to the ring. + */ +void uvd_v2_2_fence_emit(struct radeon_device *rdev, + struct radeon_fence *fence) +{ + struct radeon_ring *ring = &rdev->ring[fence->ring]; + uint64_t addr = rdev->fence_drv[fence->ring].gpu_addr; + + radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0)); + radeon_ring_write(ring, fence->seq); + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0)); + radeon_ring_write(ring, lower_32_bits(addr)); + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0)); + radeon_ring_write(ring, upper_32_bits(addr) & 0xff); + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0)); + radeon_ring_write(ring, 0); + + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0)); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0)); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0)); + radeon_ring_write(ring, 2); +} + +/** + * uvd_v2_2_semaphore_emit - emit semaphore command + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * @semaphore: semaphore to emit commands for + * @emit_wait: true if we should emit a wait command + * + * Emit a semaphore command (either wait or signal) to the UVD ring. + */ +bool uvd_v2_2_semaphore_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait) +{ + uint64_t addr = semaphore->gpu_addr; + + radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0)); + radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF); + + radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0)); + radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF); + + radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0)); + radeon_ring_write(ring, emit_wait ? 1 : 0); + + return true; +} + +/** + * uvd_v2_2_resume - memory controller programming + * + * @rdev: radeon_device pointer + * + * Let the UVD memory controller know it's offsets + */ +int uvd_v2_2_resume(struct radeon_device *rdev) +{ + uint64_t addr; + uint32_t chip_id, size; + int r; + + /* RV770 uses V1.0 MC */ + if (rdev->family == CHIP_RV770) + return uvd_v1_0_resume(rdev); + + r = radeon_uvd_resume(rdev); + if (r) + return r; + + /* programm the VCPU memory controller bits 0-27 */ + addr = rdev->uvd.gpu_addr >> 3; + size = RADEON_GPU_PAGE_ALIGN(rdev->uvd_fw->size + 4) >> 3; + WREG32(UVD_VCPU_CACHE_OFFSET0, addr); + WREG32(UVD_VCPU_CACHE_SIZE0, size); + + addr += size; + size = RADEON_UVD_STACK_SIZE >> 3; + WREG32(UVD_VCPU_CACHE_OFFSET1, addr); + WREG32(UVD_VCPU_CACHE_SIZE1, size); + + addr += size; + size = RADEON_UVD_HEAP_SIZE >> 3; + WREG32(UVD_VCPU_CACHE_OFFSET2, addr); + WREG32(UVD_VCPU_CACHE_SIZE2, size); + + /* bits 28-31 */ + addr = (rdev->uvd.gpu_addr >> 28) & 0xF; + WREG32(UVD_LMI_ADDR_EXT, (addr << 12) | (addr << 0)); + + /* bits 32-39 */ + addr = (rdev->uvd.gpu_addr >> 32) & 0xFF; + WREG32(UVD_LMI_EXT40_ADDR, addr | (0x9 << 16) | (0x1 << 31)); + + /* tell firmware which hardware it is running on */ + switch (rdev->family) { + default: + return -EINVAL; + case CHIP_RV710: + chip_id = 0x01000005; + break; + case CHIP_RV730: + chip_id = 0x01000006; + break; + case CHIP_RV740: + chip_id = 0x01000007; + break; + case CHIP_CYPRESS: + case CHIP_HEMLOCK: + chip_id = 0x01000008; + break; + case CHIP_JUNIPER: + chip_id = 0x01000009; + break; + case CHIP_REDWOOD: + chip_id = 0x0100000a; + break; + case CHIP_CEDAR: + chip_id = 0x0100000b; + break; + case CHIP_SUMO: + case CHIP_SUMO2: + chip_id = 0x0100000c; + break; + case CHIP_PALM: + chip_id = 0x0100000e; + break; + case CHIP_CAYMAN: + chip_id = 0x0100000f; + break; + case CHIP_BARTS: + chip_id = 0x01000010; + break; + case CHIP_TURKS: + chip_id = 0x01000011; + break; + case CHIP_CAICOS: + chip_id = 0x01000012; + break; + case CHIP_TAHITI: + chip_id = 0x01000014; + break; + case CHIP_VERDE: + chip_id = 0x01000015; + break; + case CHIP_PITCAIRN: + case CHIP_OLAND: + chip_id = 0x01000016; + break; + case CHIP_ARUBA: + chip_id = 0x01000017; + break; + } + WREG32(UVD_VCPU_CHIP_ID, chip_id); + + return 0; +} diff --git a/sys/dev/pci/drm/radeon/uvd_v3_1.c b/sys/dev/pci/drm/radeon/uvd_v3_1.c new file mode 100644 index 00000000000..0d55c16e02f --- /dev/null +++ b/sys/dev/pci/drm/radeon/uvd_v3_1.c @@ -0,0 +1,57 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "nid.h" + +/** + * uvd_v3_1_semaphore_emit - emit semaphore command + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * @semaphore: semaphore to emit commands for + * @emit_wait: true if we should emit a wait command + * + * Emit a semaphore command (either wait or signal) to the UVD ring. + */ +bool uvd_v3_1_semaphore_emit(struct radeon_device *rdev, + struct radeon_ring *ring, + struct radeon_semaphore *semaphore, + bool emit_wait) +{ + uint64_t addr = semaphore->gpu_addr; + + radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0)); + radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF); + + radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0)); + radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF); + + radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0)); + radeon_ring_write(ring, 0x80 | (emit_wait ? 1 : 0)); + + return true; +} diff --git a/sys/dev/pci/drm/radeon/uvd_v4_2.c b/sys/dev/pci/drm/radeon/uvd_v4_2.c new file mode 100644 index 00000000000..53aab066e68 --- /dev/null +++ b/sys/dev/pci/drm/radeon/uvd_v4_2.c @@ -0,0 +1,67 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "cikd.h" + +/** + * uvd_v4_2_resume - memory controller programming + * + * @rdev: radeon_device pointer + * + * Let the UVD memory controller know it's offsets + */ +int uvd_v4_2_resume(struct radeon_device *rdev) +{ + uint64_t addr; + uint32_t size; + + /* programm the VCPU memory controller bits 0-27 */ + addr = rdev->uvd.gpu_addr >> 3; + size = RADEON_GPU_PAGE_ALIGN(rdev->uvd_fw->size + 4) >> 3; + WREG32(UVD_VCPU_CACHE_OFFSET0, addr); + WREG32(UVD_VCPU_CACHE_SIZE0, size); + + addr += size; + size = RADEON_UVD_STACK_SIZE >> 3; + WREG32(UVD_VCPU_CACHE_OFFSET1, addr); + WREG32(UVD_VCPU_CACHE_SIZE1, size); + + addr += size; + size = RADEON_UVD_HEAP_SIZE >> 3; + WREG32(UVD_VCPU_CACHE_OFFSET2, addr); + WREG32(UVD_VCPU_CACHE_SIZE2, size); + + /* bits 28-31 */ + addr = (rdev->uvd.gpu_addr >> 28) & 0xF; + WREG32(UVD_LMI_ADDR_EXT, (addr << 12) | (addr << 0)); + + /* bits 32-39 */ + addr = (rdev->uvd.gpu_addr >> 32) & 0xFF; + WREG32(UVD_LMI_EXT40_ADDR, addr | (0x9 << 16) | (0x1 << 31)); + + return 0; +} diff --git a/sys/dev/pci/drm/radeon/vce_v1_0.c b/sys/dev/pci/drm/radeon/vce_v1_0.c new file mode 100644 index 00000000000..391555d8f96 --- /dev/null +++ b/sys/dev/pci/drm/radeon/vce_v1_0.c @@ -0,0 +1,383 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * Authors: Christian König + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "sid.h" + +#define VCE_V1_0_FW_SIZE (256 * 1024) +#define VCE_V1_0_STACK_SIZE (64 * 1024) +#define VCE_V1_0_DATA_SIZE (7808 * (RADEON_MAX_VCE_HANDLES + 1)) + +struct vce_v1_0_fw_signature +{ + int32_t off; + uint32_t len; + int32_t num; + struct { + uint32_t chip_id; + uint32_t keyselect; + uint32_t nonce[4]; + uint32_t sigval[4]; + } val[8]; +}; + +/** + * vce_v1_0_get_rptr - get read pointer + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * + * Returns the current hardware read pointer + */ +uint32_t vce_v1_0_get_rptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + if (ring->idx == TN_RING_TYPE_VCE1_INDEX) + return RREG32(VCE_RB_RPTR); + else + return RREG32(VCE_RB_RPTR2); +} + +/** + * vce_v1_0_get_wptr - get write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * + * Returns the current hardware write pointer + */ +uint32_t vce_v1_0_get_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + if (ring->idx == TN_RING_TYPE_VCE1_INDEX) + return RREG32(VCE_RB_WPTR); + else + return RREG32(VCE_RB_WPTR2); +} + +/** + * vce_v1_0_set_wptr - set write pointer + * + * @rdev: radeon_device pointer + * @ring: radeon_ring pointer + * + * Commits the write pointer to the hardware + */ +void vce_v1_0_set_wptr(struct radeon_device *rdev, + struct radeon_ring *ring) +{ + if (ring->idx == TN_RING_TYPE_VCE1_INDEX) + WREG32(VCE_RB_WPTR, ring->wptr); + else + WREG32(VCE_RB_WPTR2, ring->wptr); +} + +void vce_v1_0_enable_mgcg(struct radeon_device *rdev, bool enable) +{ + u32 tmp; + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_VCE_MGCG)) { + tmp = RREG32(VCE_CLOCK_GATING_A); + tmp |= CGC_DYN_CLOCK_MODE; + WREG32(VCE_CLOCK_GATING_A, tmp); + + tmp = RREG32(VCE_UENC_CLOCK_GATING); + tmp &= ~0x1ff000; + tmp |= 0xff800000; + WREG32(VCE_UENC_CLOCK_GATING, tmp); + + tmp = RREG32(VCE_UENC_REG_CLOCK_GATING); + tmp &= ~0x3ff; + WREG32(VCE_UENC_REG_CLOCK_GATING, tmp); + } else { + tmp = RREG32(VCE_CLOCK_GATING_A); + tmp &= ~CGC_DYN_CLOCK_MODE; + WREG32(VCE_CLOCK_GATING_A, tmp); + + tmp = RREG32(VCE_UENC_CLOCK_GATING); + tmp |= 0x1ff000; + tmp &= ~0xff800000; + WREG32(VCE_UENC_CLOCK_GATING, tmp); + + tmp = RREG32(VCE_UENC_REG_CLOCK_GATING); + tmp |= 0x3ff; + WREG32(VCE_UENC_REG_CLOCK_GATING, tmp); + } +} + +static void vce_v1_0_init_cg(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32(VCE_CLOCK_GATING_A); + tmp |= CGC_DYN_CLOCK_MODE; + WREG32(VCE_CLOCK_GATING_A, tmp); + + tmp = RREG32(VCE_CLOCK_GATING_B); + tmp |= 0x1e; + tmp &= ~0xe100e1; + WREG32(VCE_CLOCK_GATING_B, tmp); + + tmp = RREG32(VCE_UENC_CLOCK_GATING); + tmp &= ~0xff9ff000; + WREG32(VCE_UENC_CLOCK_GATING, tmp); + + tmp = RREG32(VCE_UENC_REG_CLOCK_GATING); + tmp &= ~0x3ff; + WREG32(VCE_UENC_REG_CLOCK_GATING, tmp); +} + +int vce_v1_0_load_fw(struct radeon_device *rdev, uint32_t *data) +{ + struct vce_v1_0_fw_signature *sign = (void*)rdev->vce_fw->data; + uint32_t chip_id; + int i; + + switch (rdev->family) { + case CHIP_TAHITI: + chip_id = 0x01000014; + break; + case CHIP_VERDE: + chip_id = 0x01000015; + break; + case CHIP_PITCAIRN: + case CHIP_OLAND: + chip_id = 0x01000016; + break; + case CHIP_ARUBA: + chip_id = 0x01000017; + break; + default: + return -EINVAL; + } + + for (i = 0; i < le32_to_cpu(sign->num); ++i) { + if (le32_to_cpu(sign->val[i].chip_id) == chip_id) + break; + } + + if (i == le32_to_cpu(sign->num)) + return -EINVAL; + + data += (256 - 64) / 4; + data[0] = sign->val[i].nonce[0]; + data[1] = sign->val[i].nonce[1]; + data[2] = sign->val[i].nonce[2]; + data[3] = sign->val[i].nonce[3]; + data[4] = cpu_to_le32(le32_to_cpu(sign->len) + 64); + + memset(&data[5], 0, 44); + memcpy(&data[16], &sign[1], rdev->vce_fw->size - sizeof(*sign)); + + data += le32_to_cpu(data[4]) / 4; + data[0] = sign->val[i].sigval[0]; + data[1] = sign->val[i].sigval[1]; + data[2] = sign->val[i].sigval[2]; + data[3] = sign->val[i].sigval[3]; + + rdev->vce.keyselect = le32_to_cpu(sign->val[i].keyselect); + + return 0; +} + +unsigned vce_v1_0_bo_size(struct radeon_device *rdev) +{ + WARN_ON(VCE_V1_0_FW_SIZE < rdev->vce_fw->size); + return VCE_V1_0_FW_SIZE + VCE_V1_0_STACK_SIZE + VCE_V1_0_DATA_SIZE; +} + +int vce_v1_0_resume(struct radeon_device *rdev) +{ + uint64_t addr = rdev->vce.gpu_addr; + uint32_t size; + int i; + + WREG32_P(VCE_CLOCK_GATING_A, 0, ~(1 << 16)); + WREG32_P(VCE_UENC_CLOCK_GATING, 0x1FF000, ~0xFF9FF000); + WREG32_P(VCE_UENC_REG_CLOCK_GATING, 0x3F, ~0x3F); + WREG32(VCE_CLOCK_GATING_B, 0); + + WREG32_P(VCE_LMI_FW_PERIODIC_CTRL, 0x4, ~0x4); + + WREG32(VCE_LMI_CTRL, 0x00398000); + WREG32_P(VCE_LMI_CACHE_CTRL, 0x0, ~0x1); + WREG32(VCE_LMI_SWAP_CNTL, 0); + WREG32(VCE_LMI_SWAP_CNTL1, 0); + WREG32(VCE_LMI_VM_CTRL, 0); + + WREG32(VCE_VCPU_SCRATCH7, RADEON_MAX_VCE_HANDLES); + + addr += 256; + size = VCE_V1_0_FW_SIZE; + WREG32(VCE_VCPU_CACHE_OFFSET0, addr & 0x7fffffff); + WREG32(VCE_VCPU_CACHE_SIZE0, size); + + addr += size; + size = VCE_V1_0_STACK_SIZE; + WREG32(VCE_VCPU_CACHE_OFFSET1, addr & 0x7fffffff); + WREG32(VCE_VCPU_CACHE_SIZE1, size); + + addr += size; + size = VCE_V1_0_DATA_SIZE; + WREG32(VCE_VCPU_CACHE_OFFSET2, addr & 0x7fffffff); + WREG32(VCE_VCPU_CACHE_SIZE2, size); + + WREG32_P(VCE_LMI_CTRL2, 0x0, ~0x100); + + WREG32(VCE_LMI_FW_START_KEYSEL, rdev->vce.keyselect); + + for (i = 0; i < 10; ++i) { + mdelay(10); + if (RREG32(VCE_FW_REG_STATUS) & VCE_FW_REG_STATUS_DONE) + break; + } + + if (i == 10) + return -ETIMEDOUT; + + if (!(RREG32(VCE_FW_REG_STATUS) & VCE_FW_REG_STATUS_PASS)) + return -EINVAL; + + for (i = 0; i < 10; ++i) { + mdelay(10); + if (!(RREG32(VCE_FW_REG_STATUS) & VCE_FW_REG_STATUS_BUSY)) + break; + } + + if (i == 10) + return -ETIMEDOUT; + + vce_v1_0_init_cg(rdev); + + return 0; +} + +/** + * vce_v1_0_start - start VCE block + * + * @rdev: radeon_device pointer + * + * Setup and start the VCE block + */ +int vce_v1_0_start(struct radeon_device *rdev) +{ + struct radeon_ring *ring; + int i, j, r; + + /* set BUSY flag */ + WREG32_P(VCE_STATUS, 1, ~1); + + ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; + WREG32(VCE_RB_RPTR, ring->wptr); + WREG32(VCE_RB_WPTR, ring->wptr); + WREG32(VCE_RB_BASE_LO, ring->gpu_addr); + WREG32(VCE_RB_BASE_HI, upper_32_bits(ring->gpu_addr)); + WREG32(VCE_RB_SIZE, ring->ring_size / 4); + + ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; + WREG32(VCE_RB_RPTR2, ring->wptr); + WREG32(VCE_RB_WPTR2, ring->wptr); + WREG32(VCE_RB_BASE_LO2, ring->gpu_addr); + WREG32(VCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr)); + WREG32(VCE_RB_SIZE2, ring->ring_size / 4); + + WREG32_P(VCE_VCPU_CNTL, VCE_CLK_EN, ~VCE_CLK_EN); + + WREG32_P(VCE_SOFT_RESET, + VCE_ECPU_SOFT_RESET | + VCE_FME_SOFT_RESET, ~( + VCE_ECPU_SOFT_RESET | + VCE_FME_SOFT_RESET)); + + mdelay(100); + + WREG32_P(VCE_SOFT_RESET, 0, ~( + VCE_ECPU_SOFT_RESET | + VCE_FME_SOFT_RESET)); + + for (i = 0; i < 10; ++i) { + uint32_t status; + for (j = 0; j < 100; ++j) { + status = RREG32(VCE_STATUS); + if (status & 2) + break; + mdelay(10); + } + r = 0; + if (status & 2) + break; + + DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n"); + WREG32_P(VCE_SOFT_RESET, VCE_ECPU_SOFT_RESET, ~VCE_ECPU_SOFT_RESET); + mdelay(10); + WREG32_P(VCE_SOFT_RESET, 0, ~VCE_ECPU_SOFT_RESET); + mdelay(10); + r = -1; + } + + /* clear BUSY flag */ + WREG32_P(VCE_STATUS, 0, ~1); + + if (r) { + DRM_ERROR("VCE not responding, giving up!!!\n"); + return r; + } + + return 0; +} + +int vce_v1_0_init(struct radeon_device *rdev) +{ + struct radeon_ring *ring; + int r; + + r = vce_v1_0_start(rdev); + if (r) + return r; + + ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; + ring->ready = true; + r = radeon_ring_test(rdev, TN_RING_TYPE_VCE1_INDEX, ring); + if (r) { + ring->ready = false; + return r; + } + + ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; + ring->ready = true; + r = radeon_ring_test(rdev, TN_RING_TYPE_VCE2_INDEX, ring); + if (r) { + ring->ready = false; + return r; + } + + DRM_INFO("VCE initialized successfully.\n"); + + return 0; +} diff --git a/sys/dev/pci/drm/radeon/vce_v2_0.c b/sys/dev/pci/drm/radeon/vce_v2_0.c new file mode 100644 index 00000000000..a3c9f1b6549 --- /dev/null +++ b/sys/dev/pci/drm/radeon/vce_v2_0.c @@ -0,0 +1,193 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * Authors: Christian König + */ + +#include +#include "radeon.h" +#include "radeon_asic.h" +#include "cikd.h" + +#define VCE_V2_0_FW_SIZE (256 * 1024) +#define VCE_V2_0_STACK_SIZE (64 * 1024) +#define VCE_V2_0_DATA_SIZE (23552 * RADEON_MAX_VCE_HANDLES) + +static void vce_v2_0_set_sw_cg(struct radeon_device *rdev, bool gated) +{ + u32 tmp; + + if (gated) { + tmp = RREG32(VCE_CLOCK_GATING_B); + tmp |= 0xe70000; + WREG32(VCE_CLOCK_GATING_B, tmp); + + tmp = RREG32(VCE_UENC_CLOCK_GATING); + tmp |= 0xff000000; + WREG32(VCE_UENC_CLOCK_GATING, tmp); + + tmp = RREG32(VCE_UENC_REG_CLOCK_GATING); + tmp &= ~0x3fc; + WREG32(VCE_UENC_REG_CLOCK_GATING, tmp); + + WREG32(VCE_CGTT_CLK_OVERRIDE, 0); + } else { + tmp = RREG32(VCE_CLOCK_GATING_B); + tmp |= 0xe7; + tmp &= ~0xe70000; + WREG32(VCE_CLOCK_GATING_B, tmp); + + tmp = RREG32(VCE_UENC_CLOCK_GATING); + tmp |= 0x1fe000; + tmp &= ~0xff000000; + WREG32(VCE_UENC_CLOCK_GATING, tmp); + + tmp = RREG32(VCE_UENC_REG_CLOCK_GATING); + tmp |= 0x3fc; + WREG32(VCE_UENC_REG_CLOCK_GATING, tmp); + } +} + +static void vce_v2_0_set_dyn_cg(struct radeon_device *rdev, bool gated) +{ + u32 orig, tmp; + + tmp = RREG32(VCE_CLOCK_GATING_B); + tmp &= ~0x00060006; + if (gated) { + tmp |= 0xe10000; + } else { + tmp |= 0xe1; + tmp &= ~0xe10000; + } + WREG32(VCE_CLOCK_GATING_B, tmp); + + orig = tmp = RREG32(VCE_UENC_CLOCK_GATING); + tmp &= ~0x1fe000; + tmp &= ~0xff000000; + if (tmp != orig) + WREG32(VCE_UENC_CLOCK_GATING, tmp); + + orig = tmp = RREG32(VCE_UENC_REG_CLOCK_GATING); + tmp &= ~0x3fc; + if (tmp != orig) + WREG32(VCE_UENC_REG_CLOCK_GATING, tmp); + + if (gated) + WREG32(VCE_CGTT_CLK_OVERRIDE, 0); +} + +static void vce_v2_0_disable_cg(struct radeon_device *rdev) +{ + WREG32(VCE_CGTT_CLK_OVERRIDE, 7); +} + +void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable) +{ + bool sw_cg = false; + + if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_VCE_MGCG)) { + if (sw_cg) + vce_v2_0_set_sw_cg(rdev, true); + else + vce_v2_0_set_dyn_cg(rdev, true); + } else { + vce_v2_0_disable_cg(rdev); + + if (sw_cg) + vce_v2_0_set_sw_cg(rdev, false); + else + vce_v2_0_set_dyn_cg(rdev, false); + } +} + +static void vce_v2_0_init_cg(struct radeon_device *rdev) +{ + u32 tmp; + + tmp = RREG32(VCE_CLOCK_GATING_A); + tmp &= ~(CGC_CLK_GATE_DLY_TIMER_MASK | CGC_CLK_GATER_OFF_DLY_TIMER_MASK); + tmp |= (CGC_CLK_GATE_DLY_TIMER(0) | CGC_CLK_GATER_OFF_DLY_TIMER(4)); + tmp |= CGC_UENC_WAIT_AWAKE; + WREG32(VCE_CLOCK_GATING_A, tmp); + + tmp = RREG32(VCE_UENC_CLOCK_GATING); + tmp &= ~(CLOCK_ON_DELAY_MASK | CLOCK_OFF_DELAY_MASK); + tmp |= (CLOCK_ON_DELAY(0) | CLOCK_OFF_DELAY(4)); + WREG32(VCE_UENC_CLOCK_GATING, tmp); + + tmp = RREG32(VCE_CLOCK_GATING_B); + tmp |= 0x10; + tmp &= ~0x100000; + WREG32(VCE_CLOCK_GATING_B, tmp); +} + +unsigned vce_v2_0_bo_size(struct radeon_device *rdev) +{ + WARN_ON(rdev->vce_fw->size > VCE_V2_0_FW_SIZE); + return VCE_V2_0_FW_SIZE + VCE_V2_0_STACK_SIZE + VCE_V2_0_DATA_SIZE; +} + +int vce_v2_0_resume(struct radeon_device *rdev) +{ + uint64_t addr = rdev->vce.gpu_addr; + uint32_t size; + + WREG32_P(VCE_CLOCK_GATING_A, 0, ~(1 << 16)); + WREG32_P(VCE_UENC_CLOCK_GATING, 0x1FF000, ~0xFF9FF000); + WREG32_P(VCE_UENC_REG_CLOCK_GATING, 0x3F, ~0x3F); + WREG32(VCE_CLOCK_GATING_B, 0xf7); + + WREG32(VCE_LMI_CTRL, 0x00398000); + WREG32_P(VCE_LMI_CACHE_CTRL, 0x0, ~0x1); + WREG32(VCE_LMI_SWAP_CNTL, 0); + WREG32(VCE_LMI_SWAP_CNTL1, 0); + WREG32(VCE_LMI_VM_CTRL, 0); + + WREG32(VCE_LMI_VCPU_CACHE_40BIT_BAR, addr >> 8); + + addr &= 0xff; + size = VCE_V2_0_FW_SIZE; + WREG32(VCE_VCPU_CACHE_OFFSET0, addr & 0x7fffffff); + WREG32(VCE_VCPU_CACHE_SIZE0, size); + + addr += size; + size = VCE_V2_0_STACK_SIZE; + WREG32(VCE_VCPU_CACHE_OFFSET1, addr & 0x7fffffff); + WREG32(VCE_VCPU_CACHE_SIZE1, size); + + addr += size; + size = VCE_V2_0_DATA_SIZE; + WREG32(VCE_VCPU_CACHE_OFFSET2, addr & 0x7fffffff); + WREG32(VCE_VCPU_CACHE_SIZE2, size); + + WREG32_P(VCE_LMI_CTRL2, 0x0, ~0x100); + + WREG32_P(VCE_SYS_INT_EN, VCE_SYS_INT_TRAP_INTERRUPT_EN, + ~VCE_SYS_INT_TRAP_INTERRUPT_EN); + + vce_v2_0_init_cg(rdev); + + return 0; +} diff --git a/sys/dev/pci/drm/radeon_drm.h b/sys/dev/pci/drm/radeon_drm.h index 109e93d191d..7872a8d30bf 100644 --- a/sys/dev/pci/drm/radeon_drm.h +++ b/sys/dev/pci/drm/radeon_drm.h @@ -1,4 +1,3 @@ -/* $OpenBSD: radeon_drm.h,v 1.14 2018/04/20 16:09:36 deraadt Exp $ */ /* radeon_drm.h -- Public header for the radeon driver -*- linux-c -*- * * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas. @@ -512,6 +511,7 @@ typedef struct { #define DRM_RADEON_GEM_BUSY 0x2a #define DRM_RADEON_GEM_VA 0x2b #define DRM_RADEON_GEM_OP 0x2c +#define DRM_RADEON_GEM_USERPTR 0x2d #define DRM_IOCTL_RADEON_CP_INIT DRM_IOW( DRM_COMMAND_BASE + DRM_RADEON_CP_INIT, drm_radeon_init_t) #define DRM_IOCTL_RADEON_CP_START DRM_IO( DRM_COMMAND_BASE + DRM_RADEON_CP_START) @@ -555,6 +555,7 @@ typedef struct { #define DRM_IOCTL_RADEON_GEM_BUSY DRM_IOWR(DRM_COMMAND_BASE + DRM_RADEON_GEM_BUSY, struct drm_radeon_gem_busy) #define DRM_IOCTL_RADEON_GEM_VA DRM_IOWR(DRM_COMMAND_BASE + DRM_RADEON_GEM_VA, struct drm_radeon_gem_va) #define DRM_IOCTL_RADEON_GEM_OP DRM_IOWR(DRM_COMMAND_BASE + DRM_RADEON_GEM_OP, struct drm_radeon_gem_op) +#define DRM_IOCTL_RADEON_GEM_USERPTR DRM_IOWR(DRM_COMMAND_BASE + DRM_RADEON_GEM_USERPTR, struct drm_radeon_gem_userptr) typedef struct drm_radeon_init { enum { @@ -715,7 +716,6 @@ typedef struct drm_radeon_indirect { #define RADEON_PARAM_NUM_GB_PIPES 15 /* num GB pipes */ #define RADEON_PARAM_DEVICE_ID 16 #define RADEON_PARAM_NUM_Z_PIPES 17 /* num Z pipes */ -#define RADEON_PARAM_LAST_SWI 18 typedef struct drm_radeon_getparam { int param; @@ -798,7 +798,13 @@ struct drm_radeon_gem_info { uint64_t vram_visible; }; -#define RADEON_GEM_NO_BACKING_STORE 1 +#define RADEON_GEM_NO_BACKING_STORE (1 << 0) +#define RADEON_GEM_GTT_UC (1 << 1) +#define RADEON_GEM_GTT_WC (1 << 2) +/* BO is expected to be accessed by the CPU */ +#define RADEON_GEM_CPU_ACCESS (1 << 3) +/* CPU access is not expected to work for this BO */ +#define RADEON_GEM_NO_CPU_ACCESS (1 << 4) struct drm_radeon_gem_create { uint64_t size; @@ -808,10 +814,26 @@ struct drm_radeon_gem_create { uint32_t flags; }; +/* + * This is not a reliable API and you should expect it to fail for any + * number of reasons and have fallback path that do not use userptr to + * perform any operation. + */ +#define RADEON_GEM_USERPTR_READONLY (1 << 0) +#define RADEON_GEM_USERPTR_ANONONLY (1 << 1) +#define RADEON_GEM_USERPTR_VALIDATE (1 << 2) +#define RADEON_GEM_USERPTR_REGISTER (1 << 3) + +struct drm_radeon_gem_userptr { + uint64_t addr; + uint64_t size; + uint32_t flags; + uint32_t handle; +}; + #define RADEON_TILING_MACRO 0x1 #define RADEON_TILING_MICRO 0x2 #define RADEON_TILING_SWAP_16BIT 0x4 -#define RADEON_TILING_R600_NO_SCANOUT RADEON_TILING_SWAP_16BIT #define RADEON_TILING_SWAP_32BIT 0x8 /* this object requires a surface when mapped - i.e. front buffer */ #define RADEON_TILING_SURFACE 0x10 @@ -924,7 +946,6 @@ struct drm_radeon_gem_va { #define RADEON_CHUNK_ID_IB 0x02 #define RADEON_CHUNK_ID_FLAGS 0x03 #define RADEON_CHUNK_ID_CONST_IB 0x04 -#define RADEON_CHUNK_ID_OLD 0xff /* The first dword of RADEON_CHUNK_ID_FLAGS is a uint32 of these flags: */ #define RADEON_CS_KEEP_TILING_FLAGS 0x01 @@ -946,6 +967,7 @@ struct drm_radeon_cs_chunk { }; /* drm_radeon_cs_reloc.flags */ +#define RADEON_RELOC_PRIO_MASK (0xf << 0) struct drm_radeon_cs_reloc { uint32_t handle; @@ -1011,7 +1033,13 @@ struct drm_radeon_cs { #define RADEON_INFO_NUM_BYTES_MOVED 0x1d #define RADEON_INFO_VRAM_USAGE 0x1e #define RADEON_INFO_GTT_USAGE 0x1f - +#define RADEON_INFO_ACTIVE_CU_COUNT 0x20 +#define RADEON_INFO_CURRENT_GPU_TEMP 0x21 +#define RADEON_INFO_CURRENT_GPU_SCLK 0x22 +#define RADEON_INFO_CURRENT_GPU_MCLK 0x23 +#define RADEON_INFO_READ_REG 0x24 +#define RADEON_INFO_VA_UNMAP_WORKING 0x25 +#define RADEON_INFO_GPU_RESET_COUNTER 0x26 struct drm_radeon_info { uint32_t request; @@ -1037,13 +1065,6 @@ struct drm_radeon_info { #define SI_TILE_MODE_DEPTH_STENCIL_2D_4AA 3 #define SI_TILE_MODE_DEPTH_STENCIL_2D_8AA 2 -#define CIK_TILE_MODE_COLOR_2D 14 -#define CIK_TILE_MODE_COLOR_2D_SCANOUT 10 -#define CIK_TILE_MODE_DEPTH_STENCIL_2D_TILESPLIT_64 0 -#define CIK_TILE_MODE_DEPTH_STENCIL_2D_TILESPLIT_128 1 -#define CIK_TILE_MODE_DEPTH_STENCIL_2D_TILESPLIT_256 2 -#define CIK_TILE_MODE_DEPTH_STENCIL_2D_TILESPLIT_512 3 -#define CIK_TILE_MODE_DEPTH_STENCIL_2D_TILESPLIT_ROW_SIZE 4 #define CIK_TILE_MODE_DEPTH_STENCIL_1D 5 #endif diff --git a/sys/dev/pci/drm/ttm/ttm_agp_backend.c b/sys/dev/pci/drm/ttm/ttm_agp_backend.c index 9ece4dee0ec..571453e695d 100644 --- a/sys/dev/pci/drm/ttm/ttm_agp_backend.c +++ b/sys/dev/pci/drm/ttm/ttm_agp_backend.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_agp_backend.c,v 1.5 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -32,6 +31,7 @@ #define pr_fmt(fmt) "[TTM] " fmt +#include #include #include #include diff --git a/sys/dev/pci/drm/ttm/ttm_bo.c b/sys/dev/pci/drm/ttm/ttm_bo.c index d0a49f6f715..7f03a515f03 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo.c +++ b/sys/dev/pci/drm/ttm/ttm_bo.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_bo.c,v 1.22 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -34,18 +33,18 @@ #include #include #include +#include +#include +#include +#include #define TTM_ASSERT_LOCKED(param) #define TTM_DEBUG(fmt, arg...) #define TTM_BO_HASH_ORDER 13 -static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); static void ttm_bo_global_kobj_release(struct kobject *kobj); -int ttm_bo_move_buffer(struct ttm_buffer_object *, struct ttm_placement *, - bool, bool); - #ifdef notyet static struct attribute ttm_bo_count = { .name = "bo_count", @@ -59,12 +58,13 @@ ttm_get_kobj(void) return (NULL); } -static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) +static inline int ttm_mem_type_from_place(const struct ttm_place *place, + uint32_t *mem_type) { int i; for (i = 0; i <= TTM_PL_PRIV5; i++) - if (flags & (1 << i)) { + if (place->flags & (1 << i)) { *mem_type = i; return 0; } @@ -78,7 +78,7 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) pr_err(" has_type: %d\n", man->has_type); pr_err(" use_type: %d\n", man->use_type); pr_err(" flags: 0x%08X\n", man->flags); - pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); + pr_err(" gpu_offset: 0x%08llX\n", man->gpu_offset); pr_err(" size: %llu\n", man->size); pr_err(" available_caching: 0x%08X\n", man->available_caching); pr_err(" default_caching: 0x%08X\n", man->default_caching); @@ -95,12 +95,12 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, bo, bo->mem.num_pages, bo->mem.size >> 10, bo->mem.size >> 20); for (i = 0; i < placement->num_placement; i++) { - ret = ttm_mem_type_from_flags(placement->placement[i], + ret = ttm_mem_type_from_place(&placement->placement[i], &mem_type); if (ret) return; pr_err(" placement[%d]=0x%08X (%d)\n", - i, placement->placement[i], mem_type); + i, placement->placement[i].flags, mem_type); ttm_mem_type_debug(bo->bdev, mem_type); } } @@ -135,6 +135,7 @@ static struct kobj_type ttm_bo_glob_kobj_type = { #endif }; + static inline uint32_t ttm_bo_type_flags(unsigned type) { return 1 << (type); @@ -150,7 +151,6 @@ static void ttm_bo_release_list(struct kref *list_kref) BUG_ON(atomic_read(&bo->list_kref.refcount)); BUG_ON(atomic_read(&bo->kref.refcount)); BUG_ON(atomic_read(&bo->cpu_writers)); - BUG_ON(bo->sync_obj != NULL); BUG_ON(bo->mem.mm_node != NULL); BUG_ON(!list_empty(&bo->lru)); BUG_ON(!list_empty(&bo->ddestroy)); @@ -158,6 +158,9 @@ static void ttm_bo_release_list(struct kref *list_kref) if (bo->ttm) ttm_tt_destroy(bo->ttm); atomic_dec(&bo->glob->bo_count); + if (bo->resv == &bo->ttm_resv) + reservation_object_fini(&bo->ttm_resv); + mutex_destroy(&bo->wu_mutex); if (bo->destroy) bo->destroy(bo); else { @@ -166,28 +169,14 @@ static void ttm_bo_release_list(struct kref *list_kref) ttm_mem_global_free(bdev->glob->mem_glob, acc_size); } -int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) -{ - int ret = 0; - - while (ret == 0) { - if (!ttm_bo_is_reserved(bo)) - break; - ret = -tsleep(&bo->event_queue, - PZERO | (interruptible ? PCATCH : 0), "ttmwt", 0); - - } - - return (ret); -} -EXPORT_SYMBOL(ttm_bo_wait_unreserved); - void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; - BUG_ON(!ttm_bo_is_reserved(bo)); +#ifdef notyet + lockdep_assert_held(&bo->resv->lock.base); +#endif if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { @@ -203,6 +192,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) } } } +EXPORT_SYMBOL(ttm_bo_add_to_lru); int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) { @@ -225,62 +215,6 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) return put_count; } -int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, uint32_t sequence) -{ - struct ttm_bo_global *glob = bo->glob; - int ret; - - while (unlikely(atomic_read(&bo->reserved) != 0)) { - /** - * Deadlock avoidance for multi-bo reserving. - */ - if (use_sequence && bo->seq_valid) { - /** - * We've already reserved this one. - */ - if (unlikely(sequence == bo->val_seq)) - return -EDEADLK; - /** - * Already reserved by a thread that will not back - * off for us. We need to back off. - */ - if (unlikely(sequence - bo->val_seq < (1 << 31))) - return -EAGAIN; - } - - if (no_wait) - return -EBUSY; - - spin_unlock(&glob->lru_lock); - ret = ttm_bo_wait_unreserved(bo, interruptible); - spin_lock(&glob->lru_lock); - - if (unlikely(ret)) - return ret; - } - - atomic_set(&bo->reserved, 1); - if (use_sequence) { - /** - * Wake up waiters that may need to recheck for deadlock, - * if we decreased the sequence number. - */ - if (unlikely((bo->val_seq - sequence < (1 << 31)) - || !bo->seq_valid)) - wake_up_all(&bo->event_queue); - - bo->val_seq = sequence; - bo->seq_valid = true; - } else { - bo->seq_valid = false; - } - - return 0; -} -EXPORT_SYMBOL(ttm_bo_reserve); - static void ttm_bo_ref_bug(struct kref *list_kref) { BUG(); @@ -293,42 +227,16 @@ void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); } -int ttm_bo_reserve(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, uint32_t sequence) +void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) { - struct ttm_bo_global *glob = bo->glob; - int put_count = 0; - int ret; - - spin_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, - sequence); - if (likely(ret == 0)) - put_count = ttm_bo_del_from_lru(bo); - spin_unlock(&glob->lru_lock); + int put_count; + spin_lock(&bo->glob->lru_lock); + put_count = ttm_bo_del_from_lru(bo); + spin_unlock(&bo->glob->lru_lock); ttm_bo_list_ref_sub(bo, put_count, true); - - return ret; -} - -void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) -{ - ttm_bo_add_to_lru(bo); - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); -} - -void ttm_bo_unreserve(struct ttm_buffer_object *bo) -{ - struct ttm_bo_global *glob = bo->glob; - - spin_lock(&glob->lru_lock); - ttm_bo_unreserve_locked(bo); - spin_unlock(&glob->lru_lock); } -EXPORT_SYMBOL(ttm_bo_unreserve); +EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); /* * Call bo->mutex locked. @@ -340,9 +248,7 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) int ret = 0; uint32_t page_flags = 0; -#ifdef notyet - rw_assert_wrlock(&bo->mutex); -#endif + TTM_ASSERT_LOCKED(&bo->mutex); bo->ttm = NULL; if (bdev->need_dma32) @@ -504,61 +410,69 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) } ttm_bo_mem_put(bo, &bo->mem); - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + ww_mutex_unlock (&bo->resv->lock); +} - /* - * Since the final reference to this bo may not be dropped by - * the current task we have to put a memory barrier here to make - * sure the changes done in this function are always visible. - * - * This function only needs protection against the final kref_put. - */ - smp_mb__before_atomic_dec(); +static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) +{ + struct reservation_object_list *fobj; + struct fence *fence; + int i; + + fobj = reservation_object_get_list(bo->resv); + fence = reservation_object_get_excl(bo->resv); + if (fence && !fence->ops->signaled) + fence_enable_sw_signaling(fence); + + for (i = 0; fobj && i < fobj->shared_count; ++i) { + fence = rcu_dereference_protected(fobj->shared[i], + reservation_object_held(bo->resv)); + + if (!fence->ops->signaled) + fence_enable_sw_signaling(fence); + } } static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_global *glob = bo->glob; - struct ttm_bo_driver *driver = bdev->driver; - void *sync_obj = NULL; int put_count; int ret; spin_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + ret = __ttm_bo_reserve(bo, false, true, false, NULL); - spin_lock(&bdev->fence_lock); - (void) ttm_bo_wait(bo, false, false, true); - if (!ret && !bo->sync_obj) { - spin_unlock(&bdev->fence_lock); - put_count = ttm_bo_del_from_lru(bo); + if (!ret) { + if (!ttm_bo_wait(bo, false, false, true)) { + put_count = ttm_bo_del_from_lru(bo); - spin_unlock(&glob->lru_lock); - ttm_bo_cleanup_memtype_use(bo); + spin_unlock(&glob->lru_lock); + ttm_bo_cleanup_memtype_use(bo); - ttm_bo_list_ref_sub(bo, put_count, true); + ttm_bo_list_ref_sub(bo, put_count, true); - return; - } - if (bo->sync_obj) - sync_obj = driver->sync_obj_ref(bo->sync_obj); - spin_unlock(&bdev->fence_lock); + return; + } else + ttm_bo_flush_all_fences(bo); - if (!ret) { - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + /* + * Make NO_EVICT bos immediately available to + * shrinkers, now that they are queued for + * destruction. + */ + if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { + bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; + ttm_bo_add_to_lru(bo); + } + + __ttm_bo_unreserve(bo); } kref_get(&bo->list_kref); list_add_tail(&bo->ddestroy, &bdev->ddestroy); spin_unlock(&glob->lru_lock); - if (sync_obj) { - driver->sync_obj_flush(sync_obj); - driver->sync_obj_unref(&sync_obj); - } schedule_delayed_work(&bdev->wq, ((HZ / 100) < 1) ? 1 : HZ / 100); } @@ -579,48 +493,29 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, bool interruptible, bool no_wait_gpu) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_driver *driver = bdev->driver; struct ttm_bo_global *glob = bo->glob; int put_count; int ret; - spin_lock(&bdev->fence_lock); ret = ttm_bo_wait(bo, false, false, true); if (ret && !no_wait_gpu) { - void *sync_obj; - - /* - * Take a reference to the fence and unreserve, - * at this point the buffer should be dead, so - * no new sync objects can be attached. - */ - sync_obj = driver->sync_obj_ref(bo->sync_obj); - spin_unlock(&bdev->fence_lock); - - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + long lret; + ww_mutex_unlock(&bo->resv->lock); spin_unlock(&glob->lru_lock); - ret = driver->sync_obj_wait(sync_obj, false, interruptible); - driver->sync_obj_unref(&sync_obj); - if (ret) - return ret; + lret = reservation_object_wait_timeout_rcu(bo->resv, + true, + interruptible, + 30 * HZ); - /* - * remove sync_obj with ttm_bo_wait, the wait should be - * finished, and no new wait object should have been added. - */ - spin_lock(&bdev->fence_lock); - ret = ttm_bo_wait(bo, false, false, true); - WARN_ON(ret); - spin_unlock(&bdev->fence_lock); - if (ret) - return ret; + if (lret < 0) + return lret; + else if (lret == 0) + return -EBUSY; spin_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + ret = __ttm_bo_reserve(bo, false, true, false, NULL); /* * We raced, and lost, someone else holds the reservation now, @@ -634,12 +529,17 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, spin_unlock(&glob->lru_lock); return 0; } - } else - spin_unlock(&bdev->fence_lock); + + /* + * remove sync_obj with ttm_bo_wait, the wait should be + * finished, and no new wait object should have been added. + */ + ret = ttm_bo_wait(bo, false, false, true); + WARN_ON(ret); + } if (ret || unlikely(list_empty(&bo->ddestroy))) { - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + __ttm_bo_unreserve(bo); spin_unlock(&glob->lru_lock); return ret; } @@ -684,7 +584,14 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) kref_get(&nentry->list_kref); } - ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0); + ret = __ttm_bo_reserve(entry, false, true, false, NULL); + if (remove_all && ret) { + spin_unlock(&glob->lru_lock); + ret = __ttm_bo_reserve(entry, false, false, + false, NULL); + spin_lock(&glob->lru_lock); + } + if (!ret) ret = ttm_bo_cleanup_refs_and_unlock(entry, false, !remove_all); @@ -767,9 +674,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, struct ttm_placement placement; int ret = 0; - spin_lock(&bdev->fence_lock); ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); - spin_unlock(&bdev->fence_lock); if (unlikely(ret != 0)) { if (ret != -ERESTARTSYS) { @@ -778,15 +683,15 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, goto out; } - BUG_ON(!ttm_bo_is_reserved(bo)); +#ifdef notyet + lockdep_assert_held(&bo->resv->lock.base); +#endif evict_mem = bo->mem; evict_mem.mm_node = NULL; evict_mem.bus.io_reserved_vm = false; evict_mem.bus.io_reserved_count = 0; - placement.fpfn = 0; - placement.lpfn = 0; placement.num_placement = 0; placement.num_busy_placement = 0; bdev->driver->evict_flags(bo, &placement); @@ -816,6 +721,7 @@ out: static int ttm_mem_evict_first(struct ttm_bo_device *bdev, uint32_t mem_type, + const struct ttm_place *place, bool interruptible, bool no_wait_gpu) { @@ -826,9 +732,22 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, spin_lock(&glob->lru_lock); list_for_each_entry(bo, &man->lru, lru) { - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); - if (!ret) + ret = __ttm_bo_reserve(bo, false, true, false, NULL); + if (!ret) { + if (place && (place->fpfn || place->lpfn)) { + /* Don't evict this BO if it's outside of the + * requested placement range + */ + if (place->fpfn >= (bo->mem.start + bo->mem.size) || + (place->lpfn && place->lpfn <= bo->mem.start)) { + __ttm_bo_unreserve(bo); + ret = -EBUSY; + continue; + } + } + break; + } } if (ret) { @@ -874,7 +793,7 @@ EXPORT_SYMBOL(ttm_bo_mem_put); */ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, uint32_t mem_type, - struct ttm_placement *placement, + const struct ttm_place *place, struct ttm_mem_reg *mem, bool interruptible, bool no_wait_gpu) @@ -884,12 +803,12 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, int ret; do { - ret = (*man->func->get_node)(man, bo, placement, mem); + ret = (*man->func->get_node)(man, bo, place, mem); if (unlikely(ret != 0)) return ret; if (mem->mm_node) break; - ret = ttm_mem_evict_first(bdev, mem_type, + ret = ttm_mem_evict_first(bdev, mem_type, place, interruptible, no_wait_gpu); if (unlikely(ret != 0)) return ret; @@ -927,18 +846,18 @@ static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, uint32_t mem_type, - uint32_t proposed_placement, + const struct ttm_place *place, uint32_t *masked_placement) { uint32_t cur_flags = ttm_bo_type_flags(mem_type); - if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) + if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) return false; - if ((proposed_placement & man->available_caching) == 0) + if ((place->flags & man->available_caching) == 0) return false; - cur_flags |= (proposed_placement & man->available_caching); + cur_flags |= (place->flags & man->available_caching); *masked_placement = cur_flags; return true; @@ -969,38 +888,38 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, mem->mm_node = NULL; for (i = 0; i < placement->num_placement; ++i) { - ret = ttm_mem_type_from_flags(placement->placement[i], - &mem_type); + const struct ttm_place *place = &placement->placement[i]; + + ret = ttm_mem_type_from_place(place, &mem_type); if (ret) return ret; man = &bdev->man[mem_type]; + if (!man->has_type || !man->use_type) + continue; - type_ok = ttm_bo_mt_compatible(man, - mem_type, - placement->placement[i], + type_ok = ttm_bo_mt_compatible(man, mem_type, place, &cur_flags); if (!type_ok) continue; + type_found = true; cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags); /* * Use the access and other non-mapping-related flag bits from * the memory placement flags to the current flags */ - ttm_flag_masked(&cur_flags, placement->placement[i], + ttm_flag_masked(&cur_flags, place->flags, ~TTM_PL_MASK_MEMTYPE); if (mem_type == TTM_PL_SYSTEM) break; - if (man->has_type && man->use_type) { - type_found = true; - ret = (*man->func->get_node)(man, bo, placement, mem); - if (unlikely(ret)) - return ret; - } + ret = (*man->func->get_node)(man, bo, place, mem); + if (unlikely(ret)) + return ret; + if (mem->mm_node) break; } @@ -1011,33 +930,28 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, return 0; } - if (!type_found) - return -EINVAL; - for (i = 0; i < placement->num_busy_placement; ++i) { - ret = ttm_mem_type_from_flags(placement->busy_placement[i], - &mem_type); + const struct ttm_place *place = &placement->busy_placement[i]; + + ret = ttm_mem_type_from_place(place, &mem_type); if (ret) return ret; man = &bdev->man[mem_type]; - if (!man->has_type) + if (!man->has_type || !man->use_type) continue; - if (!ttm_bo_mt_compatible(man, - mem_type, - placement->busy_placement[i], - &cur_flags)) + if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags)) continue; + type_found = true; cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags); /* * Use the access and other non-mapping-related flag bits from * the memory placement flags to the current flags */ - ttm_flag_masked(&cur_flags, placement->busy_placement[i], + ttm_flag_masked(&cur_flags, place->flags, ~TTM_PL_MASK_MEMTYPE); - if (mem_type == TTM_PL_SYSTEM) { mem->mem_type = mem_type; mem->placement = cur_flags; @@ -1045,39 +959,43 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, return 0; } - ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, + ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, interruptible, no_wait_gpu); if (ret == 0 && mem->mm_node) { mem->placement = cur_flags; return 0; } - if (ret == -ERESTART) + if (ret == -ERESTARTSYS) has_erestartsys = true; } - ret = (has_erestartsys) ? -ERESTART: -ENOMEM; - return ret; + + if (!type_found) { + printk(KERN_ERR TTM_PFX "No compatible memory type found.\n"); + return -EINVAL; + } + + return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; } EXPORT_SYMBOL(ttm_bo_mem_space); -int ttm_bo_move_buffer(struct ttm_buffer_object *bo, +static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, struct ttm_placement *placement, bool interruptible, bool no_wait_gpu) { int ret = 0; struct ttm_mem_reg mem; - struct ttm_bo_device *bdev = bo->bdev; - BUG_ON(!ttm_bo_is_reserved(bo)); +#ifdef notyet + lockdep_assert_held(&bo->resv->lock.base); +#endif /* * FIXME: It's possible to pipeline buffer moves. * Have the driver move function wait for idle when necessary, * instead of doing it here. */ - spin_lock(&bdev->fence_lock); ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); - spin_unlock(&bdev->fence_lock); if (ret) return ret; mem.num_pages = bo->num_pages; @@ -1100,26 +1018,33 @@ out_unlock: return ret; } -static bool ttm_bo_mem_compat(struct ttm_placement *placement, - struct ttm_mem_reg *mem, - uint32_t *new_flags) +bool ttm_bo_mem_compat(struct ttm_placement *placement, + struct ttm_mem_reg *mem, + uint32_t *new_flags) { int i; - if (mem->mm_node && placement->lpfn != 0 && - (mem->start < placement->fpfn || - mem->start + mem->num_pages > placement->lpfn)) - return false; - for (i = 0; i < placement->num_placement; i++) { - *new_flags = placement->placement[i]; + const struct ttm_place *heap = &placement->placement[i]; + if (mem->mm_node && + (mem->start < heap->fpfn || + (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) + continue; + + *new_flags = heap->flags; if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && (*new_flags & mem->placement & TTM_PL_MASK_MEM)) return true; } for (i = 0; i < placement->num_busy_placement; i++) { - *new_flags = placement->busy_placement[i]; + const struct ttm_place *heap = &placement->busy_placement[i]; + if (mem->mm_node && + (mem->start < heap->fpfn || + (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) + continue; + + *new_flags = heap->flags; if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && (*new_flags & mem->placement & TTM_PL_MASK_MEM)) return true; @@ -1127,6 +1052,7 @@ static bool ttm_bo_mem_compat(struct ttm_placement *placement, return false; } +EXPORT_SYMBOL(ttm_bo_mem_compat); int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, @@ -1136,12 +1062,9 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, int ret; uint32_t new_flags; - BUG_ON(!ttm_bo_is_reserved(bo)); - /* Check that range is valid */ - if (placement->lpfn || placement->fpfn) - if (placement->fpfn > placement->lpfn || - (placement->lpfn - placement->fpfn) < bo->num_pages) - return -EINVAL; +#ifdef notyet + lockdep_assert_held(&bo->resv->lock.base); +#endif /* * Check whether we need to move buffer. */ @@ -1170,15 +1093,6 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_validate); -int ttm_bo_check_placement(struct ttm_buffer_object *bo, - struct ttm_placement *placement) -{ - BUG_ON((placement->fpfn || placement->lpfn) && - (bo->mem.num_pages > (placement->lpfn - placement->fpfn))); - - return 0; -} - int ttm_bo_init(struct ttm_bo_device *bdev, struct ttm_buffer_object *bo, unsigned long size, @@ -1189,11 +1103,13 @@ int ttm_bo_init(struct ttm_bo_device *bdev, struct uvm_object *persistent_swap_storage, size_t acc_size, struct sg_table *sg, + struct reservation_object *resv, void (*destroy) (struct ttm_buffer_object *)) { int ret = 0; unsigned long num_pages; struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; + bool locked; ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); if (ret) { @@ -1221,12 +1137,11 @@ int ttm_bo_init(struct ttm_bo_device *bdev, kref_init(&bo->kref); kref_init(&bo->list_kref); atomic_set(&bo->cpu_writers, 0); - atomic_set(&bo->reserved, 1); - init_waitqueue_head(&bo->event_queue); INIT_LIST_HEAD(&bo->lru); INIT_LIST_HEAD(&bo->ddestroy); INIT_LIST_HEAD(&bo->swap); INIT_LIST_HEAD(&bo->io_reserve_lru); + rw_init(&bo->wu_mutex, "ttmwu"); bo->bdev = bdev; bo->glob = bdev->glob; bo->type = type; @@ -1240,38 +1155,46 @@ int ttm_bo_init(struct ttm_bo_device *bdev, bo->mem.bus.io_reserved_count = 0; bo->priv_flags = 0; bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); - bo->seq_valid = false; bo->persistent_swap_storage = persistent_swap_storage; bo->acc_size = acc_size; bo->sg = sg; + if (resv) { + bo->resv = resv; +#ifdef notyet + lockdep_assert_held(&bo->resv->lock.base); +#endif + } else { + bo->resv = &bo->ttm_resv; + reservation_object_init(&bo->ttm_resv); + } atomic_inc(&bo->glob->bo_count); drm_vma_node_reset(&bo->vma_node); - ret = ttm_bo_check_placement(bo, placement); - if (unlikely(ret != 0)) - goto out_err; - /* * For ttm_bo_type_device buffers, allocate * address space from the device. */ if (bo->type == ttm_bo_type_device || - bo->type == ttm_bo_type_sg) { - ret = ttm_bo_setup_vm(bo); - if (ret) - goto out_err; + bo->type == ttm_bo_type_sg) + ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, + bo->mem.num_pages); + + /* passed reservation objects should already be locked, + * since otherwise lockdep will be angered in radeon. + */ + if (!resv) { + locked = ww_mutex_trylock(&bo->resv->lock); + WARN_ON(!locked); } - ret = ttm_bo_validate(bo, placement, interruptible, false); - if (ret) - goto out_err; + if (likely(!ret)) + ret = ttm_bo_validate(bo, placement, interruptible, false); - ttm_bo_unreserve(bo); - return 0; + if (!resv) + ttm_bo_unreserve(bo); -out_err: - ttm_bo_unreserve(bo); - ttm_bo_unref(&bo); + if (unlikely(ret)) + ttm_bo_unref(&bo); return ret; } @@ -1300,7 +1223,7 @@ size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, size += ttm_round_pot(struct_size); size += PAGE_ALIGN(npages * sizeof(void *)); - size += PAGE_ALIGN(npages * sizeof(bus_addr_t)); + size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); size += ttm_round_pot(sizeof(struct ttm_dma_tt)); return size; } @@ -1326,7 +1249,7 @@ int ttm_bo_create(struct ttm_bo_device *bdev, acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, interruptible, persistent_swap_storage, acc_size, - NULL, NULL); + NULL, NULL, NULL); if (likely(ret == 0)) *p_bo = bo; @@ -1348,7 +1271,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, spin_lock(&glob->lru_lock); while (!list_empty(&man->lru)) { spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, false, false); + ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); if (ret) { if (allow_errors) { return ret; @@ -1422,7 +1345,7 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, BUG_ON(man->has_type); man->io_reserve_fastpath = true; man->use_io_reserve_lru = false; - rw_init(&man->io_reserve_mutex, "ttm_iores"); + rw_init(&man->io_reserve_mutex, "ior"); INIT_LIST_HEAD(&man->io_reserve_lru); ret = bdev->driver->init_mem_type(bdev, type, man); @@ -1452,7 +1375,7 @@ static void ttm_bo_global_kobj_release(struct kobject *kobj) container_of(kobj, struct ttm_bo_global, kobj); ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); - km_free(glob->dummy_read_page, PAGE_SIZE, &kv_any, &kp_dma_zero); + __free_page(glob->dummy_read_page); kfree(glob); } @@ -1472,11 +1395,10 @@ int ttm_bo_global_init(struct drm_global_reference *ref) struct ttm_bo_global *glob = ref->object; int ret; - rw_init(&glob->device_list_mutex, "ttm_devlist"); + rw_init(&glob->device_list_mutex, "gdl"); mtx_init(&glob->lru_lock, IPL_NONE); glob->mem_glob = bo_ref->mem_glob; - glob->dummy_read_page = km_alloc(PAGE_SIZE, &kv_any, &kp_dma_zero, - &kd_waitok); + glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); if (unlikely(glob->dummy_read_page == NULL)) { ret = -ENOMEM; @@ -1501,7 +1423,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref) kobject_put(&glob->kobj); return ret; out_no_shrink: - km_free(glob->dummy_read_page, PAGE_SIZE, &kv_any, &kp_dma_zero); + __free_page(glob->dummy_read_page); out_no_drp: kfree(glob); return ret; @@ -1555,6 +1477,7 @@ EXPORT_SYMBOL(ttm_bo_device_release); int ttm_bo_device_init(struct ttm_bo_device *bdev, struct ttm_bo_global *glob, struct ttm_bo_driver *driver, + struct address_space *mapping, uint64_t file_page_offset, bool need_dma32) { @@ -1576,11 +1499,10 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, 0x10000000); INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); INIT_LIST_HEAD(&bdev->ddestroy); - bdev->dev_mapping = NULL; + bdev->dev_mapping = mapping; bdev->glob = glob; bdev->need_dma32 = need_dma32; bdev->val_seq = 0; - mtx_init(&bdev->fence_lock, IPL_NONE); mutex_lock(&glob->device_list_mutex); list_add_tail(&bdev->device_list, &glob->device_list); mutex_unlock(&glob->device_list_mutex); @@ -1612,6 +1534,15 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) return true; } +#ifdef __linux__ +void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + + drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); + ttm_mem_io_free_vm(bo); +} +#else void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) { struct ttm_tt *ttm = bo->ttm; @@ -1620,26 +1551,29 @@ void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) paddr_t paddr; int i; - if (bo->mem.bus.is_iomem) { - for (i = 0; i < bo->mem.num_pages; ++i) { - addr = bo->mem.bus.base + bo->mem.bus.offset; - paddr = bus_space_mmap(bo->bdev->memt, addr, - i << PAGE_SHIFT, 0, 0); - page = PHYS_TO_VM_PAGE(paddr); - if (unlikely(page == NULL)) - continue; - pmap_page_protect(page, PROT_NONE); - } - } else if (ttm) { - for (i = 0; i < ttm->num_pages; ++i) { - page = ttm->pages[i]; - if (unlikely(page == NULL)) - continue; - pmap_page_protect(page, PROT_NONE); + if (drm_vma_node_has_offset(&bo->vma_node)) { + if (bo->mem.bus.is_iomem) { + for (i = 0; i < bo->mem.num_pages; ++i) { + addr = bo->mem.bus.base + bo->mem.bus.offset; + paddr = bus_space_mmap(bo->bdev->memt, addr, + i << PAGE_SHIFT, 0, 0); + page = PHYS_TO_VM_PAGE(paddr); + if (unlikely(page == NULL)) + continue; + pmap_page_protect(page, PROT_NONE); + } + } else if (ttm) { + for (i = 0; i < ttm->num_pages; ++i) { + page = ttm->pages[i]; + if (unlikely(page == NULL)) + continue; + pmap_page_protect(page, PROT_NONE); + } } } ttm_mem_io_free_vm(bo); } +#endif void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) { @@ -1654,95 +1588,66 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) EXPORT_SYMBOL(ttm_bo_unmap_virtual); -/** - * ttm_bo_setup_vm: - * - * @bo: the buffer to allocate address space for - * - * Allocate address space in the drm device so that applications - * can mmap the buffer and access the contents. This only - * applies to ttm_bo_type_device objects as others are not - * placed in the drm device address space. - */ - -static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - - return drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, - bo->mem.num_pages); -} - int ttm_bo_wait(struct ttm_buffer_object *bo, bool lazy, bool interruptible, bool no_wait) { - struct ttm_bo_driver *driver = bo->bdev->driver; - struct ttm_bo_device *bdev = bo->bdev; - void *sync_obj; - int ret = 0; - - if (likely(bo->sync_obj == NULL)) - return 0; + struct reservation_object_list *fobj; + struct reservation_object *resv; + struct fence *excl; + long timeout = 15 * HZ; + int i; - while (bo->sync_obj) { + resv = bo->resv; + fobj = reservation_object_get_list(resv); + excl = reservation_object_get_excl(resv); + if (excl) { + if (!fence_is_signaled(excl)) { + if (no_wait) + return -EBUSY; - if (driver->sync_obj_signaled(bo->sync_obj)) { - void *tmp_obj = bo->sync_obj; - bo->sync_obj = NULL; - clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); - spin_unlock(&bdev->fence_lock); - driver->sync_obj_unref(&tmp_obj); - spin_lock(&bdev->fence_lock); - continue; + timeout = fence_wait_timeout(excl, + interruptible, timeout); } + } - if (no_wait) - return -EBUSY; + for (i = 0; fobj && timeout > 0 && i < fobj->shared_count; ++i) { + struct fence *fence; + fence = rcu_dereference_protected(fobj->shared[i], + reservation_object_held(resv)); - sync_obj = driver->sync_obj_ref(bo->sync_obj); - spin_unlock(&bdev->fence_lock); - ret = driver->sync_obj_wait(sync_obj, - lazy, interruptible); - if (unlikely(ret != 0)) { - driver->sync_obj_unref(&sync_obj); - spin_lock(&bdev->fence_lock); - return ret; - } - spin_lock(&bdev->fence_lock); - if (likely(bo->sync_obj == sync_obj)) { - void *tmp_obj = bo->sync_obj; - bo->sync_obj = NULL; - clear_bit(TTM_BO_PRIV_FLAG_MOVING, - &bo->priv_flags); - spin_unlock(&bdev->fence_lock); - driver->sync_obj_unref(&sync_obj); - driver->sync_obj_unref(&tmp_obj); - spin_lock(&bdev->fence_lock); - } else { - spin_unlock(&bdev->fence_lock); - driver->sync_obj_unref(&sync_obj); - spin_lock(&bdev->fence_lock); + if (!fence_is_signaled(fence)) { + if (no_wait) + return -EBUSY; + + timeout = fence_wait_timeout(fence, + interruptible, timeout); } } + + if (timeout < 0) + return timeout; + + if (timeout == 0) + return -EBUSY; + + reservation_object_add_excl_fence(resv, NULL); + clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); return 0; } EXPORT_SYMBOL(ttm_bo_wait); int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) { - struct ttm_bo_device *bdev = bo->bdev; int ret = 0; /* * Using ttm_bo_reserve makes sure the lru lists are updated. */ - ret = ttm_bo_reserve(bo, true, no_wait, false, 0); + ret = ttm_bo_reserve(bo, true, no_wait, false, NULL); if (unlikely(ret != 0)) return ret; - spin_lock(&bdev->fence_lock); ret = ttm_bo_wait(bo, false, true, no_wait); - spin_unlock(&bdev->fence_lock); if (likely(ret == 0)) atomic_inc(&bo->cpu_writers); ttm_bo_unreserve(bo); @@ -1768,11 +1673,10 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) struct ttm_buffer_object *bo; int ret = -EBUSY; int put_count; - uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); spin_lock(&glob->lru_lock); list_for_each_entry(bo, &glob->swap_lru, swap) { - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + ret = __ttm_bo_reserve(bo, false, true, false, NULL); if (!ret) break; } @@ -1799,14 +1703,13 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) * Wait for GPU, then move to system cached. */ - spin_lock(&bo->bdev->fence_lock); ret = ttm_bo_wait(bo, false, false, false); - spin_unlock(&bo->bdev->fence_lock); if (unlikely(ret != 0)) goto out; - if ((bo->mem.placement & swap_placement) != swap_placement) { + if (bo->mem.mem_type != TTM_PL_SYSTEM || + bo->ttm->caching_state != tt_cached) { struct ttm_mem_reg evict_mem; evict_mem = bo->mem; @@ -1839,8 +1742,7 @@ out: * already swapped buffer. */ - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + __ttm_bo_unreserve(bo); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -1851,3 +1753,35 @@ void ttm_bo_swapout_all(struct ttm_bo_device *bdev) ; } EXPORT_SYMBOL(ttm_bo_swapout_all); + +/** + * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become + * unreserved + * + * @bo: Pointer to buffer + */ +int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) +{ + int ret; + + /* + * In the absense of a wait_unlocked API, + * Use the bo::wu_mutex to avoid triggering livelocks due to + * concurrent use of this function. Note that this use of + * bo::wu_mutex can go away if we change locking order to + * mmap_sem -> bo::reserve. + */ + ret = mutex_lock_interruptible(&bo->wu_mutex); + if (unlikely(ret != 0)) + return -ERESTARTSYS; + if (!ww_mutex_is_locked(&bo->resv->lock)) + goto out_unlock; + ret = __ttm_bo_reserve(bo, true, false, false, NULL); + if (unlikely(ret != 0)) + goto out_unlock; + __ttm_bo_unreserve(bo); + +out_unlock: + mutex_unlock(&bo->wu_mutex); + return ret; +} diff --git a/sys/dev/pci/drm/ttm/ttm_bo_api.h b/sys/dev/pci/drm/ttm/ttm_bo_api.h index 8abe4cc8adf..e2fc62fc7df 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_api.h +++ b/sys/dev/pci/drm/ttm/ttm_bo_api.h @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_bo_api.h,v 1.8 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -32,19 +31,34 @@ #ifndef _TTM_BO_API_H_ #define _TTM_BO_API_H_ -#include +#include +#include +#include +#include #include struct ttm_bo_device; struct drm_mm_node; +/** + * struct ttm_place + * + * @fpfn: first valid page frame number to put the object + * @lpfn: last valid page frame number to put the object + * @flags: memory domain and caching flags for the object + * + * Structure indicating a possible place to put an object. + */ +struct ttm_place { + unsigned fpfn; + unsigned lpfn; + uint32_t flags; +}; /** * struct ttm_placement * - * @fpfn: first valid page frame number to put the object - * @lpfn: last valid page frame number to put the object * @num_placement: number of preferred placements * @placement: preferred placements * @num_busy_placement: number of preferred placements when need to evict buffer @@ -53,12 +67,10 @@ struct drm_mm_node; * Structure indicating the placement you request for an object. */ struct ttm_placement { - unsigned fpfn; - unsigned lpfn; - unsigned num_placement; - const uint32_t *placement; - unsigned num_busy_placement; - const uint32_t *busy_placement; + unsigned num_placement; + const struct ttm_place *placement; + unsigned num_busy_placement; + const struct ttm_place *busy_placement; }; /** @@ -148,7 +160,6 @@ struct ttm_tt; * Lru lists may keep one refcount, the delayed delete list, and kref != 0 * keeps one refcount. When this refcount reaches zero, * the object is destroyed. - * @event_queue: Queue for processes waiting on buffer object status change. * @mem: structure describing current placement. * @persistent_swap_storage: Usually the swap storage is deleted for buffers * pinned in physical memory. If this behaviour is not desired, this member @@ -159,18 +170,12 @@ struct ttm_tt; * @lru: List head for the lru list. * @ddestroy: List head for the delayed destroy list. * @swap: List head for swap LRU list. - * @val_seq: Sequence of the validation holding the @reserved lock. - * Used to avoid starvation when many processes compete to validate the - * buffer. This member is protected by the bo_device::lru_lock. - * @seq_valid: The value of @val_seq is valid. This value is protected by - * the bo_device::lru_lock. - * @reserved: Deadlock-free lock used for synchronization state transitions. - * @sync_obj: Pointer to a synchronization object. * @priv_flags: Flags describing buffer object internal state. * @vma_node: Address space manager node. * @offset: The current GPU offset, which can have different meanings * depending on the memory type. For SYSTEM type memory, it should be 0. * @cur_placement: Hint of current placement. + * @wu_mutex: Wait unreserved mutex. * * Base class for TTM buffer object, that deals with data placement and CPU * mappings. GPU mappings are really up to the driver, but for simpler GPUs @@ -204,10 +209,9 @@ struct ttm_buffer_object { struct kref kref; struct kref list_kref; - wait_queue_head_t event_queue; /** - * Members protected by the bo::reserved lock. + * Members protected by the bo::resv::reserved lock. */ struct ttm_mem_reg mem; @@ -229,24 +233,11 @@ struct ttm_buffer_object { struct list_head ddestroy; struct list_head swap; struct list_head io_reserve_lru; - uint32_t val_seq; - bool seq_valid; /** - * Members protected by the bdev::lru_lock - * only when written to. + * Members protected by a bo reservation. */ - atomic_t reserved; - - /** - * Members protected by struct buffer_object_device::fence_lock - * In addition, setting sync_obj to anything else - * than NULL requires bo::reserved to be held. This allows for - * checking NULL while reserved but not holding the mentioned lock. - */ - - void *sync_obj; unsigned long priv_flags; struct drm_vma_offset_node vma_node; @@ -257,10 +248,14 @@ struct ttm_buffer_object { * either of these locks held. */ - unsigned long offset; + uint64_t offset; /* GPU address space is independent of CPU word size */ uint32_t cur_placement; struct sg_table *sg; + + struct reservation_object *resv; + struct reservation_object ttm_resv; + struct rwlock wu_mutex; }; /** @@ -320,6 +315,20 @@ ttm_bo_reference(struct ttm_buffer_object *bo) */ extern int ttm_bo_wait(struct ttm_buffer_object *bo, bool lazy, bool interruptible, bool no_wait); + +/** + * ttm_bo_mem_compat - Check if proposed placement is compatible with a bo + * + * @placement: Return immediately if buffer is busy. + * @mem: The struct ttm_mem_reg indicating the region where the bo resides + * @new_flags: Describes compatible placement found + * + * Returns true if the placement is compatible + */ +extern bool ttm_bo_mem_compat(struct ttm_placement *placement, + struct ttm_mem_reg *mem, + uint32_t *new_flags); + /** * ttm_bo_validate * @@ -464,6 +473,7 @@ size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, * point to the shmem object backing a GEM object if TTM is used to back a * GEM user interface. * @acc_size: Accounted size for this object. + * @resv: Pointer to a reservation_object, or NULL to let ttm allocate one. * @destroy: Destroy function. Use NULL for kfree(). * * This function initializes a pre-allocated struct ttm_buffer_object. @@ -491,16 +501,16 @@ extern int ttm_bo_init(struct ttm_bo_device *bdev, struct uvm_object *persistent_swap_storage, size_t acc_size, struct sg_table *sg, + struct reservation_object *resv, void (*destroy) (struct ttm_buffer_object *)); /** - * ttm_bo_synccpu_object_init + * ttm_bo_create * * @bdev: Pointer to a ttm_bo_device struct. - * @bo: Pointer to a ttm_buffer_object to be initialized. * @size: Requested size of buffer object. * @type: Requested type of buffer object. - * @flags: Initial placement flags. + * @placement: Initial placement. * @page_alignment: Data alignment in pages. * @interruptible: If needing to sleep while waiting for GPU resources, * sleep interruptible. @@ -528,20 +538,6 @@ extern int ttm_bo_create(struct ttm_bo_device *bdev, struct uvm_object *persistent_swap_storage, struct ttm_buffer_object **p_bo); -/** - * ttm_bo_check_placement - * - * @bo: the buffer object. - * @placement: placements - * - * Performs minimal validity checking on an intended change of - * placement flags. - * Returns - * -EINVAL: Intended change is invalid or not allowed. - */ -extern int ttm_bo_check_placement(struct ttm_buffer_object *bo, - struct ttm_placement *placement); - /** * ttm_bo_init_mm * @@ -713,23 +709,9 @@ extern struct uvm_object *ttm_bo_mmap(voff_t, vsize_t, */ extern ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, - const char *wbuf, char *rbuf, - size_t count, off_t *f_pos, bool write); + const char __user *wbuf, char __user *rbuf, + size_t count, loff_t *f_pos, bool write); extern void ttm_bo_swapout_all(struct ttm_bo_device *bdev); - -/** - * ttm_bo_is_reserved - return an indication if a ttm buffer object is reserved - * - * @bo: The buffer object to check. - * - * This function returns an indication if a bo is reserved or not, and should - * only be used to print an error when it is not from incorrect api usage, since - * there's no guarantee that it is the caller that is holding the reservation. - */ -static inline bool ttm_bo_is_reserved(struct ttm_buffer_object *bo) -{ - return atomic_read(&bo->reserved); -} - +extern int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo); #endif diff --git a/sys/dev/pci/drm/ttm/ttm_bo_driver.h b/sys/dev/pci/drm/ttm/ttm_bo_driver.h index 4e6ff00f784..94d080dd65f 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_driver.h +++ b/sys/dev/pci/drm/ttm/ttm_bo_driver.h @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_bo_driver.h,v 1.10 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 Vmware, Inc., Palo Alto, CA., USA @@ -35,8 +34,10 @@ #include #include #include +#include #include #include +#include struct ttm_backend_func { /** @@ -129,6 +130,7 @@ struct ttm_tt { * struct ttm_dma_tt * * @ttm: Base ttm_tt struct. + * @cpu_address: The CPU address of the pages * @dma_address: The DMA (bus) addresses of the pages * @pages_list: used by some page allocation backend * @@ -138,7 +140,8 @@ struct ttm_tt { */ struct ttm_dma_tt { struct ttm_tt ttm; - bus_addr_t *dma_address; + void **cpu_address; + dma_addr_t *dma_address; struct list_head pages_list; }; @@ -178,6 +181,7 @@ struct ttm_mem_type_manager_func { * @man: Pointer to a memory type manager. * @bo: Pointer to the buffer object we're allocating space for. * @placement: Placement details. + * @flags: Additional placement flags. * @mem: Pointer to a struct ttm_mem_reg to be filled in. * * This function should allocate space in the memory type managed @@ -201,7 +205,7 @@ struct ttm_mem_type_manager_func { */ int (*get_node)(struct ttm_mem_type_manager *man, struct ttm_buffer_object *bo, - struct ttm_placement *placement, + const struct ttm_place *place, struct ttm_mem_reg *mem); /** @@ -270,7 +274,7 @@ struct ttm_mem_type_manager { bool has_type; bool use_type; uint32_t flags; - unsigned long gpu_offset; + uint64_t gpu_offset; /* GPU address space is independent of CPU word size */ uint64_t size; uint32_t available_caching; uint32_t default_caching; @@ -305,11 +309,6 @@ struct ttm_mem_type_manager { * @move: Callback for a driver to hook in accelerated functions to * move a buffer. * If set to NULL, a potentially slow memcpy() move is used. - * @sync_obj_signaled: See ttm_fence_api.h - * @sync_obj_wait: See ttm_fence_api.h - * @sync_obj_flush: See ttm_fence_api.h - * @sync_obj_unref: See ttm_fence_api.h - * @sync_obj_ref: See ttm_fence_api.h */ struct ttm_bo_driver { @@ -411,23 +410,6 @@ struct ttm_bo_driver { int (*verify_access) (struct ttm_buffer_object *bo, struct file *filp); - /** - * In case a driver writer dislikes the TTM fence objects, - * the driver writer can replace those with sync objects of - * his / her own. If it turns out that no driver writer is - * using these. I suggest we remove these hooks and plug in - * fences directly. The bo driver needs the following functionality: - * See the corresponding functions in the fence object API - * documentation. - */ - - bool (*sync_obj_signaled) (void *sync_obj); - int (*sync_obj_wait) (void *sync_obj, - bool lazy, bool interruptible); - int (*sync_obj_flush) (void *sync_obj); - void (*sync_obj_unref) (void **sync_obj); - void *(*sync_obj_ref) (void *sync_obj); - /* hook to notify driver about a driver move so it * can do tiling things */ void (*move_notify)(struct ttm_buffer_object *bo, @@ -514,8 +496,6 @@ struct ttm_bo_global { * * @driver: Pointer to a struct ttm_bo_driver struct setup by the driver. * @man: An array of mem_type_managers. - * @fence_lock: Protects the synchronizing members on *all* bos belonging - * to this device. * @vma_manager: Address space manager * lru_lock: Spinlock that protects the buffer+device lru lists and * ddestroy lists. @@ -535,7 +515,6 @@ struct ttm_bo_device { struct ttm_bo_global *glob; struct ttm_bo_driver *driver; struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES]; - spinlock_t fence_lock; bus_space_tag_t iot; bus_space_tag_t memt; @@ -652,18 +631,6 @@ extern void ttm_tt_unbind(struct ttm_tt *ttm); */ extern int ttm_tt_swapin(struct ttm_tt *ttm); -/** - * ttm_tt_cache_flush: - * - * @pages: An array of pointers to struct page:s to flush. - * @num_pages: Number of pages to flush. - * - * Flush the data of the indicated pages from the cpu caches. - * This is used when changing caching attributes of the pages from - * cache-coherent. - */ -extern void ttm_tt_cache_flush(struct vm_page *pages[], unsigned long num_pages); - /** * ttm_tt_set_placement_caching: * @@ -681,6 +648,15 @@ extern int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement); extern int ttm_tt_swapout(struct ttm_tt *ttm, struct uvm_object *persistent_swap_storage); +/** + * ttm_tt_unpopulate - free pages from a ttm + * + * @ttm: Pointer to the ttm_tt structure + * + * Calls the driver method to free all pages from a ttm + */ +extern void ttm_tt_unpopulate(struct ttm_tt *ttm); + /* * ttm_bo.c */ @@ -738,6 +714,7 @@ extern int ttm_bo_device_release(struct ttm_bo_device *bdev); * @bdev: A pointer to a struct ttm_bo_device to initialize. * @glob: A pointer to an initialized struct ttm_bo_global. * @driver: A pointer to a struct ttm_bo_driver set up by the caller. + * @mapping: The address space to use for this bo. * @file_page_offset: Offset into the device address space that is available * for buffer data. This ensures compatibility with other users of the * address space. @@ -749,6 +726,7 @@ extern int ttm_bo_device_release(struct ttm_bo_device *bdev); extern int ttm_bo_device_init(struct ttm_bo_device *bdev, struct ttm_bo_global *glob, struct ttm_bo_driver *driver, + struct address_space *mapping, uint64_t file_page_offset, bool need_dma32); /** @@ -773,6 +751,55 @@ extern int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible); extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); +extern void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo); +extern void ttm_bo_add_to_lru(struct ttm_buffer_object *bo); + +/** + * __ttm_bo_reserve: + * + * @bo: A pointer to a struct ttm_buffer_object. + * @interruptible: Sleep interruptible if waiting. + * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY. + * @use_ticket: If @bo is already reserved, Only sleep waiting for + * it to become unreserved if @ticket->stamp is older. + * + * Will not remove reserved buffers from the lru lists. + * Otherwise identical to ttm_bo_reserve. + * + * Returns: + * -EDEADLK: The reservation may cause a deadlock. + * Release all buffer reservations, wait for @bo to become unreserved and + * try again. (only if use_sequence == 1). + * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by + * a signal. Release all buffer reservations and return to user-space. + * -EBUSY: The function needed to sleep, but @no_wait was true + * -EALREADY: Bo already reserved using @ticket. This error code will only + * be returned if @use_ticket is set to true. + */ +static inline int __ttm_bo_reserve(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_ticket, + struct ww_acquire_ctx *ticket) +{ + int ret = 0; + + if (no_wait) { + bool success; + if (WARN_ON(ticket)) + return -EBUSY; + + success = ww_mutex_trylock(&bo->resv->lock); + return success ? 0 : -EBUSY; + } + + if (interruptible) + ret = ww_mutex_lock_interruptible(&bo->resv->lock, ticket); + else + ret = ww_mutex_lock(&bo->resv->lock, ticket); + if (ret == -EINTR) + return -ERESTARTSYS; + return ret; +} /** * ttm_bo_reserve: @@ -780,8 +807,8 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); * @bo: A pointer to a struct ttm_buffer_object. * @interruptible: Sleep interruptible if waiting. * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY. - * @use_sequence: If @bo is already reserved, Only sleep waiting for - * it to become unreserved if @sequence < (@bo)->sequence. + * @use_ticket: If @bo is already reserved, Only sleep waiting for + * it to become unreserved if @ticket->stamp is older. * * Locks a buffer object for validation. (Or prevents other processes from * locking it for validation) and removes it from lru lists, while taking @@ -792,19 +819,10 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); * to make room for a buffer already reserved. (Buffers are reserved before * they are evicted). The following algorithm prevents such deadlocks from * occurring: - * 1) Buffers are reserved with the lru spinlock held. Upon successful - * reservation they are removed from the lru list. This stops a reserved buffer - * from being evicted. However the lru spinlock is released between the time - * a buffer is selected for eviction and the time it is reserved. - * Therefore a check is made when a buffer is reserved for eviction, that it - * is still the first buffer in the lru list, before it is removed from the - * list. @check_lru == 1 forces this check. If it fails, the function returns - * -EINVAL, and the caller should then choose a new buffer to evict and repeat - * the procedure. - * 2) Processes attempting to reserve multiple buffers other than for eviction, + * Processes attempting to reserve multiple buffers other than for eviction, * (typically execbuf), should first obtain a unique 32-bit * validation sequence number, - * and call this function with @use_sequence == 1 and @sequence == the unique + * and call this function with @use_ticket == 1 and @ticket->stamp == the unique * sequence number. If upon call of this function, the buffer object is already * reserved, the validation sequence is checked against the validation * sequence of the process currently reserving the buffer, @@ -819,84 +837,113 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); * will eventually succeed, preventing both deadlocks and starvation. * * Returns: - * -EAGAIN: The reservation may cause a deadlock. + * -EDEADLK: The reservation may cause a deadlock. * Release all buffer reservations, wait for @bo to become unreserved and * try again. (only if use_sequence == 1). * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by * a signal. Release all buffer reservations and return to user-space. * -EBUSY: The function needed to sleep, but @no_wait was true - * -EDEADLK: Bo already reserved using @sequence. This error code will only - * be returned if @use_sequence is set to true. + * -EALREADY: Bo already reserved using @ticket. This error code will only + * be returned if @use_ticket is set to true. */ -extern int ttm_bo_reserve(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, uint32_t sequence); +static inline int ttm_bo_reserve(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_ticket, + struct ww_acquire_ctx *ticket) +{ + int ret; + + WARN_ON(!atomic_read(&bo->kref.refcount)); + ret = __ttm_bo_reserve(bo, interruptible, no_wait, use_ticket, ticket); + if (likely(ret == 0)) + ttm_bo_del_sub_from_lru(bo); + + return ret; +} /** - * ttm_bo_reserve_locked: - * + * ttm_bo_reserve_slowpath: * @bo: A pointer to a struct ttm_buffer_object. * @interruptible: Sleep interruptible if waiting. - * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY. - * @use_sequence: If @bo is already reserved, Only sleep waiting for - * it to become unreserved if @sequence < (@bo)->sequence. - * - * Must be called with struct ttm_bo_global::lru_lock held, - * and will not remove reserved buffers from the lru lists. - * The function may release the LRU spinlock if it needs to sleep. - * Otherwise identical to ttm_bo_reserve. + * @sequence: Set (@bo)->sequence to this value after lock * - * Returns: - * -EAGAIN: The reservation may cause a deadlock. - * Release all buffer reservations, wait for @bo to become unreserved and - * try again. (only if use_sequence == 1). - * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by - * a signal. Release all buffer reservations and return to user-space. - * -EBUSY: The function needed to sleep, but @no_wait was true - * -EDEADLK: Bo already reserved using @sequence. This error code will only - * be returned if @use_sequence is set to true. + * This is called after ttm_bo_reserve returns -EAGAIN and we backed off + * from all our other reservations. Because there are no other reservations + * held by us, this function cannot deadlock any more. */ -extern int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, - uint32_t sequence); +static inline int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo, + bool interruptible, + struct ww_acquire_ctx *ticket) +{ + int ret = 0; + + WARN_ON(!atomic_read(&bo->kref.refcount)); + + if (interruptible) + ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, + ticket); + else + ww_mutex_lock_slow(&bo->resv->lock, ticket); + + if (likely(ret == 0)) + ttm_bo_del_sub_from_lru(bo); + else if (ret == -EINTR) + ret = -ERESTARTSYS; + + return ret; +} /** - * ttm_bo_unreserve - * + * __ttm_bo_unreserve * @bo: A pointer to a struct ttm_buffer_object. * - * Unreserve a previous reservation of @bo. + * Unreserve a previous reservation of @bo where the buffer object is + * already on lru lists. */ -extern void ttm_bo_unreserve(struct ttm_buffer_object *bo); +static inline void __ttm_bo_unreserve(struct ttm_buffer_object *bo) +{ + ww_mutex_unlock(&bo->resv->lock); +} /** - * ttm_bo_unreserve_locked + * ttm_bo_unreserve * * @bo: A pointer to a struct ttm_buffer_object. * * Unreserve a previous reservation of @bo. - * Needs to be called with struct ttm_bo_global::lru_lock held. */ -extern void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo); +static inline void ttm_bo_unreserve(struct ttm_buffer_object *bo) +{ + if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { + spin_lock(&bo->glob->lru_lock); + ttm_bo_add_to_lru(bo); + spin_unlock(&bo->glob->lru_lock); + } + __ttm_bo_unreserve(bo); +} /** - * ttm_bo_wait_unreserved - * + * ttm_bo_unreserve_ticket * @bo: A pointer to a struct ttm_buffer_object. + * @ticket: ww_acquire_ctx used for reserving * - * Wait for a struct ttm_buffer_object to become unreserved. - * This is typically used in the execbuf code to relax cpu-usage when - * a potential deadlock condition backoff. + * Unreserve a previous reservation of @bo made with @ticket. */ -extern int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, - bool interruptible); +static inline void ttm_bo_unreserve_ticket(struct ttm_buffer_object *bo, + struct ww_acquire_ctx *t) +{ + ttm_bo_unreserve(bo); +} /* * ttm_bo_util.c */ +int ttm_mem_io_reserve(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem); +void ttm_mem_io_free(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem); /** * ttm_bo_move_ttm * @@ -954,7 +1001,7 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); * ttm_bo_move_accel_cleanup. * * @bo: A pointer to a struct ttm_buffer_object. - * @sync_obj: A sync object that signals when moving is complete. + * @fence: A fence object that signals when moving is complete. * @evict: This is an evict move. Don't return until the buffer is idle. * @no_wait_gpu: Return immediately if the GPU is busy. * @new_mem: struct ttm_mem_reg indicating where to move. @@ -968,7 +1015,7 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); */ extern int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, - void *sync_obj, + struct fence *fence, bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem); /** diff --git a/sys/dev/pci/drm/ttm/ttm_bo_manager.c b/sys/dev/pci/drm/ttm/ttm_bo_manager.c index d229d130373..7485030e7c9 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_manager.c +++ b/sys/dev/pci/drm/ttm/ttm_bo_manager.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_bo_manager.c,v 1.9 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA @@ -35,7 +34,7 @@ #include /** - * Currently we use a mutex for the lock, but a rwlock *may* be + * Currently we use a spinlock for the lock, but a mutex *may* be * more appropriate to reduce scheduling latency if the range manager * ends up with very fragmented allocation patterns. */ @@ -47,7 +46,7 @@ struct ttm_range_manager { static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, struct ttm_buffer_object *bo, - struct ttm_placement *placement, + const struct ttm_place *place, struct ttm_mem_reg *mem) { struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; @@ -58,7 +57,7 @@ static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, unsigned long lpfn; int ret; - lpfn = placement->lpfn; + lpfn = place->lpfn; if (!lpfn) lpfn = man->size; @@ -66,10 +65,15 @@ static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, if (!node) return -ENOMEM; + if (place->flags & TTM_PL_FLAG_TOPDOWN) { + sflags = DRM_MM_SEARCH_BELOW; + aflags = DRM_MM_CREATE_TOP; + } + spin_lock(&rman->lock); ret = drm_mm_insert_node_in_range_generic(mm, node, mem->num_pages, mem->page_alignment, 0, - placement->fpfn, lpfn, + place->fpfn, lpfn, sflags, aflags); spin_unlock(&rman->lock); @@ -133,14 +137,11 @@ static int ttm_bo_man_takedown(struct ttm_mem_type_manager *man) static void ttm_bo_man_debug(struct ttm_mem_type_manager *man, const char *prefix) { - printf("%s stub\n", __func__); -#ifdef notyet struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; spin_lock(&rman->lock); drm_mm_debug_table(&rman->mm, prefix); spin_unlock(&rman->lock); -#endif } const struct ttm_mem_type_manager_func ttm_bo_manager_func = { diff --git a/sys/dev/pci/drm/ttm/ttm_bo_util.c b/sys/dev/pci/drm/ttm/ttm_bo_util.c index 4f268efc7b1..da6c459bd39 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_util.c +++ b/sys/dev/pci/drm/ttm/ttm_bo_util.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_bo_util.c,v 1.18 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA @@ -32,11 +31,8 @@ #include #include #include - -int ttm_mem_reg_ioremap(struct ttm_bo_device *, struct ttm_mem_reg *, - void **); -void ttm_mem_reg_iounmap(struct ttm_bo_device *, struct ttm_mem_reg *, - void *); +#include +#include void ttm_bo_free_old_node(struct ttm_buffer_object *bo) { @@ -87,6 +83,7 @@ int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) mutex_lock(&man->io_reserve_mutex); return 0; } +EXPORT_SYMBOL(ttm_mem_io_lock); void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) { @@ -95,6 +92,7 @@ void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) mutex_unlock(&man->io_reserve_mutex); } +EXPORT_SYMBOL(ttm_mem_io_unlock); static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) { @@ -112,8 +110,9 @@ static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) return 0; } -static int ttm_mem_io_reserve(struct ttm_bo_device *bdev, - struct ttm_mem_reg *mem) + +int ttm_mem_io_reserve(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; int ret = 0; @@ -135,9 +134,10 @@ retry: } return ret; } +EXPORT_SYMBOL(ttm_mem_io_reserve); -static void ttm_mem_io_free(struct ttm_bo_device *bdev, - struct ttm_mem_reg *mem) +void ttm_mem_io_free(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; @@ -150,6 +150,7 @@ static void ttm_mem_io_free(struct ttm_bo_device *bdev, bdev->driver->io_mem_free(bdev, mem); } +EXPORT_SYMBOL(ttm_mem_io_free); int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) { @@ -182,7 +183,7 @@ void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) } } -int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, +static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, void **virtual) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; @@ -206,7 +207,8 @@ int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, flags = 0; if (bus_space_map(bdev->memt, mem->bus.base + mem->bus.offset, - mem->bus.size, BUS_SPACE_MAP_LINEAR | flags, &mem->bus.bsh)) { + mem->bus.size, BUS_SPACE_MAP_LINEAR | flags, + &mem->bus.bsh)) { printf("%s bus_space_map failed\n", __func__); return -ENOMEM; } @@ -224,7 +226,7 @@ int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, return 0; } -void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, +static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, void *virtual) { struct ttm_mem_type_manager *man; @@ -263,26 +265,34 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); +#ifdef CONFIG_X86 + dst = kmap_atomic_prot(d, prot); +#else if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) dst = vmap(&d, 1, 0, prot); else dst = kmap(d); +#endif if (!dst) return -ENOMEM; memcpy_fromio(dst, src, PAGE_SIZE); +#ifdef CONFIG_X86 + kunmap_atomic(dst); +#else if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) vunmap(dst, PAGE_SIZE); else kunmap(d); +#endif return 0; } static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, unsigned long page, - vm_prot_t prot) + pgprot_t prot) { struct vm_page *s = ttm->pages[page]; void *src; @@ -291,19 +301,27 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, return -ENOMEM; dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); +#ifdef CONFIG_X86 + src = kmap_atomic_prot(s, prot); +#else if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) src = vmap(&s, 1, 0, prot); else src = kmap(s); +#endif if (!src) return -ENOMEM; memcpy_toio(dst, src, PAGE_SIZE); +#ifdef CONFIG_X86 + kunmap_atomic(src); +#else if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) vunmap(src, PAGE_SIZE); else kunmap(s); +#endif return 0; } @@ -339,10 +357,14 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, goto out2; /* - * Move nonexistent data. NOP. + * Don't move nonexistent data. Clear destination instead. */ - if (old_iomap == NULL && ttm == NULL) + if (old_iomap == NULL && + (ttm == NULL || (ttm->state == tt_unpopulated && + !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { + memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); goto out2; + } /* * TTM might be null for moves within the same region. @@ -429,8 +451,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, struct ttm_buffer_object **new_obj) { struct ttm_buffer_object *fbo; - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_driver *driver = bdev->driver; + int ret; fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); if (!fbo) @@ -443,7 +464,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, * TODO: Explicit member copy would probably be better here. */ - init_waitqueue_head(&fbo->event_queue); INIT_LIST_HEAD(&fbo->ddestroy); INIT_LIST_HEAD(&fbo->lru); INIT_LIST_HEAD(&fbo->swap); @@ -451,21 +471,47 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, drm_vma_node_reset(&fbo->vma_node); atomic_set(&fbo->cpu_writers, 0); - spin_lock(&bdev->fence_lock); - if (bo->sync_obj) - fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); - else - fbo->sync_obj = NULL; - spin_unlock(&bdev->fence_lock); kref_init(&fbo->list_kref); kref_init(&fbo->kref); fbo->destroy = &ttm_transfered_destroy; fbo->acc_size = 0; + fbo->resv = &fbo->ttm_resv; + reservation_object_init(fbo->resv); + ret = ww_mutex_trylock(&fbo->resv->lock); + WARN_ON(!ret); *new_obj = fbo; return 0; } +#ifdef __linux__ +pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) +{ + /* Cached mappings need no adjustment */ + if (caching_flags & TTM_PL_FLAG_CACHED) + return tmp; + +#if defined(__i386__) || defined(__x86_64__) + if (caching_flags & TTM_PL_FLAG_WC) + tmp = pgprot_writecombine(tmp); + else if (boot_cpu_data.x86 > 3) + tmp = pgprot_noncached(tmp); +#endif +#if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ + defined(__powerpc__) + if (caching_flags & TTM_PL_FLAG_WC) + tmp = pgprot_writecombine(tmp); + else + tmp = pgprot_noncached(tmp); +#endif +#if defined(__sparc__) || defined(__mips__) + tmp = pgprot_noncached(tmp); +#endif + return tmp; +} +EXPORT_SYMBOL(ttm_io_prot); +#endif + pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) { #ifdef PMAP_WC @@ -475,15 +521,14 @@ pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) #endif return PMAP_NOCACHE; } -EXPORT_SYMBOL(ttm_io_prot); static int ttm_bo_ioremap(struct ttm_buffer_object *bo, unsigned long offset, unsigned long size, struct ttm_bo_kmap_obj *map) { - struct ttm_mem_reg *mem = &bo->mem; int flags; + struct ttm_mem_reg *mem = &bo->mem; if (bo->mem.bus.addr) { map->bo_kmap_type = ttm_bo_map_premapped; @@ -539,9 +584,7 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, * We need to use vmap to get the desired page protection * or to make the buffer object look contiguous. */ - prot = (mem->placement & TTM_PL_FLAG_CACHED) ? - PAGE_KERNEL : - ttm_io_prot(mem->placement, PAGE_KERNEL); + prot = ttm_io_prot(mem->placement, PAGE_KERNEL); map->bo_kmap_type = ttm_bo_map_vmap; map->virtual = vmap(ttm->pages + start_page, num_pages, 0, prot); @@ -566,7 +609,7 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, if (start_page > bo->num_pages) return -EINVAL; #if 0 - if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) + if (num_pages > 1 && !capable(CAP_SYS_ADMIN)) return -EPERM; #endif (void) ttm_mem_io_lock(man, false); @@ -617,30 +660,20 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) EXPORT_SYMBOL(ttm_bo_kunmap); int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, - void *sync_obj, + struct fence *fence, bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_driver *driver = bdev->driver; struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; struct ttm_mem_reg *old_mem = &bo->mem; int ret; struct ttm_buffer_object *ghost_obj; - void *tmp_obj = NULL; - spin_lock(&bdev->fence_lock); - if (bo->sync_obj) { - tmp_obj = bo->sync_obj; - bo->sync_obj = NULL; - } - bo->sync_obj = driver->sync_obj_ref(sync_obj); + reservation_object_add_excl_fence(bo->resv, fence); if (evict) { ret = ttm_bo_wait(bo, false, false, false); - spin_unlock(&bdev->fence_lock); - if (tmp_obj) - driver->sync_obj_unref(&tmp_obj); if (ret) return ret; @@ -661,14 +694,13 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, */ set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); - spin_unlock(&bdev->fence_lock); - if (tmp_obj) - driver->sync_obj_unref(&tmp_obj); ret = ttm_buffer_object_transfer(bo, &ghost_obj); if (ret) return ret; + reservation_object_add_excl_fence(ghost_obj->resv, fence); + /** * If we're not moving to fixed memory, the TTM object * needs to stay alive. Otherwhise hang it on the ghost diff --git a/sys/dev/pci/drm/ttm/ttm_bo_vm.c b/sys/dev/pci/drm/ttm/ttm_bo_vm.c index c987aa03b22..de7cee96a22 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_vm.c +++ b/sys/dev/pci/drm/ttm/ttm_bo_vm.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_bo_vm.c,v 1.13 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -38,14 +37,235 @@ #define TTM_BO_VM_NUM_PREFAULT 16 -ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *, const char __user *, - char __user *, size_t, off_t *, bool); +static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo) +{ + int ret = 0; + + if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags))) + goto out_unlock; + + /* + * Quick non-stalling check for idle. + */ + ret = ttm_bo_wait(bo, false, false, true); + if (likely(ret == 0)) + goto out_unlock; + + /* + * If possible, avoid waiting for GPU with mmap_sem + * held. + */ +#ifdef notyet + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + ret = VM_FAULT_RETRY; + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + goto out_unlock; + + ttm_bo_reference(bo); + up_read(&vma->vm_mm->mmap_sem); + (void) ttm_bo_wait(bo, false, true, false); + ttm_bo_unreserve(bo); + ttm_bo_unref(&bo); + goto out_unlock; + } +#endif + + /* + * Ordinary wait. + */ + ret = ttm_bo_wait(bo, false, true, false); + if (unlikely(ret != 0)) + ret = (ret != -ERESTARTSYS) ? VM_PAGER_ERROR : + VM_PAGER_REFAULT; + +out_unlock: + return ret; +} + +#ifdef __linux__ +static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct ttm_buffer_object *bo = (struct ttm_buffer_object *) + vma->vm_private_data; + struct ttm_bo_device *bdev = bo->bdev; + unsigned long page_offset; + unsigned long page_last; + unsigned long pfn; + struct ttm_tt *ttm = NULL; + struct vm_page *page; + int ret; + int i; + unsigned long address = (unsigned long)vmf->virtual_address; + int retval = VM_FAULT_NOPAGE; + struct ttm_mem_type_manager *man = + &bdev->man[bo->mem.mem_type]; + struct vm_area_struct cvma; + + /* + * Work around locking order reversal in fault / nopfn + * between mmap_sem and bo_reserve: Perform a trylock operation + * for reserve, and if it fails, retry the fault after waiting + * for the buffer to become unreserved. + */ + ret = ttm_bo_reserve(bo, true, true, false, NULL); + if (unlikely(ret != 0)) { + if (ret != -EBUSY) + return VM_FAULT_NOPAGE; + + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + ttm_bo_reference(bo); + up_read(&vma->vm_mm->mmap_sem); + (void) ttm_bo_wait_unreserved(bo); + ttm_bo_unref(&bo); + } + + return VM_FAULT_RETRY; + } + + /* + * If we'd want to change locking order to + * mmap_sem -> bo::reserve, we'd use a blocking reserve here + * instead of retrying the fault... + */ + return VM_FAULT_NOPAGE; + } + + /* + * Refuse to fault imported pages. This should be handled + * (if at all) by redirecting mmap to the exporter. + */ + if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { + retval = VM_FAULT_SIGBUS; + goto out_unlock; + } -int ttm_bo_vm_fault(struct uvm_faultinfo *, vaddr_t, vm_page_t *, - int, int, vm_fault_t, vm_prot_t, int); -void ttm_bo_vm_reference(struct uvm_object *); -void ttm_bo_vm_detach(struct uvm_object *); + if (bdev->driver->fault_reserve_notify) { + ret = bdev->driver->fault_reserve_notify(bo); + switch (ret) { + case 0: + break; + case -EBUSY: + case -ERESTARTSYS: + retval = VM_FAULT_NOPAGE; + goto out_unlock; + default: + retval = VM_FAULT_SIGBUS; + goto out_unlock; + } + } + /* + * Wait for buffer data in transit, due to a pipelined + * move. + */ + ret = ttm_bo_vm_fault_idle(bo); + if (unlikely(ret != 0)) { + retval = ret; + + if (retval == VM_FAULT_RETRY && + !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + /* The BO has already been unreserved. */ + return retval; + } + + goto out_unlock; + } + + ret = ttm_mem_io_lock(man, true); + if (unlikely(ret != 0)) { + retval = VM_FAULT_NOPAGE; + goto out_unlock; + } + ret = ttm_mem_io_reserve_vm(bo); + if (unlikely(ret != 0)) { + retval = VM_FAULT_SIGBUS; + goto out_io_unlock; + } + + page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + + vma->vm_pgoff - drm_vma_node_start(&bo->vma_node); + page_last = vma_pages(vma) + vma->vm_pgoff - + drm_vma_node_start(&bo->vma_node); + + if (unlikely(page_offset >= bo->num_pages)) { + retval = VM_FAULT_SIGBUS; + goto out_io_unlock; + } + + /* + * Make a local vma copy to modify the page_prot member + * and vm_flags if necessary. The vma parameter is protected + * by mmap_sem in write mode. + */ + cvma = *vma; + cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags); + + if (bo->mem.bus.is_iomem) { + cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, + cvma.vm_page_prot); + } else { + ttm = bo->ttm; + cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, + cvma.vm_page_prot); + + /* Allocate all page at once, most common usage */ + if (ttm->bdev->driver->ttm_tt_populate(ttm)) { + retval = VM_FAULT_OOM; + goto out_io_unlock; + } + } + + /* + * Speculatively prefault a number of pages. Only error on + * first page. + */ + for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { + if (bo->mem.bus.is_iomem) + pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset; + else { + page = ttm->pages[page_offset]; + if (unlikely(!page && i == 0)) { + retval = VM_FAULT_OOM; + goto out_io_unlock; + } else if (unlikely(!page)) { + break; + } + page->mapping = vma->vm_file->f_mapping; + page->index = drm_vma_node_start(&bo->vma_node) + + page_offset; + pfn = page_to_pfn(page); + } + + if (vma->vm_flags & VM_MIXEDMAP) + ret = vm_insert_mixed(&cvma, address, pfn); + else + ret = vm_insert_pfn(&cvma, address, pfn); + + /* + * Somebody beat us to this PTE or prefaulting to + * an already populated PTE, or prefaulting error. + */ + + if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0))) + break; + else if (unlikely(ret != 0)) { + retval = + (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto out_io_unlock; + } + + address += PAGE_SIZE; + if (unlikely(++page_offset >= page_last)) + break; + } +out_io_unlock: + ttm_mem_io_unlock(man); +out_unlock: + ttm_bo_unreserve(bo); + return retval; +} +#else int ttm_bo_vm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps, int npages, int centeridx, vm_fault_t fault_type, @@ -73,10 +293,10 @@ ttm_bo_vm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps, /* * Work around locking order reversal in fault / nopfn * between mmap_sem and bo_reserve: Perform a trylock operation - * for reserve, and if it fails, retry the fault after scheduling. + * for reserve, and if it fails, retry the fault after waiting + * for the buffer to become unreserved. */ - - ret = ttm_bo_reserve(bo, true, true, false, 0); + ret = ttm_bo_reserve(bo, true, true, false, NULL); if (unlikely(ret != 0)) { uvmfault_unlockall(ufi, NULL, uobj, NULL); ret = ttm_bo_reserve(bo, true, false, false, 0); @@ -85,17 +305,21 @@ ttm_bo_vm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps, return VM_PAGER_REFAULT; } + /* + * Refuse to fault imported pages. This should be handled + * (if at all) by redirecting mmap to the exporter. + */ + if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { + retval = VM_PAGER_ERROR; + goto out_unlock; + } + if (bdev->driver->fault_reserve_notify) { ret = bdev->driver->fault_reserve_notify(bo); switch (ret) { case 0: break; case -EBUSY: -#if 0 - set_need_resched(); -#else - printf("resched?\n"); -#endif case -ERESTARTSYS: retval = VM_PAGER_REFAULT; goto out_unlock; @@ -109,18 +333,13 @@ ttm_bo_vm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps, * Wait for buffer data in transit, due to a pipelined * move. */ - - spin_lock(&bdev->fence_lock); - if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { - ret = ttm_bo_wait(bo, false, true, false); - spin_unlock(&bdev->fence_lock); - if (unlikely(ret != 0)) { - retval = (ret != -ERESTARTSYS) ? - VM_PAGER_ERROR : VM_PAGER_REFAULT; - goto out_unlock; - } - } else - spin_unlock(&bdev->fence_lock); + ret = ttm_bo_vm_fault_idle(bo); + if (unlikely(ret != 0)) { + retval = ret; + retval = (ret != -ERESTARTSYS) ? + VM_PAGER_ERROR : VM_PAGER_REFAULT; + goto out_unlock; + } ret = ttm_mem_io_lock(man, true); if (unlikely(ret != 0)) { @@ -144,17 +363,9 @@ ttm_bo_vm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps, } /* - * Strictly, we're not allowed to modify vma->vm_page_prot here, - * since the mmap_sem is only held in read mode. However, we - * modify only the caching bits of vma->vm_page_prot and - * consider those bits protected by - * the bo->rwlock, as we should be the only writers. - * There shouldn't really be any readers of these bits except - * within vm_insert_mixed()? fork? - * - * TODO: Add a list of vmas to the bo, and change the - * vma->vm_page_prot when the object changes caching policy, with - * the correct locks held. + * Make a local vma copy to modify the page_prot member + * and vm_flags if necessary. The vma parameter is protected + * by mmap_sem in write mode. */ mapprot = ufi->entry->protection; if (bo->mem.bus.is_iomem) { @@ -200,7 +411,7 @@ ttm_bo_vm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps, * an already populated PTE, or prefaulting error. */ - if (ret != 0 && i > 0) + if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0))) break; else if (unlikely(ret != 0)) { uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, @@ -222,6 +433,33 @@ out_unlock: ttm_bo_unreserve(bo); return retval; } +#endif + +#ifdef notyet +static void ttm_bo_vm_open(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = + (struct ttm_buffer_object *)vma->vm_private_data; + + WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping); + + (void)ttm_bo_reference(bo); +} + +static void ttm_bo_vm_close(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data; + + ttm_bo_unref(&bo); + vma->vm_private_data = NULL; +} + +static const struct vm_operations_struct ttm_bo_vm_ops = { + .fault = ttm_bo_vm_fault, + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close +}; +#endif void ttm_bo_vm_reference(struct uvm_object *uobj) @@ -272,214 +510,94 @@ static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, return bo; } -struct uvm_object * -ttm_bo_mmap(voff_t off, vsize_t size, struct ttm_bo_device *bdev) +#ifdef __linux__ +int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, + struct ttm_bo_device *bdev) { struct ttm_bo_driver *driver; struct ttm_buffer_object *bo; int ret; - bo = ttm_bo_vm_lookup(bdev, off >> PAGE_SHIFT, size >> PAGE_SHIFT); - if (unlikely(!bo)) { - ret = -EINVAL; - return NULL; - } + bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); + if (unlikely(!bo)) + return -EINVAL; driver = bo->bdev->driver; if (unlikely(!driver->verify_access)) { ret = -EPERM; goto out_unref; } -#ifdef notyet ret = driver->verify_access(bo, filp); if (unlikely(ret != 0)) goto out_unref; -#endif - bo->uobj.pgops = &ttm_bo_vm_ops; + vma->vm_ops = &ttm_bo_vm_ops; -#if 0 /* * Note: We're transferring the bo reference to * vma->vm_private_data here. */ vma->vm_private_data = bo; - vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP; -#else - bo->uobj.uo_refs++; -#endif - return &bo->uobj; -out_unref: - ttm_bo_unref(&bo); - return NULL; -} -EXPORT_SYMBOL(ttm_bo_mmap); - -#ifdef notyet -int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) -{ - if (vma->vm_pgoff != 0) - return -EACCES; - vma->vm_ops = &ttm_bo_vm_ops; - vma->vm_private_data = ttm_bo_reference(bo); - vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; + /* + * We'd like to use VM_PFNMAP on shared mappings, where + * (vma->vm_flags & VM_SHARED) != 0, for performance reasons, + * but for some reason VM_PFNMAP + x86 PAT + write-combine is very + * bad for performance. Until that has been sorted out, use + * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 + */ + vma->vm_flags |= VM_MIXEDMAP; + vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; return 0; +out_unref: + ttm_bo_unref(&bo); + return ret; } -#endif -EXPORT_SYMBOL(ttm_fbdev_mmap); - - -ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, - const char __user *wbuf, char __user *rbuf, size_t count, - off_t *f_pos, bool write) +#else +struct uvm_object * +ttm_bo_mmap(voff_t off, vsize_t size, struct ttm_bo_device *bdev) { - struct ttm_buffer_object *bo; struct ttm_bo_driver *driver; - struct ttm_bo_kmap_obj map; - unsigned long dev_offset = (*f_pos >> PAGE_SHIFT); - unsigned long kmap_offset; - unsigned long kmap_end; - unsigned long kmap_num; - size_t io_size; - unsigned int page_offset; - char *virtual; + struct ttm_buffer_object *bo; int ret; - bool no_wait = false; - bool dummy; - bo = ttm_bo_vm_lookup(bdev, dev_offset, 1); - if (unlikely(bo == NULL)) - return -EFAULT; + bo = ttm_bo_vm_lookup(bdev, off >> PAGE_SHIFT, size >> PAGE_SHIFT); + if (unlikely(!bo)) + return NULL; driver = bo->bdev->driver; if (unlikely(!driver->verify_access)) { ret = -EPERM; goto out_unref; } - +#ifdef notyet ret = driver->verify_access(bo, filp); if (unlikely(ret != 0)) goto out_unref; +#endif - kmap_offset = dev_offset - drm_vma_node_start(&bo->vma_node); - if (unlikely(kmap_offset >= bo->num_pages)) { - ret = -EFBIG; - goto out_unref; - } - - page_offset = *f_pos & PAGE_MASK; - io_size = bo->num_pages - kmap_offset; - io_size = (io_size << PAGE_SHIFT) - page_offset; - if (count < io_size) - io_size = count; - - kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; - kmap_num = kmap_end - kmap_offset + 1; - - ret = ttm_bo_reserve(bo, true, no_wait, false, 0); - - switch (ret) { - case 0: - break; - case -EBUSY: - ret = -EAGAIN; - goto out_unref; - default: - goto out_unref; - } - - ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); - if (unlikely(ret != 0)) { - ttm_bo_unreserve(bo); - goto out_unref; - } - - virtual = ttm_kmap_obj_virtual(&map, &dummy); - virtual += page_offset; - - if (write) - ret = copy_from_user(virtual, wbuf, io_size); - else - ret = copy_to_user(rbuf, virtual, io_size); - - ttm_bo_kunmap(&map); - ttm_bo_unreserve(bo); - ttm_bo_unref(&bo); - - if (unlikely(ret != 0)) - return -EFBIG; - - *f_pos += io_size; - - return io_size; + bo->uobj.pgops = &ttm_bo_vm_ops; + bo->uobj.uo_refs++; + return &bo->uobj; out_unref: ttm_bo_unref(&bo); - return ret; + return NULL; } +#endif +EXPORT_SYMBOL(ttm_bo_mmap); -ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf, - char __user *rbuf, size_t count, off_t *f_pos, - bool write) +#ifdef notyet +int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) { - struct ttm_bo_kmap_obj map; - unsigned long kmap_offset; - unsigned long kmap_end; - unsigned long kmap_num; - size_t io_size; - unsigned int page_offset; - char *virtual; - int ret; - bool no_wait = false; - bool dummy; - - kmap_offset = (*f_pos >> PAGE_SHIFT); - if (unlikely(kmap_offset >= bo->num_pages)) - return -EFBIG; - - page_offset = *f_pos & PAGE_MASK; - io_size = bo->num_pages - kmap_offset; - io_size = (io_size << PAGE_SHIFT) - page_offset; - if (count < io_size) - io_size = count; - - kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; - kmap_num = kmap_end - kmap_offset + 1; - - ret = ttm_bo_reserve(bo, true, no_wait, false, 0); - - switch (ret) { - case 0: - break; - case -EBUSY: - return -EAGAIN; - default: - return ret; - } - - ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); - if (unlikely(ret != 0)) { - ttm_bo_unreserve(bo); - return ret; - } - - virtual = ttm_kmap_obj_virtual(&map, &dummy); - virtual += page_offset; - - if (write) - ret = copy_from_user(virtual, wbuf, io_size); - else - ret = copy_to_user(rbuf, virtual, io_size); - - ttm_bo_kunmap(&map); - ttm_bo_unreserve(bo); - ttm_bo_unref(&bo); - - if (unlikely(ret != 0)) - return ret; - - *f_pos += io_size; + if (vma->vm_pgoff != 0) + return -EACCES; - return io_size; + vma->vm_ops = &ttm_bo_vm_ops; + vma->vm_private_data = ttm_bo_reference(bo); + vma->vm_flags |= VM_MIXEDMAP; + vma->vm_flags |= VM_IO | VM_DONTEXPAND; + return 0; } +EXPORT_SYMBOL(ttm_fbdev_mmap); +#endif diff --git a/sys/dev/pci/drm/ttm/ttm_execbuf_util.c b/sys/dev/pci/drm/ttm/ttm_execbuf_util.c index f773eacee4b..7d139a9d277 100644 --- a/sys/dev/pci/drm/ttm/ttm_execbuf_util.c +++ b/sys/dev/pci/drm/ttm/ttm_execbuf_util.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_execbuf_util.c,v 1.5 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -30,23 +29,13 @@ #include #include -static void ttm_eu_backoff_reservation_locked(struct list_head *list) +static void ttm_eu_backoff_reservation_reverse(struct list_head *list, + struct ttm_validate_buffer *entry) { - struct ttm_validate_buffer *entry; - - list_for_each_entry(entry, list, head) { + list_for_each_entry_continue_reverse(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; - if (!entry->reserved) - continue; - if (entry->removed) { - ttm_bo_add_to_lru(bo); - entry->removed = false; - - } - entry->reserved = false; - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + __ttm_bo_unreserve(bo); } } @@ -56,47 +45,14 @@ static void ttm_eu_del_from_lru_locked(struct list_head *list) list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; - if (!entry->reserved) - continue; + unsigned put_count = ttm_bo_del_from_lru(bo); - if (!entry->removed) { - entry->put_count = ttm_bo_del_from_lru(bo); - entry->removed = true; - } + ttm_bo_list_ref_sub(bo, put_count, true); } } -static void ttm_eu_list_ref_sub(struct list_head *list) -{ - struct ttm_validate_buffer *entry; - - list_for_each_entry(entry, list, head) { - struct ttm_buffer_object *bo = entry->bo; - - if (entry->put_count) { - ttm_bo_list_ref_sub(bo, entry->put_count, true); - entry->put_count = 0; - } - } -} - -static int ttm_eu_wait_unreserved_locked(struct list_head *list, - struct ttm_buffer_object *bo) -{ - struct ttm_bo_global *glob = bo->glob; - int ret; - - ttm_eu_del_from_lru_locked(list); - spin_unlock(&glob->lru_lock); - ret = ttm_bo_wait_unreserved(bo, true); - spin_lock(&glob->lru_lock); - if (unlikely(ret != 0)) - ttm_eu_backoff_reservation_locked(list); - return ret; -} - - -void ttm_eu_backoff_reservation(struct list_head *list) +void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, + struct list_head *list) { struct ttm_validate_buffer *entry; struct ttm_bo_global *glob; @@ -106,9 +62,18 @@ void ttm_eu_backoff_reservation(struct list_head *list) entry = list_first_entry(list, struct ttm_validate_buffer, head); glob = entry->bo->glob; + spin_lock(&glob->lru_lock); - ttm_eu_backoff_reservation_locked(list); + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + ttm_bo_add_to_lru(bo); + __ttm_bo_unreserve(bo); + } spin_unlock(&glob->lru_lock); + + if (ticket) + ww_acquire_fini(ticket); } EXPORT_SYMBOL(ttm_eu_backoff_reservation); @@ -124,78 +89,95 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation); * buffers in different orders. */ -int ttm_eu_reserve_buffers(struct list_head *list) +int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + struct list_head *list, bool intr, + struct list_head *dups) { struct ttm_bo_global *glob; struct ttm_validate_buffer *entry; int ret; - uint32_t val_seq; if (list_empty(list)) return 0; - list_for_each_entry(entry, list, head) { - entry->reserved = false; - entry->put_count = 0; - entry->removed = false; - } - entry = list_first_entry(list, struct ttm_validate_buffer, head); glob = entry->bo->glob; -retry: - spin_lock(&glob->lru_lock); - val_seq = entry->bo->bdev->val_seq++; + if (ticket) + ww_acquire_init(ticket, &reservation_ww_class); list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; -retry_this_bo: - ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq); - switch (ret) { - case 0: - break; - case -EBUSY: - ret = ttm_eu_wait_unreserved_locked(list, bo); - if (unlikely(ret != 0)) { - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return ret; + ret = __ttm_bo_reserve(bo, intr, (ticket == NULL), true, + ticket); + if (!ret && unlikely(atomic_read(&bo->cpu_writers) > 0)) { + __ttm_bo_unreserve(bo); + + ret = -EBUSY; + + } else if (ret == -EALREADY && dups) { + struct ttm_validate_buffer *safe = entry; + entry = list_prev_entry(entry, head); + list_del(&safe->head); + list_add(&safe->head, dups); + continue; + } + + if (!ret) { + if (!entry->shared) + continue; + + ret = reservation_object_reserve_shared(bo->resv); + if (!ret) + continue; + } + + /* uh oh, we lost out, drop every reservation and try + * to only reserve this buffer, then start over if + * this succeeds. + */ + ttm_eu_backoff_reservation_reverse(list, entry); + + if (ret == -EDEADLK && intr) { + ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, + ticket); + } else if (ret == -EDEADLK) { + ww_mutex_lock_slow(&bo->resv->lock, ticket); + ret = 0; + } + + if (!ret && entry->shared) + ret = reservation_object_reserve_shared(bo->resv); + + if (unlikely(ret != 0)) { + if (ret == -EINTR) + ret = -ERESTARTSYS; + if (ticket) { + ww_acquire_done(ticket); + ww_acquire_fini(ticket); } - goto retry_this_bo; - case -EAGAIN: - ttm_eu_backoff_reservation_locked(list); - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - ret = ttm_bo_wait_unreserved(bo, true); - if (unlikely(ret != 0)) - return ret; - goto retry; - default: - ttm_eu_backoff_reservation_locked(list); - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); return ret; } - entry->reserved = true; - if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { - ttm_eu_backoff_reservation_locked(list); - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return -EBUSY; - } + /* move this item to the front of the list, + * forces correct iteration of the loop without keeping track + */ + list_del(&entry->head); + list_add(&entry->head, list); } + if (ticket) + ww_acquire_done(ticket); + spin_lock(&glob->lru_lock); ttm_eu_del_from_lru_locked(list); spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return 0; } EXPORT_SYMBOL(ttm_eu_reserve_buffers); -void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) +void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + struct list_head *list, struct fence *fence) { struct ttm_validate_buffer *entry; struct ttm_buffer_object *bo; @@ -212,21 +194,18 @@ void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) glob = bo->glob; spin_lock(&glob->lru_lock); - spin_lock(&bdev->fence_lock); list_for_each_entry(entry, list, head) { bo = entry->bo; - entry->old_sync_obj = bo->sync_obj; - bo->sync_obj = driver->sync_obj_ref(sync_obj); - ttm_bo_unreserve_locked(bo); - entry->reserved = false; + if (entry->shared) + reservation_object_add_shared_fence(bo->resv, fence); + else + reservation_object_add_excl_fence(bo->resv, fence); + ttm_bo_add_to_lru(bo); + __ttm_bo_unreserve(bo); } - spin_unlock(&bdev->fence_lock); spin_unlock(&glob->lru_lock); - - list_for_each_entry(entry, list, head) { - if (entry->old_sync_obj) - driver->sync_obj_unref(&entry->old_sync_obj); - } + if (ticket) + ww_acquire_fini(ticket); } EXPORT_SYMBOL(ttm_eu_fence_buffer_objects); diff --git a/sys/dev/pci/drm/ttm/ttm_execbuf_util.h b/sys/dev/pci/drm/ttm/ttm_execbuf_util.h index 5c8fa203ff8..ca884e611dd 100644 --- a/sys/dev/pci/drm/ttm/ttm_execbuf_util.h +++ b/sys/dev/pci/drm/ttm/ttm_execbuf_util.h @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_execbuf_util.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -39,36 +38,36 @@ * * @head: list head for thread-private list. * @bo: refcounted buffer object pointer. - * @reserved: Indicates whether @bo has been reserved for validation. - * @removed: Indicates whether @bo has been removed from lru lists. - * @put_count: Number of outstanding references on bo::list_kref. - * @old_sync_obj: Pointer to a sync object about to be unreferenced + * @shared: should the fence be added shared? */ struct ttm_validate_buffer { struct list_head head; struct ttm_buffer_object *bo; - bool reserved; - bool removed; - int put_count; - void *old_sync_obj; + bool shared; }; /** * function ttm_eu_backoff_reservation * + * @ticket: ww_acquire_ctx from reserve call * @list: thread private list of ttm_validate_buffer structs. * * Undoes all buffer validation reservations for bos pointed to by * the list entries. */ -extern void ttm_eu_backoff_reservation(struct list_head *list); +extern void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, + struct list_head *list); /** * function ttm_eu_reserve_buffers * + * @ticket: [out] ww_acquire_ctx filled in by call, or NULL if only + * non-blocking reserves should be tried. * @list: thread private list of ttm_validate_buffer structs. + * @intr: should the wait be interruptible + * @dups: [out] optional list of duplicates. * * Tries to reserve bos pointed to by the list entries for validation. * If the function returns 0, all buffers are marked as "unfenced", @@ -80,9 +79,14 @@ extern void ttm_eu_backoff_reservation(struct list_head *list); * CPU write reservations to be cleared, and for other threads to * unreserve their buffers. * - * This function may return -ERESTART or -EAGAIN if the calling process - * receives a signal while waiting. In that case, no buffers on the list - * will be reserved upon return. + * If intr is set to true, this function may return -ERESTARTSYS if the + * calling process receives a signal while waiting. In that case, no + * buffers on the list will be reserved upon return. + * + * If dups is non NULL all buffers already reserved by the current thread + * (e.g. duplicates) are added to this list, otherwise -EALREADY is returned + * on the first already reserved buffer and all buffers from the list are + * unreserved again. * * Buffers reserved by this function should be unreserved by * a call to either ttm_eu_backoff_reservation() or @@ -90,13 +94,16 @@ extern void ttm_eu_backoff_reservation(struct list_head *list); * has failed. */ -extern int ttm_eu_reserve_buffers(struct list_head *list); +extern int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + struct list_head *list, bool intr, + struct list_head *dups); /** * function ttm_eu_fence_buffer_objects. * + * @ticket: ww_acquire_ctx from reserve call * @list: thread private list of ttm_validate_buffer structs. - * @sync_obj: The new sync object for the buffers. + * @fence: The new exclusive fence for the buffers. * * This function should be called when command submission is complete, and * it will add a new sync object to bos pointed to by entries on @list. @@ -104,6 +111,8 @@ extern int ttm_eu_reserve_buffers(struct list_head *list); * */ -extern void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj); +extern void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + struct list_head *list, + struct fence *fence); #endif diff --git a/sys/dev/pci/drm/ttm/ttm_lock.c b/sys/dev/pci/drm/ttm/ttm_lock.c deleted file mode 100644 index 5eafda1ba49..00000000000 --- a/sys/dev/pci/drm/ttm/ttm_lock.c +++ /dev/null @@ -1,342 +0,0 @@ -/* $OpenBSD: ttm_lock.c,v 1.7 2018/04/20 21:12:50 naddy Exp $ */ -/************************************************************************** - * - * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -/* - * Authors: Thomas Hellstrom - */ - -#include -#include -#include - -#define TTM_WRITE_LOCK_PENDING (1 << 0) -#define TTM_VT_LOCK_PENDING (1 << 1) -#define TTM_SUSPEND_LOCK_PENDING (1 << 2) -#define TTM_VT_LOCK (1 << 3) -#define TTM_SUSPEND_LOCK (1 << 4) - -void ttm_write_lock_downgrade(struct ttm_lock *); - -void ttm_lock_init(struct ttm_lock *lock) -{ - mtx_init(&lock->lock, IPL_NONE); - init_waitqueue_head(&lock->queue); - lock->rw = 0; - lock->flags = 0; - lock->kill_takers = false; - lock->signal = SIGKILL; -} -EXPORT_SYMBOL(ttm_lock_init); - -void ttm_read_unlock(struct ttm_lock *lock) -{ - spin_lock(&lock->lock); - if (--lock->rw == 0) - wake_up_all(&lock->queue); - spin_unlock(&lock->lock); -} -EXPORT_SYMBOL(ttm_read_unlock); - -#ifdef notyet -static bool __ttm_read_lock(struct ttm_lock *lock) -{ - bool locked = false; - - spin_lock(&lock->lock); - if (unlikely(lock->kill_takers)) { - send_sig(lock->signal, current, 0); - spin_unlock(&lock->lock); - return false; - } - if (lock->rw >= 0 && lock->flags == 0) { - ++lock->rw; - locked = true; - } - spin_unlock(&lock->lock); - return locked; -} -#endif - -int ttm_read_lock(struct ttm_lock *lock, bool interruptible) -{ - printf("%s stub\n", __func__); - return -ENOSYS; -#ifdef notyet - int ret = 0; - - if (interruptible) - ret = wait_event_interruptible(lock->queue, - __ttm_read_lock(lock)); - else - wait_event(lock->queue, __ttm_read_lock(lock)); - return ret; -#endif -} -EXPORT_SYMBOL(ttm_read_lock); - -#ifdef notyet -static bool __ttm_read_trylock(struct ttm_lock *lock, bool *locked) -{ - bool block = true; - - *locked = false; - - spin_lock(&lock->lock); - if (unlikely(lock->kill_takers)) { - send_sig(lock->signal, current, 0); - spin_unlock(&lock->lock); - return false; - } - if (lock->rw >= 0 && lock->flags == 0) { - ++lock->rw; - block = false; - *locked = true; - } else if (lock->flags == 0) { - block = false; - } - spin_unlock(&lock->lock); - - return !block; -} -#endif - -int ttm_read_trylock(struct ttm_lock *lock, bool interruptible) -{ - printf("%s stub\n", __func__); - return -ENOSYS; -#ifdef notyet - int ret = 0; - bool locked; - - if (interruptible) - ret = wait_event_interruptible - (lock->queue, __ttm_read_trylock(lock, &locked)); - else - wait_event(lock->queue, __ttm_read_trylock(lock, &locked)); - - if (unlikely(ret != 0)) { - BUG_ON(locked); - return ret; - } - - return (locked) ? 0 : -EBUSY; -#endif -} - -void ttm_write_unlock(struct ttm_lock *lock) -{ - spin_lock(&lock->lock); - lock->rw = 0; - wake_up_all(&lock->queue); - spin_unlock(&lock->lock); -} -EXPORT_SYMBOL(ttm_write_unlock); - -#ifdef notyet -static bool __ttm_write_lock(struct ttm_lock *lock) -{ - bool locked = false; - - spin_lock(&lock->lock); - if (unlikely(lock->kill_takers)) { - send_sig(lock->signal, current, 0); - spin_unlock(&lock->lock); - return false; - } - if (lock->rw == 0 && ((lock->flags & ~TTM_WRITE_LOCK_PENDING) == 0)) { - lock->rw = -1; - lock->flags &= ~TTM_WRITE_LOCK_PENDING; - locked = true; - } else { - lock->flags |= TTM_WRITE_LOCK_PENDING; - } - spin_unlock(&lock->lock); - return locked; -} -#endif - -int ttm_write_lock(struct ttm_lock *lock, bool interruptible) -{ - printf("%s stub\n", __func__); - return -ENOSYS; -#ifdef notyet - int ret = 0; - - if (interruptible) { - ret = wait_event_interruptible(lock->queue, - __ttm_write_lock(lock)); - if (unlikely(ret != 0)) { - spin_lock(&lock->lock); - lock->flags &= ~TTM_WRITE_LOCK_PENDING; - wake_up_all(&lock->queue); - spin_unlock(&lock->lock); - } - } else - wait_event(lock->queue, __ttm_read_lock(lock)); - - return ret; -#endif -} -EXPORT_SYMBOL(ttm_write_lock); - -void ttm_write_lock_downgrade(struct ttm_lock *lock) -{ - spin_lock(&lock->lock); - lock->rw = 1; - wake_up_all(&lock->queue); - spin_unlock(&lock->lock); -} - -#ifdef notyet -static int __ttm_vt_unlock(struct ttm_lock *lock) -{ - int ret = 0; - - spin_lock(&lock->lock); - if (unlikely(!(lock->flags & TTM_VT_LOCK))) - ret = -EINVAL; - lock->flags &= ~TTM_VT_LOCK; - wake_up_all(&lock->queue); - spin_unlock(&lock->lock); - - return ret; -} -#endif - -#ifdef notyet -static void ttm_vt_lock_remove(struct ttm_base_object **p_base) -{ - struct ttm_base_object *base = *p_base; - struct ttm_lock *lock = container_of(base, struct ttm_lock, base); - int ret; - - *p_base = NULL; - ret = __ttm_vt_unlock(lock); - BUG_ON(ret != 0); -} -#endif - -#ifdef notyet -static bool __ttm_vt_lock(struct ttm_lock *lock) -{ - bool locked = false; - - spin_lock(&lock->lock); - if (lock->rw == 0) { - lock->flags &= ~TTM_VT_LOCK_PENDING; - lock->flags |= TTM_VT_LOCK; - locked = true; - } else { - lock->flags |= TTM_VT_LOCK_PENDING; - } - spin_unlock(&lock->lock); - return locked; -} -#endif - -int ttm_vt_lock(struct ttm_lock *lock, - bool interruptible, - struct ttm_object_file *tfile) -{ - printf("%s stub\n", __func__); - return -ENOSYS; -#ifdef notyet - int ret = 0; - - if (interruptible) { - ret = wait_event_interruptible(lock->queue, - __ttm_vt_lock(lock)); - if (unlikely(ret != 0)) { - spin_lock(&lock->lock); - lock->flags &= ~TTM_VT_LOCK_PENDING; - wake_up_all(&lock->queue); - spin_unlock(&lock->lock); - return ret; - } - } else - wait_event(lock->queue, __ttm_vt_lock(lock)); - - /* - * Add a base-object, the destructor of which will - * make sure the lock is released if the client dies - * while holding it. - */ - - ret = ttm_base_object_init(tfile, &lock->base, false, - ttm_lock_type, &ttm_vt_lock_remove, NULL); - if (ret) - (void)__ttm_vt_unlock(lock); - else - lock->vt_holder = tfile; - - return ret; -#endif -} -EXPORT_SYMBOL(ttm_vt_lock); - -int ttm_vt_unlock(struct ttm_lock *lock) -{ - return ttm_ref_object_base_unref(lock->vt_holder, - lock->base.hash.key, TTM_REF_USAGE); -} -EXPORT_SYMBOL(ttm_vt_unlock); - -void ttm_suspend_unlock(struct ttm_lock *lock) -{ - spin_lock(&lock->lock); - lock->flags &= ~TTM_SUSPEND_LOCK; - wake_up_all(&lock->queue); - spin_unlock(&lock->lock); -} -EXPORT_SYMBOL(ttm_suspend_unlock); - -#ifdef notyet -static bool __ttm_suspend_lock(struct ttm_lock *lock) -{ - bool locked = false; - - spin_lock(&lock->lock); - if (lock->rw == 0) { - lock->flags &= ~TTM_SUSPEND_LOCK_PENDING; - lock->flags |= TTM_SUSPEND_LOCK; - locked = true; - } else { - lock->flags |= TTM_SUSPEND_LOCK_PENDING; - } - spin_unlock(&lock->lock); - return locked; -} -#endif - -void ttm_suspend_lock(struct ttm_lock *lock) -{ - printf("%s stub\n", __func__); -#ifdef notyet - wait_event(lock->queue, __ttm_suspend_lock(lock)); -#endif -} -EXPORT_SYMBOL(ttm_suspend_lock); diff --git a/sys/dev/pci/drm/ttm/ttm_lock.h b/sys/dev/pci/drm/ttm/ttm_lock.h deleted file mode 100644 index e2ae0deab0b..00000000000 --- a/sys/dev/pci/drm/ttm/ttm_lock.h +++ /dev/null @@ -1,247 +0,0 @@ -/* $OpenBSD: ttm_lock.h,v 1.5 2018/04/20 21:12:50 naddy Exp $ */ -/************************************************************************** - * - * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -/* - * Authors: Thomas Hellstrom - */ - -/** @file ttm_lock.h - * This file implements a simple replacement for the buffer manager use - * of the DRM heavyweight hardware lock. - * The lock is a read-write lock. Taking it in read mode and write mode - * is relatively fast, and intended for in-kernel use only. - * - * The vt mode is used only when there is a need to block all - * user-space processes from validating buffers. - * It's allowed to leave kernel space with the vt lock held. - * If a user-space process dies while having the vt-lock, - * it will be released during the file descriptor release. The vt lock - * excludes write lock and read lock. - * - * The suspend mode is used to lock out all TTM users when preparing for - * and executing suspend operations. - * - */ - -#ifndef _TTM_LOCK_H_ -#define _TTM_LOCK_H_ - -#include -#include - -/** - * struct ttm_lock - * - * @base: ttm base object used solely to release the lock if the client - * holding the lock dies. - * @queue: Queue for processes waiting for lock change-of-status. - * @lock: Spinlock protecting some lock members. - * @rw: Read-write lock counter. Protected by @lock. - * @flags: Lock state. Protected by @lock. - * @kill_takers: Boolean whether to kill takers of the lock. - * @signal: Signal to send when kill_takers is true. - */ - -struct ttm_lock { - struct ttm_base_object base; - wait_queue_head_t queue; - spinlock_t lock; - int32_t rw; - uint32_t flags; - bool kill_takers; - int signal; - struct ttm_object_file *vt_holder; -}; - - -/** - * ttm_lock_init - * - * @lock: Pointer to a struct ttm_lock - * Initializes the lock. - */ -extern void ttm_lock_init(struct ttm_lock *lock); - -/** - * ttm_read_unlock - * - * @lock: Pointer to a struct ttm_lock - * - * Releases a read lock. - */ -extern void ttm_read_unlock(struct ttm_lock *lock); - -/** - * ttm_read_lock - * - * @lock: Pointer to a struct ttm_lock - * @interruptible: Interruptible sleeping while waiting for a lock. - * - * Takes the lock in read mode. - * Returns: - * -ERESTARTSYS If interrupted by a signal and interruptible is true. - */ -extern int ttm_read_lock(struct ttm_lock *lock, bool interruptible); - -/** - * ttm_read_trylock - * - * @lock: Pointer to a struct ttm_lock - * @interruptible: Interruptible sleeping while waiting for a lock. - * - * Tries to take the lock in read mode. If the lock is already held - * in write mode, the function will return -EBUSY. If the lock is held - * in vt or suspend mode, the function will sleep until these modes - * are unlocked. - * - * Returns: - * -EBUSY The lock was already held in write mode. - * -ERESTARTSYS If interrupted by a signal and interruptible is true. - */ -extern int ttm_read_trylock(struct ttm_lock *lock, bool interruptible); - -/** - * ttm_write_unlock - * - * @lock: Pointer to a struct ttm_lock - * - * Releases a write lock. - */ -extern void ttm_write_unlock(struct ttm_lock *lock); - -/** - * ttm_write_lock - * - * @lock: Pointer to a struct ttm_lock - * @interruptible: Interruptible sleeping while waiting for a lock. - * - * Takes the lock in write mode. - * Returns: - * -ERESTARTSYS If interrupted by a signal and interruptible is true. - */ -extern int ttm_write_lock(struct ttm_lock *lock, bool interruptible); - -/** - * ttm_lock_downgrade - * - * @lock: Pointer to a struct ttm_lock - * - * Downgrades a write lock to a read lock. - */ -extern void ttm_lock_downgrade(struct ttm_lock *lock); - -/** - * ttm_suspend_lock - * - * @lock: Pointer to a struct ttm_lock - * - * Takes the lock in suspend mode. Excludes read and write mode. - */ -extern void ttm_suspend_lock(struct ttm_lock *lock); - -/** - * ttm_suspend_unlock - * - * @lock: Pointer to a struct ttm_lock - * - * Releases a suspend lock - */ -extern void ttm_suspend_unlock(struct ttm_lock *lock); - -/** - * ttm_vt_lock - * - * @lock: Pointer to a struct ttm_lock - * @interruptible: Interruptible sleeping while waiting for a lock. - * @tfile: Pointer to a struct ttm_object_file to register the lock with. - * - * Takes the lock in vt mode. - * Returns: - * -ERESTARTSYS If interrupted by a signal and interruptible is true. - * -ENOMEM: Out of memory when locking. - */ -extern int ttm_vt_lock(struct ttm_lock *lock, bool interruptible, - struct ttm_object_file *tfile); - -/** - * ttm_vt_unlock - * - * @lock: Pointer to a struct ttm_lock - * - * Releases a vt lock. - * Returns: - * -EINVAL If the lock was not held. - */ -extern int ttm_vt_unlock(struct ttm_lock *lock); - -/** - * ttm_write_unlock - * - * @lock: Pointer to a struct ttm_lock - * - * Releases a write lock. - */ -extern void ttm_write_unlock(struct ttm_lock *lock); - -/** - * ttm_write_lock - * - * @lock: Pointer to a struct ttm_lock - * @interruptible: Interruptible sleeping while waiting for a lock. - * - * Takes the lock in write mode. - * Returns: - * -ERESTARTSYS If interrupted by a signal and interruptible is true. - */ -extern int ttm_write_lock(struct ttm_lock *lock, bool interruptible); - -/** - * ttm_lock_set_kill - * - * @lock: Pointer to a struct ttm_lock - * @val: Boolean whether to kill processes taking the lock. - * @signal: Signal to send to the process taking the lock. - * - * The kill-when-taking-lock functionality is used to kill processes that keep - * on using the TTM functionality when its resources has been taken down, for - * example when the X server exits. A typical sequence would look like this: - * - X server takes lock in write mode. - * - ttm_lock_set_kill() is called with @val set to true. - * - As part of X server exit, TTM resources are taken down. - * - X server releases the lock on file release. - * - Another dri client wants to render, takes the lock and is killed. - * - */ -static inline void ttm_lock_set_kill(struct ttm_lock *lock, bool val, - int signal) -{ - lock->kill_takers = val; - if (val) - lock->signal = signal; -} - -#endif diff --git a/sys/dev/pci/drm/ttm/ttm_memory.c b/sys/dev/pci/drm/ttm/ttm_memory.c index 63fe874d192..5b4f74791fa 100644 --- a/sys/dev/pci/drm/ttm/ttm_memory.c +++ b/sys/dev/pci/drm/ttm/ttm_memory.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_memory.c,v 1.12 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -28,7 +27,7 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include +#include #include #include #include @@ -144,9 +143,7 @@ static ssize_t ttm_mem_zone_store(struct kobject *kobj, return size; } -#endif -#ifdef notyet static struct attribute *ttm_mem_zone_attrs[] = { &ttm_mem_sys, &ttm_mem_emer, @@ -194,7 +191,7 @@ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, if (from_wq) target = zone->swap_limit; - else if (DRM_SUSER(curproc)) + else if (capable(CAP_SYS_ADMIN)) target = zone->emer_mem; else target = zone->max_mem; @@ -233,13 +230,15 @@ static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, goto out; } out: - glob->task_queued = false; spin_unlock(&glob->lock); } -static void ttm_shrink_work(void *arg1) + + +static void ttm_shrink_work(struct work_struct *work) { - struct ttm_mem_global *glob = arg1; + struct ttm_mem_global *glob = + container_of(work, struct ttm_mem_global, work); ttm_shrink(glob, true, 0ULL); } @@ -294,7 +293,8 @@ static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob, zone->glob = glob; glob->zone_highmem = zone; ret = kobject_init_and_add( - &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); + &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s", + zone->name); if (unlikely(ret != 0)) { kobject_put(&zone->kobj); return ret; @@ -351,16 +351,12 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) { uint64_t mem; int ret; -#ifdef DRMDEBUG int i; struct ttm_mem_zone *zone; -#endif mtx_init(&glob->lock, IPL_TTY); - glob->swap_queue = taskq_create("ttm_swap", 1, IPL_TTY, 0); - glob->task_queued = false; - task_set(&glob->task, ttm_shrink_work, glob); - + glob->swap_queue = create_singlethread_workqueue("ttm_swap"); + INIT_WORK(&glob->work, ttm_shrink_work); ret = kobject_init_and_add( &glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting"); if (unlikely(ret != 0)) { @@ -382,13 +378,11 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) if (unlikely(ret != 0)) goto out_no_zone; #endif -#ifdef DRMDEBUG for (i = 0; i < glob->num_zones; ++i) { zone = glob->zones[i]; pr_info("Zone %7s: Available graphics memory: %llu kiB\n", zone->name, (unsigned long long)zone->max_mem >> 10); } -#endif ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); return 0; @@ -407,13 +401,14 @@ void ttm_mem_global_release(struct ttm_mem_global *glob) ttm_page_alloc_fini(); ttm_dma_page_alloc_fini(); - taskq_destroy(glob->swap_queue); + flush_workqueue(glob->swap_queue); + destroy_workqueue(glob->swap_queue); glob->swap_queue = NULL; for (i = 0; i < glob->num_zones; ++i) { zone = glob->zones[i]; kobject_del(&zone->kobj); kobject_put(&zone->kobj); - } + } kobject_del(&glob->kobj); kobject_put(&glob->kobj); } @@ -434,14 +429,11 @@ static void ttm_check_swapping(struct ttm_mem_global *glob) } } - if (glob->task_queued) - needs_swapping = false; - else - glob->task_queued = true; spin_unlock(&glob->lock); if (unlikely(needs_swapping)) - task_add(glob->swap_queue, &glob->task); + (void)queue_work(glob->swap_queue, &glob->work); + } static void ttm_mem_global_free_zone(struct ttm_mem_global *glob, @@ -483,7 +475,7 @@ static int ttm_mem_global_reserve(struct ttm_mem_global *glob, if (single_zone && zone != single_zone) continue; - limit = (DRM_SUSER(curproc)) ? + limit = (capable(CAP_SYS_ADMIN)) ? zone->emer_mem : zone->max_mem; if (zone->used_mem > limit) diff --git a/sys/dev/pci/drm/ttm/ttm_memory.h b/sys/dev/pci/drm/ttm/ttm_memory.h index e686c6338a0..402a283f032 100644 --- a/sys/dev/pci/drm/ttm/ttm_memory.h +++ b/sys/dev/pci/drm/ttm/ttm_memory.h @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_memory.h,v 1.7 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -72,9 +71,8 @@ struct ttm_mem_zone; struct ttm_mem_global { struct kobject kobj; struct ttm_mem_shrink *shrink; - struct taskq *swap_queue; - struct task task; - bool task_queued; + struct workqueue_struct *swap_queue; + struct work_struct work; spinlock_t lock; struct ttm_mem_zone *zones[TTM_MEM_MAX_ZONES]; unsigned int num_zones; diff --git a/sys/dev/pci/drm/ttm/ttm_module.c b/sys/dev/pci/drm/ttm/ttm_module.c deleted file mode 100644 index cd9037fecab..00000000000 --- a/sys/dev/pci/drm/ttm/ttm_module.c +++ /dev/null @@ -1,100 +0,0 @@ -/* $OpenBSD: ttm_module.c,v 1.3 2018/04/20 21:12:50 naddy Exp $ */ -/************************************************************************** - * - * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -/* - * Authors: Thomas Hellstrom - * Jerome Glisse - */ -#include -#include - -static DECLARE_WAIT_QUEUE_HEAD(exit_q); -atomic_t device_released; - -static struct device_type ttm_drm_class_type = { - .name = "ttm", - /** - * Add pm ops here. - */ -}; - -static void ttm_drm_class_device_release(struct device *dev) -{ - atomic_set(&device_released, 1); - wake_up_all(&exit_q); -} - -static struct device ttm_drm_class_device = { - .type = &ttm_drm_class_type, - .release = &ttm_drm_class_device_release -}; - -struct kobject *ttm_get_kobj(void) -{ - struct kobject *kobj = &ttm_drm_class_device.kobj; - BUG_ON(kobj == NULL); - return kobj; -} - -static int __init ttm_init(void) -{ - int ret; - - ret = dev_set_name(&ttm_drm_class_device, "ttm"); - if (unlikely(ret != 0)) - return ret; - - atomic_set(&device_released, 0); - ret = drm_class_device_register(&ttm_drm_class_device); - if (unlikely(ret != 0)) - goto out_no_dev_reg; - - return 0; -out_no_dev_reg: - atomic_set(&device_released, 1); - wake_up_all(&exit_q); - return ret; -} - -static void __exit ttm_exit(void) -{ - drm_class_device_unregister(&ttm_drm_class_device); - - /** - * Refuse to unload until the TTM device is released. - * Not sure this is 100% needed. - */ - - wait_event(exit_q, atomic_read(&device_released) == 1); -} - -module_init(ttm_init); -module_exit(ttm_exit); - -MODULE_AUTHOR("Thomas Hellstrom, Jerome Glisse"); -MODULE_DESCRIPTION("TTM memory manager subsystem (for DRM device)"); -MODULE_LICENSE("GPL and additional rights"); diff --git a/sys/dev/pci/drm/ttm/ttm_module.h b/sys/dev/pci/drm/ttm/ttm_module.h index 59f8bfb6f99..7281ee748cf 100644 --- a/sys/dev/pci/drm/ttm/ttm_module.h +++ b/sys/dev/pci/drm/ttm/ttm_module.h @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_module.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright 2008-2009 VMware, Inc., Palo Alto, CA., USA diff --git a/sys/dev/pci/drm/ttm/ttm_object.c b/sys/dev/pci/drm/ttm/ttm_object.c deleted file mode 100644 index d73f641988d..00000000000 --- a/sys/dev/pci/drm/ttm/ttm_object.c +++ /dev/null @@ -1,451 +0,0 @@ -/* $OpenBSD: ttm_object.c,v 1.11 2018/04/20 21:12:50 naddy Exp $ */ -/************************************************************************** - * - * Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -/* - * Authors: Thomas Hellstrom - */ -/** @file ttm_ref_object.c - * - * Base- and reference object implementation for the various - * ttm objects. Implements reference counting, minimal security checks - * and release on file close. - */ - -/** - * struct ttm_object_file - * - * @tdev: Pointer to the ttm_object_device. - * - * @lock: Lock that protects the ref_list list and the - * ref_hash hash tables. - * - * @ref_list: List of ttm_ref_objects to be destroyed at - * file release. - * - * @ref_hash: Hash tables of ref objects, one per ttm_ref_type, - * for fast lookup of ref objects given a base object. - */ - -#define pr_fmt(fmt) "[TTM] " fmt - -#include -#include -#include - -struct ttm_object_file { - struct ttm_object_device *tdev; - rwlock_t lock; - struct list_head ref_list; - struct drm_open_hash ref_hash[TTM_REF_NUM]; - struct kref refcount; -}; - -/** - * struct ttm_object_device - * - * @object_lock: lock that protects the object_hash hash table. - * - * @object_hash: hash table for fast lookup of object global names. - * - * @object_count: Per device object count. - * - * This is the per-device data structure needed for ttm object management. - */ - -struct ttm_object_device { - spinlock_t object_lock; - struct drm_open_hash object_hash; - atomic_t object_count; - struct ttm_mem_global *mem_glob; -}; - -/** - * struct ttm_ref_object - * - * @hash: Hash entry for the per-file object reference hash. - * - * @head: List entry for the per-file list of ref-objects. - * - * @kref: Ref count. - * - * @obj: Base object this ref object is referencing. - * - * @ref_type: Type of ref object. - * - * This is similar to an idr object, but it also has a hash table entry - * that allows lookup with a pointer to the referenced object as a key. In - * that way, one can easily detect whether a base object is referenced by - * a particular ttm_object_file. It also carries a ref count to avoid creating - * multiple ref objects if a ttm_object_file references the same base - * object more than once. - */ - -struct ttm_ref_object { - struct drm_hash_item hash; - struct list_head head; - struct kref kref; - enum ttm_ref_type ref_type; - struct ttm_base_object *obj; - struct ttm_object_file *tfile; -}; - -static inline struct ttm_object_file * -ttm_object_file_ref(struct ttm_object_file *tfile) -{ - kref_get(&tfile->refcount); - return tfile; -} - -static void ttm_object_file_destroy(struct kref *kref) -{ - struct ttm_object_file *tfile = - container_of(kref, struct ttm_object_file, refcount); - - kfree(tfile); -} - - -static inline void ttm_object_file_unref(struct ttm_object_file **p_tfile) -{ - struct ttm_object_file *tfile = *p_tfile; - - *p_tfile = NULL; - kref_put(&tfile->refcount, ttm_object_file_destroy); -} - - -int ttm_base_object_init(struct ttm_object_file *tfile, - struct ttm_base_object *base, - bool shareable, - enum ttm_object_type object_type, - void (*refcount_release) (struct ttm_base_object **), - void (*ref_obj_release) (struct ttm_base_object *, - enum ttm_ref_type ref_type)) -{ - struct ttm_object_device *tdev = tfile->tdev; - int ret; - - base->shareable = shareable; - base->tfile = ttm_object_file_ref(tfile); - base->refcount_release = refcount_release; - base->ref_obj_release = ref_obj_release; - base->object_type = object_type; - kref_init(&base->refcount); - spin_lock(&tdev->object_lock); - ret = drm_ht_just_insert_please_rcu(&tdev->object_hash, - &base->hash, - (unsigned long)base, 31, 0, 0); - spin_unlock(&tdev->object_lock); - if (unlikely(ret != 0)) - goto out_err0; - - ret = ttm_ref_object_add(tfile, base, TTM_REF_USAGE, NULL); - if (unlikely(ret != 0)) - goto out_err1; - - ttm_base_object_unref(&base); - - return 0; -out_err1: - spin_lock(&tdev->object_lock); - (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); - spin_unlock(&tdev->object_lock); -out_err0: - return ret; -} -EXPORT_SYMBOL(ttm_base_object_init); - -static void ttm_release_base(struct kref *kref) -{ - struct ttm_base_object *base = - container_of(kref, struct ttm_base_object, refcount); - struct ttm_object_device *tdev = base->tfile->tdev; - - spin_lock(&tdev->object_lock); - (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); - spin_unlock(&tdev->object_lock); - - /* - * Note: We don't use synchronize_rcu() here because it's far - * too slow. It's up to the user to free the object using - * call_rcu() or ttm_base_object_kfree(). - */ - - if (base->refcount_release) { - ttm_object_file_unref(&base->tfile); - base->refcount_release(&base); - } -} - -void ttm_base_object_unref(struct ttm_base_object **p_base) -{ - struct ttm_base_object *base = *p_base; - - *p_base = NULL; - - kref_put(&base->refcount, ttm_release_base); -} -EXPORT_SYMBOL(ttm_base_object_unref); - -struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file *tfile, - uint32_t key) -{ - struct ttm_object_device *tdev = tfile->tdev; - struct ttm_base_object *base; - struct drm_hash_item *hash; - int ret; - - mtx_enter(&tdev->object_lock); - ret = drm_ht_find_item(&tdev->object_hash, key, &hash); - - if (likely(ret == 0)) { - base = drm_hash_entry(hash, struct ttm_base_object, hash); - ret = kref_get_unless_zero(&base->refcount) ? 0 : -EINVAL; - } - mtx_leave(&tdev->object_lock); - - if (unlikely(ret != 0)) - return NULL; - - if (tfile != base->tfile && !base->shareable) { - pr_err("Attempted access of non-shareable object\n"); - ttm_base_object_unref(&base); - return NULL; - } - - return base; -} -EXPORT_SYMBOL(ttm_base_object_lookup); - -int ttm_ref_object_add(struct ttm_object_file *tfile, - struct ttm_base_object *base, - enum ttm_ref_type ref_type, bool *existed) -{ - struct drm_open_hash *ht = &tfile->ref_hash[ref_type]; - struct ttm_ref_object *ref; - struct drm_hash_item *hash; - struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; - int ret = -EINVAL; - - if (existed != NULL) - *existed = true; - - while (ret == -EINVAL) { - read_lock(&tfile->lock); - ret = drm_ht_find_item(ht, base->hash.key, &hash); - - if (ret == 0) { - ref = drm_hash_entry(hash, struct ttm_ref_object, hash); - kref_get(&ref->kref); - read_unlock(&tfile->lock); - break; - } - - read_unlock(&tfile->lock); - ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref), - false, false); - if (unlikely(ret != 0)) - return ret; - ref = kmalloc(sizeof(*ref), GFP_KERNEL); - if (unlikely(ref == NULL)) { - ttm_mem_global_free(mem_glob, sizeof(*ref)); - return -ENOMEM; - } - - ref->hash.key = base->hash.key; - ref->obj = base; - ref->tfile = tfile; - ref->ref_type = ref_type; - kref_init(&ref->kref); - - write_lock(&tfile->lock); - ret = drm_ht_insert_item(ht, &ref->hash); - - if (likely(ret == 0)) { - list_add_tail(&ref->head, &tfile->ref_list); - kref_get(&base->refcount); - write_unlock(&tfile->lock); - if (existed != NULL) - *existed = false; - break; - } - - write_unlock(&tfile->lock); - BUG_ON(ret != -EINVAL); - - ttm_mem_global_free(mem_glob, sizeof(*ref)); - kfree(ref); - } - - return ret; -} -EXPORT_SYMBOL(ttm_ref_object_add); - -static void ttm_ref_object_release(struct kref *kref) -{ - struct ttm_ref_object *ref = - container_of(kref, struct ttm_ref_object, kref); - struct ttm_base_object *base = ref->obj; - struct ttm_object_file *tfile = ref->tfile; - struct drm_open_hash *ht; - struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; - - ht = &tfile->ref_hash[ref->ref_type]; - (void)drm_ht_remove_item(ht, &ref->hash); - list_del(&ref->head); - write_unlock(&tfile->lock); - - if (ref->ref_type != TTM_REF_USAGE && base->ref_obj_release) - base->ref_obj_release(base, ref->ref_type); - - ttm_base_object_unref(&ref->obj); - ttm_mem_global_free(mem_glob, sizeof(*ref)); - kfree(ref); - write_lock(&tfile->lock); -} - -int ttm_ref_object_base_unref(struct ttm_object_file *tfile, - unsigned long key, enum ttm_ref_type ref_type) -{ - struct drm_open_hash *ht = &tfile->ref_hash[ref_type]; - struct ttm_ref_object *ref; - struct drm_hash_item *hash; - int ret; - - write_lock(&tfile->lock); - ret = drm_ht_find_item(ht, key, &hash); - if (unlikely(ret != 0)) { - write_unlock(&tfile->lock); - return -EINVAL; - } - ref = drm_hash_entry(hash, struct ttm_ref_object, hash); - kref_put(&ref->kref, ttm_ref_object_release); - write_unlock(&tfile->lock); - return 0; -} -EXPORT_SYMBOL(ttm_ref_object_base_unref); - -void ttm_object_file_release(struct ttm_object_file **p_tfile) -{ - struct ttm_ref_object *ref; - struct list_head *list; - unsigned int i; - struct ttm_object_file *tfile = *p_tfile; - - *p_tfile = NULL; - write_lock(&tfile->lock); - - /* - * Since we release the lock within the loop, we have to - * restart it from the beginning each time. - */ - - while (!list_empty(&tfile->ref_list)) { - list = tfile->ref_list.next; - ref = list_entry(list, struct ttm_ref_object, head); - ttm_ref_object_release(&ref->kref); - } - - for (i = 0; i < TTM_REF_NUM; ++i) - drm_ht_remove(&tfile->ref_hash[i]); - - write_unlock(&tfile->lock); - ttm_object_file_unref(&tfile); -} -EXPORT_SYMBOL(ttm_object_file_release); - -struct ttm_object_file *ttm_object_file_init(struct ttm_object_device *tdev, - unsigned int hash_order) -{ - struct ttm_object_file *tfile = kmalloc(sizeof(*tfile), GFP_KERNEL); - unsigned int i; - unsigned int j = 0; - int ret; - - if (unlikely(tfile == NULL)) - return NULL; - - rw_init(&tfile->lock, "ttmfl"); - tfile->tdev = tdev; - kref_init(&tfile->refcount); - INIT_LIST_HEAD(&tfile->ref_list); - - for (i = 0; i < TTM_REF_NUM; ++i) { - ret = drm_ht_create(&tfile->ref_hash[i], hash_order); - if (ret) { - j = i; - goto out_err; - } - } - - return tfile; -out_err: - for (i = 0; i < j; ++i) - drm_ht_remove(&tfile->ref_hash[i]); - - kfree(tfile); - - return NULL; -} -EXPORT_SYMBOL(ttm_object_file_init); - -struct ttm_object_device *ttm_object_device_init(struct ttm_mem_global - *mem_glob, - unsigned int hash_order) -{ - struct ttm_object_device *tdev = kmalloc(sizeof(*tdev), GFP_KERNEL); - int ret; - - if (unlikely(tdev == NULL)) - return NULL; - - tdev->mem_glob = mem_glob; - mtx_init(&tdev->object_lock, IPL_NONE); - atomic_set(&tdev->object_count, 0); - ret = drm_ht_create(&tdev->object_hash, hash_order); - - if (likely(ret == 0)) - return tdev; - - kfree(tdev); - return NULL; -} -EXPORT_SYMBOL(ttm_object_device_init); - -void ttm_object_device_release(struct ttm_object_device **p_tdev) -{ - struct ttm_object_device *tdev = *p_tdev; - - *p_tdev = NULL; - - spin_lock(&tdev->object_lock); - drm_ht_remove(&tdev->object_hash); - spin_unlock(&tdev->object_lock); - - kfree(tdev); -} -EXPORT_SYMBOL(ttm_object_device_release); diff --git a/sys/dev/pci/drm/ttm/ttm_object.h b/sys/dev/pci/drm/ttm/ttm_object.h deleted file mode 100644 index b59da0c78c5..00000000000 --- a/sys/dev/pci/drm/ttm/ttm_object.h +++ /dev/null @@ -1,275 +0,0 @@ -/* $OpenBSD: ttm_object.h,v 1.5 2018/04/20 21:12:50 naddy Exp $ */ -/************************************************************************** - * - * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -/* - * Authors: Thomas Hellstrom - */ -/** @file ttm_object.h - * - * Base- and reference object implementation for the various - * ttm objects. Implements reference counting, minimal security checks - * and release on file close. - */ - -#ifndef _TTM_OBJECT_H_ -#define _TTM_OBJECT_H_ - -#include -#include - -/** - * enum ttm_ref_type - * - * Describes what type of reference a ref object holds. - * - * TTM_REF_USAGE is a simple refcount on a base object. - * - * TTM_REF_SYNCCPU_READ is a SYNCCPU_READ reference on a - * buffer object. - * - * TTM_REF_SYNCCPU_WRITE is a SYNCCPU_WRITE reference on a - * buffer object. - * - */ - -enum ttm_ref_type { - TTM_REF_USAGE, - TTM_REF_SYNCCPU_READ, - TTM_REF_SYNCCPU_WRITE, - TTM_REF_NUM -}; - -/** - * enum ttm_object_type - * - * One entry per ttm object type. - * Device-specific types should use the - * ttm_driver_typex types. - */ - -enum ttm_object_type { - ttm_fence_type, - ttm_buffer_type, - ttm_lock_type, - ttm_driver_type0 = 256, - ttm_driver_type1, - ttm_driver_type2, - ttm_driver_type3, - ttm_driver_type4, - ttm_driver_type5 -}; - -struct ttm_object_file; -struct ttm_object_device; - -/** - * struct ttm_base_object - * - * @hash: hash entry for the per-device object hash. - * @type: derived type this object is base class for. - * @shareable: Other ttm_object_files can access this object. - * - * @tfile: Pointer to ttm_object_file of the creator. - * NULL if the object was not created by a user request. - * (kernel object). - * - * @refcount: Number of references to this object, not - * including the hash entry. A reference to a base object can - * only be held by a ref object. - * - * @refcount_release: A function to be called when there are - * no more references to this object. This function should - * destroy the object (or make sure destruction eventually happens), - * and when it is called, the object has - * already been taken out of the per-device hash. The parameter - * "base" should be set to NULL by the function. - * - * @ref_obj_release: A function to be called when a reference object - * with another ttm_ref_type than TTM_REF_USAGE is deleted. - * This function may, for example, release a lock held by a user-space - * process. - * - * This struct is intended to be used as a base struct for objects that - * are visible to user-space. It provides a global name, race-safe - * access and refcounting, minimal access contol and hooks for unref actions. - */ - -struct ttm_base_object { -#ifdef notyet - struct rcu_head rhead; -#endif - struct drm_hash_item hash; - enum ttm_object_type object_type; - bool shareable; - struct ttm_object_file *tfile; - struct kref refcount; - void (*refcount_release) (struct ttm_base_object **base); - void (*ref_obj_release) (struct ttm_base_object *base, - enum ttm_ref_type ref_type); -}; - -/** - * ttm_base_object_init - * - * @tfile: Pointer to a struct ttm_object_file. - * @base: The struct ttm_base_object to initialize. - * @shareable: This object is shareable with other applcations. - * (different @tfile pointers.) - * @type: The object type. - * @refcount_release: See the struct ttm_base_object description. - * @ref_obj_release: See the struct ttm_base_object description. - * - * Initializes a struct ttm_base_object. - */ - -extern int ttm_base_object_init(struct ttm_object_file *tfile, - struct ttm_base_object *base, - bool shareable, - enum ttm_object_type type, - void (*refcount_release) (struct ttm_base_object - **), - void (*ref_obj_release) (struct ttm_base_object - *, - enum ttm_ref_type - ref_type)); - -/** - * ttm_base_object_lookup - * - * @tfile: Pointer to a struct ttm_object_file. - * @key: Hash key - * - * Looks up a struct ttm_base_object with the key @key. - * Also verifies that the object is visible to the application, by - * comparing the @tfile argument and checking the object shareable flag. - */ - -extern struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file - *tfile, uint32_t key); - -/** - * ttm_base_object_unref - * - * @p_base: Pointer to a pointer referencing a struct ttm_base_object. - * - * Decrements the base object refcount and clears the pointer pointed to by - * p_base. - */ - -extern void ttm_base_object_unref(struct ttm_base_object **p_base); - -/** - * ttm_ref_object_add. - * - * @tfile: A struct ttm_object_file representing the application owning the - * ref_object. - * @base: The base object to reference. - * @ref_type: The type of reference. - * @existed: Upon completion, indicates that an identical reference object - * already existed, and the refcount was upped on that object instead. - * - * Adding a ref object to a base object is basically like referencing the - * base object, but a user-space application holds the reference. When the - * file corresponding to @tfile is closed, all its reference objects are - * deleted. A reference object can have different types depending on what - * it's intended for. It can be refcounting to prevent object destruction, - * When user-space takes a lock, it can add a ref object to that lock to - * make sure the lock is released if the application dies. A ref object - * will hold a single reference on a base object. - */ -extern int ttm_ref_object_add(struct ttm_object_file *tfile, - struct ttm_base_object *base, - enum ttm_ref_type ref_type, bool *existed); -/** - * ttm_ref_object_base_unref - * - * @key: Key representing the base object. - * @ref_type: Ref type of the ref object to be dereferenced. - * - * Unreference a ref object with type @ref_type - * on the base object identified by @key. If there are no duplicate - * references, the ref object will be destroyed and the base object - * will be unreferenced. - */ -extern int ttm_ref_object_base_unref(struct ttm_object_file *tfile, - unsigned long key, - enum ttm_ref_type ref_type); - -/** - * ttm_object_file_init - initialize a struct ttm_object file - * - * @tdev: A struct ttm_object device this file is initialized on. - * @hash_order: Order of the hash table used to hold the reference objects. - * - * This is typically called by the file_ops::open function. - */ - -extern struct ttm_object_file *ttm_object_file_init(struct ttm_object_device - *tdev, - unsigned int hash_order); - -/** - * ttm_object_file_release - release data held by a ttm_object_file - * - * @p_tfile: Pointer to pointer to the ttm_object_file object to release. - * *p_tfile will be set to NULL by this function. - * - * Releases all data associated by a ttm_object_file. - * Typically called from file_ops::release. The caller must - * ensure that there are no concurrent users of tfile. - */ - -extern void ttm_object_file_release(struct ttm_object_file **p_tfile); - -/** - * ttm_object device init - initialize a struct ttm_object_device - * - * @hash_order: Order of hash table used to hash the base objects. - * - * This function is typically called on device initialization to prepare - * data structures needed for ttm base and ref objects. - */ - -extern struct ttm_object_device *ttm_object_device_init - (struct ttm_mem_global *mem_glob, unsigned int hash_order); - -/** - * ttm_object_device_release - release data held by a ttm_object_device - * - * @p_tdev: Pointer to pointer to the ttm_object_device object to release. - * *p_tdev will be set to NULL by this function. - * - * Releases all data associated by a ttm_object_device. - * Typically called from driver::unload before the destruction of the - * device private data structure. - */ - -extern void ttm_object_device_release(struct ttm_object_device **p_tdev); - -#define ttm_base_object_kfree(__object, __base)\ - kfree_rcu(__object, __base.rhead) -#endif diff --git a/sys/dev/pci/drm/ttm/ttm_page_alloc.c b/sys/dev/pci/drm/ttm/ttm_page_alloc.c index eb32c3e5873..7bc8de531a4 100644 --- a/sys/dev/pci/drm/ttm/ttm_page_alloc.c +++ b/sys/dev/pci/drm/ttm/ttm_page_alloc.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_page_alloc.c,v 1.13 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright (c) Red Hat Inc. @@ -42,8 +41,6 @@ #include #endif -#include - #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct vm_page *)) #define SMALL_ALLOCATION 16 #define FREE_ALL_PAGES (~0U) @@ -65,7 +62,7 @@ struct ttm_page_pool { spinlock_t lock; bool fill_lock; struct pglist list; - int ttm_page_alloc_flags; + gfp_t gfp_flags; unsigned npages; char *name; unsigned long nfrees; @@ -139,36 +136,6 @@ static struct attribute *ttm_pool_attrs[] = { }; #endif -struct vm_page *ttm_uvm_alloc_page(void); -void ttm_uvm_free_page(struct vm_page *); - -struct vm_page * -ttm_uvm_alloc_page(void) -{ - struct pglist mlist; - int error; - - TAILQ_INIT(&mlist); - error = uvm_pglistalloc(PAGE_SIZE, dma_constraint.ucr_low, - dma_constraint.ucr_high, 0, 0, &mlist, - 1, UVM_PLA_WAITOK | UVM_PLA_ZERO); - if (error) - return NULL; - - return TAILQ_FIRST(&mlist); -} - -void -ttm_uvm_free_page(struct vm_page *m) -{ -#ifdef notyet - KASSERT(m->uobject == NULL); - KASSERT(m->wire_count == 1); - KASSERT((m->pg_flags & PG_FAKE) != 0); -#endif - uvm_pagefree(m); -} - static void ttm_pool_kobj_release(struct kobject *kobj) { struct ttm_pool_manager *m = @@ -234,7 +201,7 @@ static const struct sysfs_ops ttm_pool_sysfs_ops = { .show = &ttm_pool_show, .store = &ttm_pool_store, }; -#endif // notyet +#endif static struct kobj_type ttm_pool_kobj_type = { .release = &ttm_pool_kobj_release, @@ -283,7 +250,7 @@ static int set_pages_array_wc(struct vm_page **pages, int addrinarray) static int set_pages_array_uc(struct vm_page **pages, int addrinarray) { #ifdef TTM_HAS_AGP - printf("%s stub\n", __func__); + STUB(); return -ENOSYS; #ifdef notyet int i; @@ -323,7 +290,7 @@ static void ttm_pages_put(struct vm_page *pages[], unsigned npages) if (set_pages_array_wb(pages, npages)) pr_err("Failed to set %d pages to wb!\n", npages); for (i = 0; i < npages; ++i) - ttm_uvm_free_page(pages[i]); + __free_page(pages[i]); } static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, @@ -341,9 +308,12 @@ static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, * * @pool: to free the pages from * @free_all: If set to true will free all pages in pool + * @use_static: Safe to use static buffer **/ -static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) +static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, + bool use_static) { + static struct vm_page *static_buf[NUM_PAGES_TO_ALLOC]; unsigned long irq_flags; struct vm_page *p, *p1; struct vm_page **pages_to_free; @@ -354,8 +324,11 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) if (NUM_PAGES_TO_ALLOC < nr_free) npages_to_free = NUM_PAGES_TO_ALLOC; - pages_to_free = kmalloc(npages_to_free * sizeof(struct vm_page *), - GFP_KERNEL); + if (use_static) + pages_to_free = static_buf; + else + pages_to_free = kmalloc(npages_to_free * sizeof(struct vm_page *), + GFP_KERNEL); if (!pages_to_free) { pr_err("Failed to allocate memory for pool free operation\n"); return 0; @@ -420,54 +393,66 @@ restart: if (freed_pages) ttm_pages_put(pages_to_free, freed_pages); out: - kfree(pages_to_free); + if (pages_to_free != static_buf) + kfree(pages_to_free); return nr_free; } -#ifdef notyet -/* Get good estimation how many pages are free in pools */ -static int ttm_pool_get_num_unused_pages(void) -{ - unsigned i; - int total = 0; - for (i = 0; i < NUM_POOLS; ++i) - total += _manager->pools[i].npages; - - return total; -} -#endif - /** * Callback for mm to request pool to reduce number of page held. + * + * XXX: (dchinner) Deadlock warning! + * + * This code is crying out for a shrinker per pool.... */ #ifdef notyet -static int ttm_pool_mm_shrink(struct shrinker *shrink, - struct shrink_control *sc) +static unsigned long +ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) { - static atomic_t start_pool = ATOMIC_INIT(0); + static DEFINE_MUTEX(lock); + static unsigned start_pool; unsigned i; - unsigned pool_offset = atomic_add_return(1, &start_pool); + unsigned pool_offset; struct ttm_page_pool *pool; int shrink_pages = sc->nr_to_scan; + unsigned long freed = 0; - pool_offset = pool_offset % NUM_POOLS; + if (!mutex_trylock(&lock)) + return SHRINK_STOP; + pool_offset = ++start_pool % NUM_POOLS; /* select start pool in round robin fashion */ for (i = 0; i < NUM_POOLS; ++i) { unsigned nr_free = shrink_pages; if (shrink_pages == 0) break; pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; - shrink_pages = ttm_page_pool_free(pool, nr_free); + /* OK to use static buffer since global mutex is held. */ + shrink_pages = ttm_page_pool_free(pool, nr_free, true); + freed += nr_free - shrink_pages; } - /* return estimated number of unused pages in pool */ - return ttm_pool_get_num_unused_pages(); + mutex_unlock(&lock); + return freed; +} + + +static unsigned long +ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned i; + unsigned long count = 0; + + for (i = 0; i < NUM_POOLS; ++i) + count += _manager->pools[i].npages; + + return count; } #endif static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) { #ifdef notyet - manager->mm_shrink.shrink = &ttm_pool_mm_shrink; + manager->mm_shrink.count_objects = ttm_pool_shrink_count; + manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; manager->mm_shrink.seeks = 1; register_shrinker(&manager->mm_shrink); #endif @@ -475,7 +460,7 @@ static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) { - printf("%s stub\n", __func__); + STUB(); #ifdef notyet unregister_shrinker(&manager->mm_shrink); #endif @@ -516,7 +501,7 @@ static void ttm_handle_caching_state_failure(struct pglist *pages, /* Failed pages have to be freed */ for (i = 0; i < cpages; ++i) { TAILQ_REMOVE(pages, failed_pages[i], pageq); - ttm_uvm_free_page(failed_pages[i]); + __free_page(failed_pages[i]); } } @@ -526,7 +511,7 @@ static void ttm_handle_caching_state_failure(struct pglist *pages, * This function is reentrant if caller updates count depending on number of * pages returned in pages array. */ -static int ttm_alloc_new_pages(struct pglist *pages, int gfp_flags, +static int ttm_alloc_new_pages(struct pglist *pages, gfp_t gfp_flags, int ttm_flags, enum ttm_caching_state cstate, unsigned count) { struct vm_page **caching_array; @@ -545,7 +530,7 @@ static int ttm_alloc_new_pages(struct pglist *pages, int gfp_flags, } for (i = 0, cpages = 0; i < count; ++i) { - p = ttm_uvm_alloc_page(); + p = alloc_page(gfp_flags); if (!p) { pr_err("Unable to get page %u\n", i); @@ -637,8 +622,8 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, spin_unlock_irqrestore(&pool->lock, *irq_flags); TAILQ_INIT(&new_pages); - r = ttm_alloc_new_pages(&new_pages, pool->ttm_page_alloc_flags, - ttm_flags, cstate, alloc_size); + r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, + cstate, alloc_size); spin_lock_irqsave(&pool->lock, *irq_flags); if (!r) { @@ -684,11 +669,31 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, pool->npages = 0; goto out; } +#ifdef __linux__ + /* find the last pages to include for requested number of pages. Split + * pool to begin and halve it to reduce search space. */ + if (count <= pool->npages/2) { + i = 0; + list_for_each(p, &pool->list) { + if (++i == count) + break; + } + } else { + i = pool->npages + 1; + list_for_each_prev(p, &pool->list) { + if (--i == count) + break; + } + } + /* Cut 'count' number of pages from the pool */ + list_cut_position(pages, &pool->list, p); +#else for (i = 0; i < count; i++) { p = TAILQ_FIRST(&pool->list); TAILQ_REMOVE(&pool->list, p, pageq); TAILQ_INSERT_TAIL(pages, p, pageq); } +#endif pool->npages -= count; count = 0; out: @@ -708,7 +713,11 @@ static void ttm_put_pages(struct vm_page **pages, unsigned npages, int flags, /* No pool for this memory type so free the pages */ for (i = 0; i < npages; i++) { if (pages[i]) { - ttm_uvm_free_page(pages[i]); +#ifdef notyet + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); +#endif + __free_page(pages[i]); pages[i] = NULL; } } @@ -718,6 +727,10 @@ static void ttm_put_pages(struct vm_page **pages, unsigned npages, int flags, spin_lock_irqsave(&pool->lock, irq_flags); for (i = 0; i < npages; i++) { if (pages[i]) { +#ifdef notyet + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); +#endif TAILQ_INSERT_TAIL(&pool->list, pages[i], pageq); pages[i] = NULL; pool->npages++; @@ -734,7 +747,7 @@ static void ttm_put_pages(struct vm_page **pages, unsigned npages, int flags, } spin_unlock_irqrestore(&pool->lock, irq_flags); if (npages) - ttm_page_pool_free(pool, npages); + ttm_page_pool_free(pool, npages, false); } /* @@ -747,28 +760,23 @@ static int ttm_get_pages(struct vm_page **pages, unsigned npages, int flags, struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); struct pglist plist; struct vm_page *p = NULL; - const struct kmem_pa_mode *kp; - int gfp_flags = 0; + gfp_t gfp_flags = GFP_USER; unsigned count; int r; + /* set zero flag for page allocation if required */ + if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + /* No pool for cached pages */ if (pool == NULL) { - - if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { - if (flags & TTM_PAGE_FLAG_DMA32) - kp = &kp_dma_zero; - else - kp = &kp_zero; - } else if (flags & TTM_PAGE_FLAG_DMA32) { - kp = &kp_dma; - } else { - kp = &kp_dirty; - } + if (flags & TTM_PAGE_FLAG_DMA32) + gfp_flags |= GFP_DMA32; + else + gfp_flags |= GFP_HIGHUSER; for (r = 0; r < npages; ++r) { -// p = km_alloc(PAGE_SIZE, &kv_any, kp, &kd_waitok); - p = ttm_uvm_alloc_page(); + p = alloc_page(gfp_flags); if (!p) { pr_err("Unable to allocate page\n"); @@ -781,7 +789,7 @@ static int ttm_get_pages(struct vm_page **pages, unsigned npages, int flags, } /* combine zero flag to pool flags */ - gfp_flags |= pool->ttm_page_alloc_flags; + gfp_flags |= pool->gfp_flags; /* First we take pages from the pool */ TAILQ_INIT(&plist); @@ -793,9 +801,18 @@ static int ttm_get_pages(struct vm_page **pages, unsigned npages, int flags, /* clear the pages coming from the pool if requested */ if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { +#ifdef __linux + list_for_each_entry(p, &plist, lru) { + if (PageHighMem(p)) + clear_highpage(p); + else + clear_page(page_address(p)); + } +#else TAILQ_FOREACH(p, &plist, pageq) { pmap_zero_page(p); } +#endif } /* If pool didn't have enough pages allocate new one. */ @@ -820,14 +837,14 @@ static int ttm_get_pages(struct vm_page **pages, unsigned npages, int flags, return 0; } -static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, +static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, char *name) { mtx_init(&pool->lock, IPL_TTY); pool->fill_lock = false; TAILQ_INIT(&pool->list); pool->npages = pool->nfrees = 0; - pool->ttm_page_alloc_flags = flags; + pool->gfp_flags = flags; pool->name = name; } @@ -840,16 +857,18 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) pr_info("Initializing pool allocator\n"); _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); + if (!_manager) + return -ENOMEM; - ttm_page_pool_init_locked(&_manager->wc_pool, 0, "wc"); + ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc"); - ttm_page_pool_init_locked(&_manager->uc_pool, 0, "uc"); + ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc"); ttm_page_pool_init_locked(&_manager->wc_pool_dma32, - TTM_PAGE_FLAG_DMA32, "wc dma"); + GFP_USER | GFP_DMA32, "wc dma"); ttm_page_pool_init_locked(&_manager->uc_pool_dma32, - TTM_PAGE_FLAG_DMA32, "uc dma"); + GFP_USER | GFP_DMA32, "uc dma"); _manager->options.max_size = max_pages; _manager->options.small = SMALL_ALLOCATION; @@ -875,8 +894,9 @@ void ttm_page_alloc_fini(void) pr_info("Finalizing pool allocator\n"); ttm_pool_mm_shrink_fini(_manager); + /* OK to use static buffer since global mutex is no longer used. */ for (i = 0; i < NUM_POOLS; ++i) - ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES); + ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES, true); kobject_put(&_manager->kobj); _manager = NULL; @@ -938,7 +958,6 @@ void ttm_pool_unpopulate(struct ttm_tt *ttm) } EXPORT_SYMBOL(ttm_pool_unpopulate); -#ifdef notyet int ttm_page_alloc_debugfs(struct seq_file *m, void *data) { struct ttm_page_pool *p; @@ -959,5 +978,4 @@ int ttm_page_alloc_debugfs(struct seq_file *m, void *data) } return 0; } -#endif EXPORT_SYMBOL(ttm_page_alloc_debugfs); diff --git a/sys/dev/pci/drm/ttm/ttm_page_alloc.h b/sys/dev/pci/drm/ttm/ttm_page_alloc.h index 96103815246..b28dace1d2f 100644 --- a/sys/dev/pci/drm/ttm/ttm_page_alloc.h +++ b/sys/dev/pci/drm/ttm/ttm_page_alloc.h @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_page_alloc.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /* * Copyright (c) Red Hat Inc. @@ -30,6 +29,8 @@ #include #include +struct device; + /** * Initialize pool allocator. */ @@ -60,12 +61,10 @@ extern void ttm_pool_unpopulate(struct ttm_tt *ttm); /** * Output the state of pools to debugfs file */ -#ifdef notyet extern int ttm_page_alloc_debugfs(struct seq_file *m, void *data); -#endif -#ifdef CONFIG_SWIOTLB +#if defined(CONFIG_SWIOTLB) || defined(CONFIG_INTEL_IOMMU) /** * Initialize pool allocator. */ @@ -93,12 +92,19 @@ static inline int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, static inline void ttm_dma_page_alloc_fini(void) { return; } -#ifdef notyet static inline int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) { return 0; } -#endif +static inline int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, + struct device *dev) +{ + return -ENOMEM; +} +static inline void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, + struct device *dev) +{ +} #endif #endif diff --git a/sys/dev/pci/drm/ttm/ttm_page_alloc_dma.c b/sys/dev/pci/drm/ttm/ttm_page_alloc_dma.c new file mode 100644 index 00000000000..c984fb7c041 --- /dev/null +++ b/sys/dev/pci/drm/ttm/ttm_page_alloc_dma.c @@ -0,0 +1,1160 @@ +/* + * Copyright 2011 (c) Oracle Corp. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Author: Konrad Rzeszutek Wilk + */ + +/* + * A simple DMA pool losely based on dmapool.c. It has certain advantages + * over the DMA pools: + * - Pool collects resently freed pages for reuse (and hooks up to + * the shrinker). + * - Tracks currently in use pages + * - Tracks whether the page is UC, WB or cached (and reverts to WB + * when freed). + */ + +#if defined(CONFIG_SWIOTLB) || defined(CONFIG_INTEL_IOMMU) +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include /* for seq_printf */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef TTM_HAS_AGP +#include +#endif + +#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct vm_page *)) +#define SMALL_ALLOCATION 4 +#define FREE_ALL_PAGES (~0U) +/* times are in msecs */ +#define IS_UNDEFINED (0) +#define IS_WC (1<<1) +#define IS_UC (1<<2) +#define IS_CACHED (1<<3) +#define IS_DMA32 (1<<4) + +enum pool_type { + POOL_IS_UNDEFINED, + POOL_IS_WC = IS_WC, + POOL_IS_UC = IS_UC, + POOL_IS_CACHED = IS_CACHED, + POOL_IS_WC_DMA32 = IS_WC | IS_DMA32, + POOL_IS_UC_DMA32 = IS_UC | IS_DMA32, + POOL_IS_CACHED_DMA32 = IS_CACHED | IS_DMA32, +}; +/* + * The pool structure. There are usually six pools: + * - generic (not restricted to DMA32): + * - write combined, uncached, cached. + * - dma32 (up to 2^32 - so up 4GB): + * - write combined, uncached, cached. + * for each 'struct device'. The 'cached' is for pages that are actively used. + * The other ones can be shrunk by the shrinker API if neccessary. + * @pools: The 'struct device->dma_pools' link. + * @type: Type of the pool + * @lock: Protects the inuse_list and free_list from concurrnet access. Must be + * used with irqsave/irqrestore variants because pool allocator maybe called + * from delayed work. + * @inuse_list: Pool of pages that are in use. The order is very important and + * it is in the order that the TTM pages that are put back are in. + * @free_list: Pool of pages that are free to be used. No order requirements. + * @dev: The device that is associated with these pools. + * @size: Size used during DMA allocation. + * @npages_free: Count of available pages for re-use. + * @npages_in_use: Count of pages that are in use. + * @nfrees: Stats when pool is shrinking. + * @nrefills: Stats when the pool is grown. + * @gfp_flags: Flags to pass for alloc_page. + * @name: Name of the pool. + * @dev_name: Name derieved from dev - similar to how dev_info works. + * Used during shutdown as the dev_info during release is unavailable. + */ +struct dma_pool { + struct list_head pools; /* The 'struct device->dma_pools link */ + enum pool_type type; + spinlock_t lock; + struct list_head inuse_list; + struct list_head free_list; + struct device *dev; + unsigned size; + unsigned npages_free; + unsigned npages_in_use; + unsigned long nfrees; /* Stats when shrunk. */ + unsigned long nrefills; /* Stats when grown. */ + gfp_t gfp_flags; + char name[13]; /* "cached dma32" */ + char dev_name[64]; /* Constructed from dev */ +}; + +/* + * The accounting page keeping track of the allocated page along with + * the DMA address. + * @page_list: The link to the 'page_list' in 'struct dma_pool'. + * @vaddr: The virtual address of the page + * @dma: The bus address of the page. If the page is not allocated + * via the DMA API, it will be -1. + */ +struct dma_page { + struct list_head page_list; + void *vaddr; + struct vm_page *p; + dma_addr_t dma; +}; + +/* + * Limits for the pool. They are handled without locks because only place where + * they may change is in sysfs store. They won't have immediate effect anyway + * so forcing serialization to access them is pointless. + */ + +struct ttm_pool_opts { + unsigned alloc_size; + unsigned max_size; + unsigned small; +}; + +/* + * Contains the list of all of the 'struct device' and their corresponding + * DMA pools. Guarded by _mutex->lock. + * @pools: The link to 'struct ttm_pool_manager->pools' + * @dev: The 'struct device' associated with the 'pool' + * @pool: The 'struct dma_pool' associated with the 'dev' + */ +struct device_pools { + struct list_head pools; + struct device *dev; + struct dma_pool *pool; +}; + +/* + * struct ttm_pool_manager - Holds memory pools for fast allocation + * + * @lock: Lock used when adding/removing from pools + * @pools: List of 'struct device' and 'struct dma_pool' tuples. + * @options: Limits for the pool. + * @npools: Total amount of pools in existence. + * @shrinker: The structure used by [un|]register_shrinker + */ +struct ttm_pool_manager { + struct rwlock lock; + struct list_head pools; + struct ttm_pool_opts options; + unsigned npools; + struct shrinker mm_shrink; + struct kobject kobj; +}; + +static struct ttm_pool_manager *_manager; + +static struct attribute ttm_page_pool_max = { + .name = "pool_max_size", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_small = { + .name = "pool_small_allocation", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_alloc_size = { + .name = "pool_allocation_size", + .mode = S_IRUGO | S_IWUSR +}; + +static struct attribute *ttm_pool_attrs[] = { + &ttm_page_pool_max, + &ttm_page_pool_small, + &ttm_page_pool_alloc_size, + NULL +}; + +static void ttm_pool_kobj_release(struct kobject *kobj) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + kfree(m); +} + +static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, + const char *buffer, size_t size) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + int chars; + unsigned val; + chars = sscanf(buffer, "%u", &val); + if (chars == 0) + return size; + + /* Convert kb to number of pages */ + val = val / (PAGE_SIZE >> 10); + + if (attr == &ttm_page_pool_max) + m->options.max_size = val; + else if (attr == &ttm_page_pool_small) + m->options.small = val; + else if (attr == &ttm_page_pool_alloc_size) { + if (val > NUM_PAGES_TO_ALLOC*8) { + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + return size; + } else if (val > NUM_PAGES_TO_ALLOC) { + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + } + m->options.alloc_size = val; + } + + return size; +} + +static ssize_t ttm_pool_show(struct kobject *kobj, struct attribute *attr, + char *buffer) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + unsigned val = 0; + + if (attr == &ttm_page_pool_max) + val = m->options.max_size; + else if (attr == &ttm_page_pool_small) + val = m->options.small; + else if (attr == &ttm_page_pool_alloc_size) + val = m->options.alloc_size; + + val = val * (PAGE_SIZE >> 10); + + return snprintf(buffer, PAGE_SIZE, "%u\n", val); +} + +static const struct sysfs_ops ttm_pool_sysfs_ops = { + .show = &ttm_pool_show, + .store = &ttm_pool_store, +}; + +static struct kobj_type ttm_pool_kobj_type = { + .release = &ttm_pool_kobj_release, + .sysfs_ops = &ttm_pool_sysfs_ops, + .default_attrs = ttm_pool_attrs, +}; + +#ifndef CONFIG_X86 +static int set_pages_array_wb(struct vm_page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + unmap_page_from_agp(pages[i]); +#endif + return 0; +} + +static int set_pages_array_wc(struct vm_page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + map_page_into_agp(pages[i]); +#endif + return 0; +} + +static int set_pages_array_uc(struct vm_page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + map_page_into_agp(pages[i]); +#endif + return 0; +} +#endif /* for !CONFIG_X86 */ + +static int ttm_set_pages_caching(struct dma_pool *pool, + struct vm_page **pages, unsigned cpages) +{ + int r = 0; + /* Set page caching */ + if (pool->type & IS_UC) { + r = set_pages_array_uc(pages, cpages); + if (r) + pr_err("%s: Failed to set %d pages to uc!\n", + pool->dev_name, cpages); + } + if (pool->type & IS_WC) { + r = set_pages_array_wc(pages, cpages); + if (r) + pr_err("%s: Failed to set %d pages to wc!\n", + pool->dev_name, cpages); + } + return r; +} + +static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) +{ + dma_addr_t dma = d_page->dma; + dma_free_coherent(pool->dev, pool->size, d_page->vaddr, dma); + + kfree(d_page); + d_page = NULL; +} +static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool) +{ + struct dma_page *d_page; + + d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL); + if (!d_page) + return NULL; + + d_page->vaddr = dma_alloc_coherent(pool->dev, pool->size, + &d_page->dma, + pool->gfp_flags); + if (d_page->vaddr) { + if (is_vmalloc_addr(d_page->vaddr)) + d_page->p = vmalloc_to_page(d_page->vaddr); + else + d_page->p = virt_to_page(d_page->vaddr); + } else { + kfree(d_page); + d_page = NULL; + } + return d_page; +} +static enum pool_type ttm_to_type(int flags, enum ttm_caching_state cstate) +{ + enum pool_type type = IS_UNDEFINED; + + if (flags & TTM_PAGE_FLAG_DMA32) + type |= IS_DMA32; + if (cstate == tt_cached) + type |= IS_CACHED; + else if (cstate == tt_uncached) + type |= IS_UC; + else + type |= IS_WC; + + return type; +} + +static void ttm_pool_update_free_locked(struct dma_pool *pool, + unsigned freed_pages) +{ + pool->npages_free -= freed_pages; + pool->nfrees += freed_pages; + +} + +/* set memory back to wb and free the pages. */ +static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, + struct vm_page *pages[], unsigned npages) +{ + struct dma_page *d_page, *tmp; + + /* Don't set WB on WB page pool. */ + if (npages && !(pool->type & IS_CACHED) && + set_pages_array_wb(pages, npages)) + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, npages); + + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); + } +} + +static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) +{ + /* Don't set WB on WB page pool. */ + if (!(pool->type & IS_CACHED) && set_pages_array_wb(&d_page->p, 1)) + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, 1); + + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); +} + +/* + * Free pages from pool. + * + * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC + * number of pages in one go. + * + * @pool: to free the pages from + * @nr_free: If set to true will free all pages in pool + * @use_static: Safe to use static buffer + **/ +static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free, + bool use_static) +{ + static struct vm_page *static_buf[NUM_PAGES_TO_ALLOC]; + unsigned long irq_flags; + struct dma_page *dma_p, *tmp; + struct vm_page **pages_to_free; + struct list_head d_pages; + unsigned freed_pages = 0, + npages_to_free = nr_free; + + if (NUM_PAGES_TO_ALLOC < nr_free) + npages_to_free = NUM_PAGES_TO_ALLOC; +#if 0 + if (nr_free > 1) { + pr_debug("%s: (%s:%d) Attempting to free %d (%d) pages\n", + pool->dev_name, pool->name, current->pid, + npages_to_free, nr_free); + } +#endif + if (use_static) + pages_to_free = static_buf; + else + pages_to_free = kmalloc(npages_to_free * sizeof(struct vm_page *), + GFP_KERNEL); + + if (!pages_to_free) { + pr_err("%s: Failed to allocate memory for pool free operation\n", + pool->dev_name); + return 0; + } + INIT_LIST_HEAD(&d_pages); +restart: + spin_lock_irqsave(&pool->lock, irq_flags); + + /* We picking the oldest ones off the list */ + list_for_each_entry_safe_reverse(dma_p, tmp, &pool->free_list, + page_list) { + if (freed_pages >= npages_to_free) + break; + + /* Move the dma_page from one list to another. */ + list_move(&dma_p->page_list, &d_pages); + + pages_to_free[freed_pages++] = dma_p->p; + /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ + if (freed_pages >= NUM_PAGES_TO_ALLOC) { + + ttm_pool_update_free_locked(pool, freed_pages); + /** + * Because changing page caching is costly + * we unlock the pool to prevent stalling. + */ + spin_unlock_irqrestore(&pool->lock, irq_flags); + + ttm_dma_pages_put(pool, &d_pages, pages_to_free, + freed_pages); + + INIT_LIST_HEAD(&d_pages); + + if (likely(nr_free != FREE_ALL_PAGES)) + nr_free -= freed_pages; + + if (NUM_PAGES_TO_ALLOC >= nr_free) + npages_to_free = nr_free; + else + npages_to_free = NUM_PAGES_TO_ALLOC; + + freed_pages = 0; + + /* free all so restart the processing */ + if (nr_free) + goto restart; + + /* Not allowed to fall through or break because + * following context is inside spinlock while we are + * outside here. + */ + goto out; + + } + } + + /* remove range of pages from the pool */ + if (freed_pages) { + ttm_pool_update_free_locked(pool, freed_pages); + nr_free -= freed_pages; + } + + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (freed_pages) + ttm_dma_pages_put(pool, &d_pages, pages_to_free, freed_pages); +out: + if (pages_to_free != static_buf) + kfree(pages_to_free); + return nr_free; +} + +static void ttm_dma_free_pool(struct device *dev, enum pool_type type) +{ + struct device_pools *p; + struct dma_pool *pool; + + if (!dev) + return; + + mutex_lock(&_manager->lock); + list_for_each_entry_reverse(p, &_manager->pools, pools) { + if (p->dev != dev) + continue; + pool = p->pool; + if (pool->type != type) + continue; + + list_del(&p->pools); + kfree(p); + _manager->npools--; + break; + } + list_for_each_entry_reverse(pool, &dev->dma_pools, pools) { + if (pool->type != type) + continue; + /* Takes a spinlock.. */ + /* OK to use static buffer since global mutex is held. */ + ttm_dma_page_pool_free(pool, FREE_ALL_PAGES, true); + WARN_ON(((pool->npages_in_use + pool->npages_free) != 0)); + /* This code path is called after _all_ references to the + * struct device has been dropped - so nobody should be + * touching it. In case somebody is trying to _add_ we are + * guarded by the mutex. */ + list_del(&pool->pools); + kfree(pool); + break; + } + mutex_unlock(&_manager->lock); +} + +/* + * On free-ing of the 'struct device' this deconstructor is run. + * Albeit the pool might have already been freed earlier. + */ +static void ttm_dma_pool_release(struct device *dev, void *res) +{ + struct dma_pool *pool = *(struct dma_pool **)res; + + if (pool) + ttm_dma_free_pool(dev, pool->type); +} + +static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data) +{ + return *(struct dma_pool **)res == match_data; +} + +static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, + enum pool_type type) +{ + char *n[] = {"wc", "uc", "cached", " dma32", "unknown",}; + enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_UNDEFINED}; + struct device_pools *sec_pool = NULL; + struct dma_pool *pool = NULL, **ptr; + unsigned i; + int ret = -ENODEV; + char *p; + + if (!dev) + return NULL; + + ptr = devres_alloc(ttm_dma_pool_release, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return NULL; + + ret = -ENOMEM; + + pool = kmalloc_node(sizeof(struct dma_pool), GFP_KERNEL, + dev_to_node(dev)); + if (!pool) + goto err_mem; + + sec_pool = kmalloc_node(sizeof(struct device_pools), GFP_KERNEL, + dev_to_node(dev)); + if (!sec_pool) + goto err_mem; + + INIT_LIST_HEAD(&sec_pool->pools); + sec_pool->dev = dev; + sec_pool->pool = pool; + + INIT_LIST_HEAD(&pool->free_list); + INIT_LIST_HEAD(&pool->inuse_list); + INIT_LIST_HEAD(&pool->pools); + mtx_init(&pool->lock, IPL_TTY); + pool->dev = dev; + pool->npages_free = pool->npages_in_use = 0; + pool->nfrees = 0; + pool->gfp_flags = flags; + pool->size = PAGE_SIZE; + pool->type = type; + pool->nrefills = 0; + p = pool->name; + for (i = 0; i < 5; i++) { + if (type & t[i]) { + p += snprintf(p, sizeof(pool->name) - (p - pool->name), + "%s", n[i]); + } + } + *p = 0; + /* We copy the name for pr_ calls b/c when dma_pool_destroy is called + * - the kobj->name has already been deallocated.*/ + snprintf(pool->dev_name, sizeof(pool->dev_name), "%s %s", + dev_driver_string(dev), dev_name(dev)); + mutex_lock(&_manager->lock); + /* You can get the dma_pool from either the global: */ + list_add(&sec_pool->pools, &_manager->pools); + _manager->npools++; + /* or from 'struct device': */ + list_add(&pool->pools, &dev->dma_pools); + mutex_unlock(&_manager->lock); + + *ptr = pool; + devres_add(dev, ptr); + + return pool; +err_mem: + devres_free(ptr); + kfree(sec_pool); + kfree(pool); + return ERR_PTR(ret); +} + +static struct dma_pool *ttm_dma_find_pool(struct device *dev, + enum pool_type type) +{ + struct dma_pool *pool, *tmp, *found = NULL; + + if (type == IS_UNDEFINED) + return found; + + /* NB: We iterate on the 'struct dev' which has no spinlock, but + * it does have a kref which we have taken. The kref is taken during + * graphic driver loading - in the drm_pci_init it calls either + * pci_dev_get or pci_register_driver which both end up taking a kref + * on 'struct device'. + * + * On teardown, the graphic drivers end up quiescing the TTM (put_pages) + * and calls the dev_res deconstructors: ttm_dma_pool_release. The nice + * thing is at that point of time there are no pages associated with the + * driver so this function will not be called. + */ + list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) { + if (pool->type != type) + continue; + found = pool; + break; + } + return found; +} + +/* + * Free pages the pages that failed to change the caching state. If there + * are pages that have changed their caching state already put them to the + * pool. + */ +static void ttm_dma_handle_caching_state_failure(struct dma_pool *pool, + struct list_head *d_pages, + struct vm_page **failed_pages, + unsigned cpages) +{ + struct dma_page *d_page, *tmp; + struct vm_page *p; + unsigned i = 0; + + p = failed_pages[0]; + if (!p) + return; + /* Find the failed page. */ + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { + if (d_page->p != p) + continue; + /* .. and then progress over the full list. */ + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); + if (++i < cpages) + p = failed_pages[i]; + else + break; + } + +} + +/* + * Allocate 'count' pages, and put 'need' number of them on the + * 'pages' and as well on the 'dma_address' starting at 'dma_offset' offset. + * The full list of pages should also be on 'd_pages'. + * We return zero for success, and negative numbers as errors. + */ +static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, + struct list_head *d_pages, + unsigned count) +{ + struct vm_page **caching_array; + struct dma_page *dma_p; + struct vm_page *p; + int r = 0; + unsigned i, cpages; + unsigned max_cpages = min(count, + (unsigned)(PAGE_SIZE/sizeof(struct vm_page *))); + + /* allocate array for page caching change */ + caching_array = kmalloc(max_cpages*sizeof(struct vm_page *), GFP_KERNEL); + + if (!caching_array) { + pr_err("%s: Unable to allocate table for new pages\n", + pool->dev_name); + return -ENOMEM; + } + + if (count > 1) { + pr_debug("%s: (%s:%d) Getting %d pages\n", + pool->dev_name, pool->name, current->pid, count); + } + + for (i = 0, cpages = 0; i < count; ++i) { + dma_p = __ttm_dma_alloc_page(pool); + if (!dma_p) { + pr_err("%s: Unable to get page %u\n", + pool->dev_name, i); + + /* store already allocated pages in the pool after + * setting the caching state */ + if (cpages) { + r = ttm_set_pages_caching(pool, caching_array, + cpages); + if (r) + ttm_dma_handle_caching_state_failure( + pool, d_pages, caching_array, + cpages); + } + r = -ENOMEM; + goto out; + } + p = dma_p->p; +#ifdef CONFIG_HIGHMEM + /* gfp flags of highmem page should never be dma32 so we + * we should be fine in such case + */ + if (!PageHighMem(p)) +#endif + { + caching_array[cpages++] = p; + if (cpages == max_cpages) { + /* Note: Cannot hold the spinlock */ + r = ttm_set_pages_caching(pool, caching_array, + cpages); + if (r) { + ttm_dma_handle_caching_state_failure( + pool, d_pages, caching_array, + cpages); + goto out; + } + cpages = 0; + } + } + list_add(&dma_p->page_list, d_pages); + } + + if (cpages) { + r = ttm_set_pages_caching(pool, caching_array, cpages); + if (r) + ttm_dma_handle_caching_state_failure(pool, d_pages, + caching_array, cpages); + } +out: + kfree(caching_array); + return r; +} + +/* + * @return count of pages still required to fulfill the request. + */ +static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, + unsigned long *irq_flags) +{ + unsigned count = _manager->options.small; + int r = pool->npages_free; + + if (count > pool->npages_free) { + struct list_head d_pages; + + INIT_LIST_HEAD(&d_pages); + + spin_unlock_irqrestore(&pool->lock, *irq_flags); + + /* Returns how many more are neccessary to fulfill the + * request. */ + r = ttm_dma_pool_alloc_new_pages(pool, &d_pages, count); + + spin_lock_irqsave(&pool->lock, *irq_flags); + if (!r) { + /* Add the fresh to the end.. */ + list_splice(&d_pages, &pool->free_list); + ++pool->nrefills; + pool->npages_free += count; + r = count; + } else { + struct dma_page *d_page; + unsigned cpages = 0; + + pr_err("%s: Failed to fill %s pool (r:%d)!\n", + pool->dev_name, pool->name, r); + + list_for_each_entry(d_page, &d_pages, page_list) { + cpages++; + } + list_splice_tail(&d_pages, &pool->free_list); + pool->npages_free += cpages; + r = cpages; + } + } + return r; +} + +/* + * @return count of pages still required to fulfill the request. + * The populate list is actually a stack (not that is matters as TTM + * allocates one page at a time. + */ +static int ttm_dma_pool_get_pages(struct dma_pool *pool, + struct ttm_dma_tt *ttm_dma, + unsigned index) +{ + struct dma_page *d_page; + struct ttm_tt *ttm = &ttm_dma->ttm; + unsigned long irq_flags; + int count, r = -ENOMEM; + + spin_lock_irqsave(&pool->lock, irq_flags); + count = ttm_dma_page_pool_fill_locked(pool, &irq_flags); + if (count) { + d_page = list_first_entry(&pool->free_list, struct dma_page, page_list); + ttm->pages[index] = d_page->p; + ttm_dma->cpu_address[index] = d_page->vaddr; + ttm_dma->dma_address[index] = d_page->dma; + list_move_tail(&d_page->page_list, &ttm_dma->pages_list); + r = 0; + pool->npages_in_use += 1; + pool->npages_free -= 1; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + return r; +} + +/* + * On success pages list will hold count number of correctly + * cached pages. On failure will hold the negative return value (-ENOMEM, etc). + */ +int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; + struct dma_pool *pool; + enum pool_type type; + unsigned i; + gfp_t gfp_flags; + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + + type = ttm_to_type(ttm->page_flags, ttm->caching_state); + if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) + gfp_flags = GFP_USER | GFP_DMA32; + else + gfp_flags = GFP_HIGHUSER; + if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + pool = ttm_dma_find_pool(dev, type); + if (!pool) { + pool = ttm_dma_pool_init(dev, gfp_flags, type); + if (IS_ERR_OR_NULL(pool)) { + return -ENOMEM; + } + } + + INIT_LIST_HEAD(&ttm_dma->pages_list); + for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_dma_pool_get_pages(pool, ttm_dma, i); + if (ret != 0) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + false, false); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return ret; + } + } + + ttm->state = tt_unbound; + return 0; +} +EXPORT_SYMBOL_GPL(ttm_dma_populate); + +/* Put all pages in pages list to correct pool to wait for reuse */ +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + struct dma_pool *pool; + struct dma_page *d_page, *next; + enum pool_type type; + bool is_cached = false; + unsigned count = 0, i, npages = 0; + unsigned long irq_flags; + + type = ttm_to_type(ttm->page_flags, ttm->caching_state); + pool = ttm_dma_find_pool(dev, type); + if (!pool) + return; + + is_cached = (ttm_dma_find_pool(pool->dev, + ttm_to_type(ttm->page_flags, tt_cached)) == pool); + + /* make sure pages array match list and count number of pages */ + list_for_each_entry(d_page, &ttm_dma->pages_list, page_list) { + ttm->pages[count] = d_page->p; + count++; + } + + spin_lock_irqsave(&pool->lock, irq_flags); + pool->npages_in_use -= count; + if (is_cached) { + pool->nfrees += count; + } else { + pool->npages_free += count; + list_splice(&ttm_dma->pages_list, &pool->free_list); + /* + * Wait to have at at least NUM_PAGES_TO_ALLOC number of pages + * to free in order to minimize calls to set_memory_wb(). + */ + if (pool->npages_free >= (_manager->options.max_size + + NUM_PAGES_TO_ALLOC)) + npages = pool->npages_free - _manager->options.max_size; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (is_cached) { + list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + d_page->p); + ttm_dma_page_put(pool, d_page); + } + } else { + for (i = 0; i < count; i++) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + ttm->pages[i]); + } + } + + INIT_LIST_HEAD(&ttm_dma->pages_list); + for (i = 0; i < ttm->num_pages; i++) { + ttm->pages[i] = NULL; + ttm_dma->cpu_address[i] = 0; + ttm_dma->dma_address[i] = 0; + } + + /* shrink pool if necessary (only on !is_cached pools)*/ + if (npages) + ttm_dma_page_pool_free(pool, npages, false); + ttm->state = tt_unpopulated; +} +EXPORT_SYMBOL_GPL(ttm_dma_unpopulate); + +/** + * Callback for mm to request pool to reduce number of page held. + * + * XXX: (dchinner) Deadlock warning! + * + * I'm getting sadder as I hear more pathetical whimpers about needing per-pool + * shrinkers + */ +static unsigned long +ttm_dma_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + static unsigned start_pool; + unsigned idx = 0; + unsigned pool_offset; + unsigned shrink_pages = sc->nr_to_scan; + struct device_pools *p; + unsigned long freed = 0; + + if (list_empty(&_manager->pools)) + return SHRINK_STOP; + + if (!mutex_trylock(&_manager->lock)) + return SHRINK_STOP; + if (!_manager->npools) + goto out; + pool_offset = ++start_pool % _manager->npools; + list_for_each_entry(p, &_manager->pools, pools) { + unsigned nr_free; + + if (!p->dev) + continue; + if (shrink_pages == 0) + break; + /* Do it in round-robin fashion. */ + if (++idx < pool_offset) + continue; + nr_free = shrink_pages; + /* OK to use static buffer since global mutex is held. */ + shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free, true); + freed += nr_free - shrink_pages; + + pr_debug("%s: (%s:%d) Asked to shrink %d, have %d more to go\n", + p->pool->dev_name, p->pool->name, current->pid, + nr_free, shrink_pages); + } +out: + mutex_unlock(&_manager->lock); + return freed; +} + +static unsigned long +ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + struct device_pools *p; + unsigned long count = 0; + + if (!mutex_trylock(&_manager->lock)) + return 0; + list_for_each_entry(p, &_manager->pools, pools) + count += p->pool->npages_free; + mutex_unlock(&_manager->lock); + return count; +} + +static void ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) +{ + manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count; + manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan; + manager->mm_shrink.seeks = 1; + register_shrinker(&manager->mm_shrink); +} + +static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) +{ + unregister_shrinker(&manager->mm_shrink); +} + +int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) +{ + int ret = -ENOMEM; + + WARN_ON(_manager); + + pr_info("Initializing DMA pool allocator\n"); + + _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); + if (!_manager) + goto err; + + rw_init(&_manager->lock, "ttm_manager"); + INIT_LIST_HEAD(&_manager->pools); + + _manager->options.max_size = max_pages; + _manager->options.small = SMALL_ALLOCATION; + _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; + + /* This takes care of auto-freeing the _manager */ + ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, + &glob->kobj, "dma_pool"); + if (unlikely(ret != 0)) { + kobject_put(&_manager->kobj); + goto err; + } + ttm_dma_pool_mm_shrink_init(_manager); + return 0; +err: + return ret; +} + +void ttm_dma_page_alloc_fini(void) +{ + struct device_pools *p, *t; + + pr_info("Finalizing DMA pool allocator\n"); + ttm_dma_pool_mm_shrink_fini(_manager); + + list_for_each_entry_safe_reverse(p, t, &_manager->pools, pools) { + dev_dbg(p->dev, "(%s:%d) Freeing.\n", p->pool->name, + current->pid); + WARN_ON(devres_destroy(p->dev, ttm_dma_pool_release, + ttm_dma_pool_match, p->pool)); + ttm_dma_free_pool(p->dev, p->pool->type); + } + kobject_put(&_manager->kobj); + _manager = NULL; +} + +int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) +{ + struct device_pools *p; + struct dma_pool *pool = NULL; + char *h[] = {"pool", "refills", "pages freed", "inuse", "available", + "name", "virt", "busaddr"}; + + if (!_manager) { + seq_printf(m, "No pool allocator running.\n"); + return 0; + } + seq_printf(m, "%13s %12s %13s %8s %8s %8s\n", + h[0], h[1], h[2], h[3], h[4], h[5]); + mutex_lock(&_manager->lock); + list_for_each_entry(p, &_manager->pools, pools) { + struct device *dev = p->dev; + if (!dev) + continue; + pool = p->pool; + seq_printf(m, "%13s %12ld %13ld %8d %8d %8s\n", + pool->name, pool->nrefills, + pool->nfrees, pool->npages_in_use, + pool->npages_free, + pool->dev_name); + } + mutex_unlock(&_manager->lock); + return 0; +} +EXPORT_SYMBOL_GPL(ttm_dma_page_alloc_debugfs); + +#endif diff --git a/sys/dev/pci/drm/ttm/ttm_placement.h b/sys/dev/pci/drm/ttm/ttm_placement.h index 2e708956bf7..8ed44f9bbdf 100644 --- a/sys/dev/pci/drm/ttm/ttm_placement.h +++ b/sys/dev/pci/drm/ttm/ttm_placement.h @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_placement.h,v 1.3 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -66,6 +65,8 @@ * reference the buffer. * TTM_PL_FLAG_NO_EVICT means that the buffer may never * be evicted to make room for other buffers. + * TTM_PL_FLAG_TOPDOWN requests to be placed from the + * top of the memory area, instead of the bottom. */ #define TTM_PL_FLAG_CACHED (1 << 16) @@ -73,6 +74,7 @@ #define TTM_PL_FLAG_WC (1 << 18) #define TTM_PL_FLAG_SHARED (1 << 20) #define TTM_PL_FLAG_NO_EVICT (1 << 21) +#define TTM_PL_FLAG_TOPDOWN (1 << 22) #define TTM_PL_MASK_CACHING (TTM_PL_FLAG_CACHED | \ TTM_PL_FLAG_UNCACHED | \ diff --git a/sys/dev/pci/drm/ttm/ttm_tt.c b/sys/dev/pci/drm/ttm/ttm_tt.c index cf9f5374c66..a9b74d8a47a 100644 --- a/sys/dev/pci/drm/ttm/ttm_tt.c +++ b/sys/dev/pci/drm/ttm/ttm_tt.c @@ -1,4 +1,3 @@ -/* $OpenBSD: ttm_tt.c,v 1.6 2018/04/20 16:09:37 deraadt Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -49,9 +48,12 @@ static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) { - ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*)); - ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages, - sizeof(*ttm->dma_address)); + ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, + sizeof(*ttm->ttm.pages) + + sizeof(*ttm->dma_address) + + sizeof(*ttm->cpu_address)); + ttm->cpu_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages); + ttm->dma_address = (void *) (ttm->cpu_address + ttm->ttm.num_pages); } #ifdef CONFIG_X86 @@ -111,11 +113,7 @@ static int ttm_tt_set_caching(struct ttm_tt *ttm, } if (ttm->caching_state == tt_cached) -#ifdef notyet drm_clflush_pages(ttm->pages, ttm->num_pages); -#else - printf("%s partial stub\n", __func__); -#endif for (i = 0; i < ttm->num_pages; ++i) { cur_page = ttm->pages[i]; @@ -168,9 +166,8 @@ void ttm_tt_destroy(struct ttm_tt *ttm) ttm_tt_unbind(ttm); } - if (ttm->state == tt_unbound) { - ttm->bdev->driver->ttm_tt_unpopulate(ttm); - } + if (ttm->state == tt_unbound) + ttm_tt_unpopulate(ttm); if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && ttm->swap_storage) @@ -227,7 +224,7 @@ int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, INIT_LIST_HEAD(&ttm_dma->pages_list); ttm_dma_tt_alloc_page_directory(ttm_dma); - if (!ttm->pages || !ttm_dma->dma_address) { + if (!ttm->pages) { ttm_tt_destroy(ttm); pr_err("Failed allocating page table\n"); return -ENOMEM; @@ -242,7 +239,7 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) drm_free_large(ttm->pages); ttm->pages = NULL; - drm_free_large(ttm_dma->dma_address); + ttm_dma->cpu_address = NULL; ttm_dma->dma_address = NULL; } EXPORT_SYMBOL(ttm_dma_tt_fini); @@ -315,6 +312,7 @@ int ttm_tt_swapin(struct ttm_tt *ttm) ttm->swap_storage = NULL; ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; + return 0; out_err: return ret; @@ -375,3 +373,28 @@ out_err: return ret; } + +static void ttm_tt_clear_mapping(struct ttm_tt *ttm) +{ + int i; + struct vm_page *page; + + if (ttm->page_flags & TTM_PAGE_FLAG_SG) + return; + + for (i = 0; i < ttm->num_pages; ++i) { + page = ttm->pages[i]; + if (unlikely(page == NULL)) + continue; + pmap_page_protect(page, PROT_NONE); + } +} + +void ttm_tt_unpopulate(struct ttm_tt *ttm) +{ + if (ttm->state == tt_unpopulated) + return; + + ttm_tt_clear_mapping(ttm); + ttm->bdev->driver->ttm_tt_unpopulate(ttm); +} -- cgit v1.2.3