diff options
Diffstat (limited to 'lib/mesa/src/amd/vulkan/radv_cmd_buffer.c')
| -rw-r--r-- | lib/mesa/src/amd/vulkan/radv_cmd_buffer.c | 1698 |
1 files changed, 419 insertions, 1279 deletions
diff --git a/lib/mesa/src/amd/vulkan/radv_cmd_buffer.c b/lib/mesa/src/amd/vulkan/radv_cmd_buffer.c index fd155411f..9517e7a13 100644 --- a/lib/mesa/src/amd/vulkan/radv_cmd_buffer.c +++ b/lib/mesa/src/amd/vulkan/radv_cmd_buffer.c @@ -32,15 +32,11 @@ #include "vk_format.h" #include "radv_meta.h" -#include "ac_debug.h" - static void radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout src_layout, VkImageLayout dst_layout, - uint32_t src_family, - uint32_t dst_family, - const VkImageSubresourceRange *range, + VkImageSubresourceRange range, VkImageAspectFlags pending_clears); const struct radv_dynamic_state default_dynamic_state = { @@ -114,25 +110,6 @@ radv_dynamic_state_copy(struct radv_dynamic_state *dest, dest->stencil_reference = src->stencil_reference; } -bool radv_cmd_buffer_uses_mec(struct radv_cmd_buffer *cmd_buffer) -{ - return cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE && - cmd_buffer->device->physical_device->rad_info.chip_class >= CIK; -} - -enum ring_type radv_queue_family_to_ring(int f) { - switch (f) { - case RADV_QUEUE_GENERAL: - return RING_GFX; - case RADV_QUEUE_COMPUTE: - return RING_COMPUTE; - case RADV_QUEUE_TRANSFER: - return RING_DMA; - default: - unreachable("Unknown queue family"); - } -} - static VkResult radv_create_cmd_buffer( struct radv_device * device, struct radv_cmd_pool * pool, @@ -141,7 +118,7 @@ static VkResult radv_create_cmd_buffer( { struct radv_cmd_buffer *cmd_buffer; VkResult result; - unsigned ring; + cmd_buffer = vk_alloc(&pool->alloc, sizeof(*cmd_buffer), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (cmd_buffer == NULL) @@ -155,19 +132,14 @@ static VkResult radv_create_cmd_buffer( if (pool) { list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers); - cmd_buffer->queue_family_index = pool->queue_family_index; - } else { /* Init the pool_link so we can safefly call list_del when we destroy * the command buffer */ list_inithead(&cmd_buffer->pool_link); - cmd_buffer->queue_family_index = RADV_QUEUE_GENERAL; } - ring = radv_queue_family_to_ring(cmd_buffer->queue_family_index); - - cmd_buffer->cs = device->ws->cs_create(device->ws, ring); + cmd_buffer->cs = device->ws->cs_create(device->ws, RING_GFX); if (!cmd_buffer->cs) { result = VK_ERROR_OUT_OF_HOST_MEMORY; goto fail; @@ -187,54 +159,6 @@ fail: return result; } -static void -radv_cmd_buffer_destroy(struct radv_cmd_buffer *cmd_buffer) -{ - list_del(&cmd_buffer->pool_link); - - list_for_each_entry_safe(struct radv_cmd_buffer_upload, up, - &cmd_buffer->upload.list, list) { - cmd_buffer->device->ws->buffer_destroy(up->upload_bo); - list_del(&up->list); - free(up); - } - - if (cmd_buffer->upload.upload_bo) - cmd_buffer->device->ws->buffer_destroy(cmd_buffer->upload.upload_bo); - cmd_buffer->device->ws->cs_destroy(cmd_buffer->cs); - free(cmd_buffer->push_descriptors.set.mapped_ptr); - vk_free(&cmd_buffer->pool->alloc, cmd_buffer); -} - -static void radv_reset_cmd_buffer(struct radv_cmd_buffer *cmd_buffer) -{ - - cmd_buffer->device->ws->cs_reset(cmd_buffer->cs); - - list_for_each_entry_safe(struct radv_cmd_buffer_upload, up, - &cmd_buffer->upload.list, list) { - cmd_buffer->device->ws->buffer_destroy(up->upload_bo); - list_del(&up->list); - free(up); - } - - cmd_buffer->scratch_size_needed = 0; - cmd_buffer->compute_scratch_size_needed = 0; - cmd_buffer->esgs_ring_size_needed = 0; - cmd_buffer->gsvs_ring_size_needed = 0; - cmd_buffer->tess_rings_needed = false; - cmd_buffer->sample_positions_needed = false; - - if (cmd_buffer->upload.upload_bo) - cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, - cmd_buffer->upload.upload_bo, 8); - cmd_buffer->upload.offset = 0; - - cmd_buffer->record_fail = false; - - cmd_buffer->ring_offsets_idx = -1; -} - static bool radv_cmd_buffer_resize_upload_buf(struct radv_cmd_buffer *cmd_buffer, uint64_t min_needed) @@ -322,32 +246,6 @@ radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer, return true; } -void radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer) -{ - struct radv_device *device = cmd_buffer->device; - struct radeon_winsys_cs *cs = cmd_buffer->cs; - uint64_t va; - - if (!device->trace_bo) - return; - - va = device->ws->buffer_get_va(device->trace_bo); - - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 7); - - ++cmd_buffer->state.trace_id; - device->ws->cs_add_buffer(cs, device->trace_bo, 8); - radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0)); - radeon_emit(cs, S_370_DST_SEL(V_370_MEM_ASYNC) | - S_370_WR_CONFIRM(1) | - S_370_ENGINE_SEL(V_370_ME)); - radeon_emit(cs, va); - radeon_emit(cs, va >> 32); - radeon_emit(cs, cmd_buffer->state.trace_id); - radeon_emit(cs, PKT3(PKT3_NOP, 0, 0)); - radeon_emit(cs, AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id)); -} - static void radv_emit_graphics_blend_state(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline) @@ -378,58 +276,6 @@ static unsigned radv_pack_float_12p4(float x) x >= 4096 ? 0xffff : x * 16; } -static uint32_t -shader_stage_to_user_data_0(gl_shader_stage stage, bool has_gs, bool has_tess) -{ - switch (stage) { - case MESA_SHADER_FRAGMENT: - return R_00B030_SPI_SHADER_USER_DATA_PS_0; - case MESA_SHADER_VERTEX: - if (has_tess) - return R_00B530_SPI_SHADER_USER_DATA_LS_0; - else - return has_gs ? R_00B330_SPI_SHADER_USER_DATA_ES_0 : R_00B130_SPI_SHADER_USER_DATA_VS_0; - case MESA_SHADER_GEOMETRY: - return R_00B230_SPI_SHADER_USER_DATA_GS_0; - case MESA_SHADER_COMPUTE: - return R_00B900_COMPUTE_USER_DATA_0; - case MESA_SHADER_TESS_CTRL: - return R_00B430_SPI_SHADER_USER_DATA_HS_0; - case MESA_SHADER_TESS_EVAL: - if (has_gs) - return R_00B330_SPI_SHADER_USER_DATA_ES_0; - else - return R_00B130_SPI_SHADER_USER_DATA_VS_0; - default: - unreachable("unknown shader"); - } -} - -static struct ac_userdata_info * -radv_lookup_user_sgpr(struct radv_pipeline *pipeline, - gl_shader_stage stage, - int idx) -{ - return &pipeline->shaders[stage]->info.user_sgprs_locs.shader_data[idx]; -} - -static void -radv_emit_userdata_address(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline, - gl_shader_stage stage, - int idx, uint64_t va) -{ - struct ac_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, idx); - uint32_t base_reg = shader_stage_to_user_data_0(stage, radv_pipeline_has_gs(pipeline), radv_pipeline_has_tess(pipeline)); - if (loc->sgpr_idx == -1) - return; - assert(loc->num_sgprs == 2); - assert(!loc->indirect); - radeon_set_sh_reg_seq(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, 2); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); -} - static void radv_update_multisample_state(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline) @@ -442,9 +288,6 @@ radv_update_multisample_state(struct radv_cmd_buffer *cmd_buffer, radeon_emit(cmd_buffer->cs, ms->pa_sc_aa_mask[0]); radeon_emit(cmd_buffer->cs, ms->pa_sc_aa_mask[1]); - radeon_set_context_reg(cmd_buffer->cs, CM_R_028804_DB_EQAA, ms->db_eqaa); - radeon_set_context_reg(cmd_buffer->cs, EG_R_028A4C_PA_SC_MODE_CNTL_1, ms->pa_sc_mode_cntl_1); - if (old_pipeline && num_samples == old_pipeline->graphics.ms.num_samples) return; @@ -452,37 +295,41 @@ radv_update_multisample_state(struct radv_cmd_buffer *cmd_buffer, radeon_emit(cmd_buffer->cs, ms->pa_sc_line_cntl); radeon_emit(cmd_buffer->cs, ms->pa_sc_aa_config); - radv_cayman_emit_msaa_sample_locs(cmd_buffer->cs, num_samples); + radeon_set_context_reg(cmd_buffer->cs, CM_R_028804_DB_EQAA, ms->db_eqaa); + radeon_set_context_reg(cmd_buffer->cs, EG_R_028A4C_PA_SC_MODE_CNTL_1, ms->pa_sc_mode_cntl_1); - if (pipeline->shaders[MESA_SHADER_FRAGMENT]->info.fs.uses_sample_positions) { - uint32_t offset; - struct ac_userdata_info *loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_FRAGMENT, AC_UD_PS_SAMPLE_POS_OFFSET); - uint32_t base_reg = shader_stage_to_user_data_0(MESA_SHADER_FRAGMENT, radv_pipeline_has_gs(pipeline), radv_pipeline_has_tess(pipeline)); - if (loc->sgpr_idx == -1) - return; - assert(loc->num_sgprs == 1); - assert(!loc->indirect); - switch (num_samples) { - default: - offset = 0; - break; - case 2: - offset = 1; - break; - case 4: - offset = 3; - break; - case 8: - offset = 7; - break; - case 16: - offset = 15; - break; - } + radv_cayman_emit_msaa_sample_locs(cmd_buffer->cs, num_samples); - radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, offset); - cmd_buffer->sample_positions_needed = true; + uint32_t samples_offset; + void *samples_ptr; + void *src; + radv_cmd_buffer_upload_alloc(cmd_buffer, num_samples * 4 * 2, 256, &samples_offset, + &samples_ptr); + switch (num_samples) { + case 1: + src = cmd_buffer->device->sample_locations_1x; + break; + case 2: + src = cmd_buffer->device->sample_locations_2x; + break; + case 4: + src = cmd_buffer->device->sample_locations_4x; + break; + case 8: + src = cmd_buffer->device->sample_locations_8x; + break; + case 16: + src = cmd_buffer->device->sample_locations_16x; + break; } + memcpy(samples_ptr, src, num_samples * 4 * 2); + + uint64_t va = cmd_buffer->device->ws->buffer_get_va(cmd_buffer->upload.upload_bo); + va += samples_offset; + + radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B030_SPI_SHADER_USER_DATA_PS_0 + AC_USERDATA_PS_SAMPLE_POS * 4, 2); + radeon_emit(cmd_buffer->cs, va); + radeon_emit(cmd_buffer->cs, va >> 32); } static void @@ -498,8 +345,7 @@ radv_emit_graphics_raster_state(struct radv_cmd_buffer *cmd_buffer, raster->spi_interp_control); radeon_set_context_reg_seq(cmd_buffer->cs, R_028A00_PA_SU_POINT_SIZE, 2); - unsigned tmp = (unsigned)(1.0 * 8.0); - radeon_emit(cmd_buffer->cs, S_028A00_HEIGHT(tmp) | S_028A00_WIDTH(tmp)); + radeon_emit(cmd_buffer->cs, 0); radeon_emit(cmd_buffer->cs, S_028A04_MIN_SIZE(radv_pack_float_12p4(0)) | S_028A04_MAX_SIZE(radv_pack_float_12p4(8192/2))); /* R_028A04_PA_SU_POINT_MINMAX */ @@ -511,39 +357,47 @@ radv_emit_graphics_raster_state(struct radv_cmd_buffer *cmd_buffer, } static void -radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline, - struct radv_shader_variant *shader, - struct ac_vs_output_info *outinfo) +radv_emit_vertex_shader(struct radv_cmd_buffer *cmd_buffer, + struct radv_pipeline *pipeline) { struct radeon_winsys *ws = cmd_buffer->device->ws; - uint64_t va = ws->buffer_get_va(shader->bo); + struct radv_shader_variant *vs; + uint64_t va; unsigned export_count; + unsigned clip_dist_mask, cull_dist_mask, total_mask; + + assert (pipeline->shaders[MESA_SHADER_VERTEX]); + + vs = pipeline->shaders[MESA_SHADER_VERTEX]; + va = ws->buffer_get_va(vs->bo); + ws->cs_add_buffer(cmd_buffer->cs, vs->bo, 8); - ws->cs_add_buffer(cmd_buffer->cs, shader->bo, 8); + clip_dist_mask = vs->info.vs.clip_dist_mask; + cull_dist_mask = vs->info.vs.cull_dist_mask; + total_mask = clip_dist_mask | cull_dist_mask; + radeon_set_context_reg(cmd_buffer->cs, R_028A40_VGT_GS_MODE, 0); + radeon_set_context_reg(cmd_buffer->cs, R_028A84_VGT_PRIMITIVEID_EN, 0); - export_count = MAX2(1, outinfo->param_exports); + export_count = MAX2(1, vs->info.vs.param_exports); radeon_set_context_reg(cmd_buffer->cs, R_0286C4_SPI_VS_OUT_CONFIG, S_0286C4_VS_EXPORT_COUNT(export_count - 1)); - radeon_set_context_reg(cmd_buffer->cs, R_02870C_SPI_SHADER_POS_FORMAT, S_02870C_POS0_EXPORT_FORMAT(V_02870C_SPI_SHADER_4COMP) | - S_02870C_POS1_EXPORT_FORMAT(outinfo->pos_exports > 1 ? + S_02870C_POS1_EXPORT_FORMAT(vs->info.vs.pos_exports > 1 ? V_02870C_SPI_SHADER_4COMP : V_02870C_SPI_SHADER_NONE) | - S_02870C_POS2_EXPORT_FORMAT(outinfo->pos_exports > 2 ? + S_02870C_POS2_EXPORT_FORMAT(vs->info.vs.pos_exports > 2 ? V_02870C_SPI_SHADER_4COMP : V_02870C_SPI_SHADER_NONE) | - S_02870C_POS3_EXPORT_FORMAT(outinfo->pos_exports > 3 ? + S_02870C_POS3_EXPORT_FORMAT(vs->info.vs.pos_exports > 3 ? V_02870C_SPI_SHADER_4COMP : V_02870C_SPI_SHADER_NONE)); - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B120_SPI_SHADER_PGM_LO_VS, 4); radeon_emit(cmd_buffer->cs, va >> 8); radeon_emit(cmd_buffer->cs, va >> 40); - radeon_emit(cmd_buffer->cs, shader->rsrc1); - radeon_emit(cmd_buffer->cs, shader->rsrc2); + radeon_emit(cmd_buffer->cs, vs->rsrc1); + radeon_emit(cmd_buffer->cs, vs->rsrc2); radeon_set_context_reg(cmd_buffer->cs, R_028818_PA_CL_VTE_CNTL, S_028818_VTX_W0_FMT(1) | @@ -551,236 +405,34 @@ radv_emit_hw_vs(struct radv_cmd_buffer *cmd_buffer, S_028818_VPORT_Y_SCALE_ENA(1) | S_028818_VPORT_Y_OFFSET_ENA(1) | S_028818_VPORT_Z_SCALE_ENA(1) | S_028818_VPORT_Z_OFFSET_ENA(1)); - radeon_set_context_reg(cmd_buffer->cs, R_02881C_PA_CL_VS_OUT_CNTL, - pipeline->graphics.pa_cl_vs_out_cntl); - - radeon_set_context_reg(cmd_buffer->cs, R_028AB4_VGT_REUSE_OFF, - S_028AB4_REUSE_OFF(outinfo->writes_viewport_index)); -} - -static void -radv_emit_hw_es(struct radv_cmd_buffer *cmd_buffer, - struct radv_shader_variant *shader, - struct ac_es_output_info *outinfo) -{ - struct radeon_winsys *ws = cmd_buffer->device->ws; - uint64_t va = ws->buffer_get_va(shader->bo); - - ws->cs_add_buffer(cmd_buffer->cs, shader->bo, 8); - - radeon_set_context_reg(cmd_buffer->cs, R_028AAC_VGT_ESGS_RING_ITEMSIZE, - outinfo->esgs_itemsize / 4); - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B320_SPI_SHADER_PGM_LO_ES, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - radeon_emit(cmd_buffer->cs, shader->rsrc1); - radeon_emit(cmd_buffer->cs, shader->rsrc2); -} - -static void -radv_emit_hw_ls(struct radv_cmd_buffer *cmd_buffer, - struct radv_shader_variant *shader) -{ - struct radeon_winsys *ws = cmd_buffer->device->ws; - uint64_t va = ws->buffer_get_va(shader->bo); - uint32_t rsrc2 = shader->rsrc2; - - ws->cs_add_buffer(cmd_buffer->cs, shader->bo, 8); - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B520_SPI_SHADER_PGM_LO_LS, 2); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - - rsrc2 |= S_00B52C_LDS_SIZE(cmd_buffer->state.pipeline->graphics.tess.lds_size); - if (cmd_buffer->device->physical_device->rad_info.chip_class == CIK && - cmd_buffer->device->physical_device->rad_info.family != CHIP_HAWAII) - radeon_set_sh_reg(cmd_buffer->cs, R_00B52C_SPI_SHADER_PGM_RSRC2_LS, rsrc2); - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B528_SPI_SHADER_PGM_RSRC1_LS, 2); - radeon_emit(cmd_buffer->cs, shader->rsrc1); - radeon_emit(cmd_buffer->cs, rsrc2); -} - -static void -radv_emit_hw_hs(struct radv_cmd_buffer *cmd_buffer, - struct radv_shader_variant *shader) -{ - struct radeon_winsys *ws = cmd_buffer->device->ws; - uint64_t va = ws->buffer_get_va(shader->bo); - - ws->cs_add_buffer(cmd_buffer->cs, shader->bo, 8); - - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B420_SPI_SHADER_PGM_LO_HS, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - radeon_emit(cmd_buffer->cs, shader->rsrc1); - radeon_emit(cmd_buffer->cs, shader->rsrc2); -} - -static void -radv_emit_vertex_shader(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - struct radv_shader_variant *vs; - - assert (pipeline->shaders[MESA_SHADER_VERTEX]); - - vs = pipeline->shaders[MESA_SHADER_VERTEX]; - - if (vs->info.vs.as_ls) - radv_emit_hw_ls(cmd_buffer, vs); - else if (vs->info.vs.as_es) - radv_emit_hw_es(cmd_buffer, vs, &vs->info.vs.es_info); - else - radv_emit_hw_vs(cmd_buffer, pipeline, vs, &vs->info.vs.outinfo); - - radeon_set_context_reg(cmd_buffer->cs, R_028A84_VGT_PRIMITIVEID_EN, 0); -} - - -static void -radv_emit_tess_shaders(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - if (!radv_pipeline_has_tess(pipeline)) - return; - - struct radv_shader_variant *tes, *tcs; - - tcs = pipeline->shaders[MESA_SHADER_TESS_CTRL]; - tes = pipeline->shaders[MESA_SHADER_TESS_EVAL]; - - if (tes->info.tes.as_es) - radv_emit_hw_es(cmd_buffer, tes, &tes->info.tes.es_info); - else - radv_emit_hw_vs(cmd_buffer, pipeline, tes, &tes->info.tes.outinfo); + S_02881C_USE_VTX_POINT_SIZE(vs->info.vs.writes_pointsize) | + S_02881C_VS_OUT_MISC_VEC_ENA(vs->info.vs.writes_pointsize) | + S_02881C_VS_OUT_CCDIST0_VEC_ENA((total_mask & 0x0f) != 0) | + S_02881C_VS_OUT_CCDIST1_VEC_ENA((total_mask & 0xf0) != 0) | + pipeline->graphics.raster.pa_cl_vs_out_cntl | + cull_dist_mask << 8 | + clip_dist_mask); - radv_emit_hw_hs(cmd_buffer, tcs); - - radeon_set_context_reg(cmd_buffer->cs, R_028B6C_VGT_TF_PARAM, - pipeline->graphics.tess.tf_param); - - if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) - radeon_set_context_reg_idx(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, 2, - pipeline->graphics.tess.ls_hs_config); - else - radeon_set_context_reg(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, - pipeline->graphics.tess.ls_hs_config); - - struct ac_userdata_info *loc; - - loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_TESS_CTRL, AC_UD_TCS_OFFCHIP_LAYOUT); - if (loc->sgpr_idx != -1) { - uint32_t base_reg = shader_stage_to_user_data_0(MESA_SHADER_TESS_CTRL, radv_pipeline_has_gs(pipeline), radv_pipeline_has_tess(pipeline)); - assert(loc->num_sgprs == 4); - assert(!loc->indirect); - radeon_set_sh_reg_seq(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, 4); - radeon_emit(cmd_buffer->cs, pipeline->graphics.tess.offchip_layout); - radeon_emit(cmd_buffer->cs, pipeline->graphics.tess.tcs_out_offsets); - radeon_emit(cmd_buffer->cs, pipeline->graphics.tess.tcs_out_layout | - pipeline->graphics.tess.num_tcs_input_cp << 26); - radeon_emit(cmd_buffer->cs, pipeline->graphics.tess.tcs_in_layout); - } - - loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_TESS_EVAL, AC_UD_TES_OFFCHIP_LAYOUT); - if (loc->sgpr_idx != -1) { - uint32_t base_reg = shader_stage_to_user_data_0(MESA_SHADER_TESS_EVAL, radv_pipeline_has_gs(pipeline), radv_pipeline_has_tess(pipeline)); - assert(loc->num_sgprs == 1); - assert(!loc->indirect); - - radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, - pipeline->graphics.tess.offchip_layout); - } - - loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_VERTEX, AC_UD_VS_LS_TCS_IN_LAYOUT); - if (loc->sgpr_idx != -1) { - uint32_t base_reg = shader_stage_to_user_data_0(MESA_SHADER_VERTEX, radv_pipeline_has_gs(pipeline), radv_pipeline_has_tess(pipeline)); - assert(loc->num_sgprs == 1); - assert(!loc->indirect); - - radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, - pipeline->graphics.tess.tcs_in_layout); - } } -static void -radv_emit_geometry_shader(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - struct radeon_winsys *ws = cmd_buffer->device->ws; - struct radv_shader_variant *gs; - uint64_t va; - - radeon_set_context_reg(cmd_buffer->cs, R_028A40_VGT_GS_MODE, pipeline->graphics.vgt_gs_mode); - - gs = pipeline->shaders[MESA_SHADER_GEOMETRY]; - if (!gs) - return; - - uint32_t gsvs_itemsize = gs->info.gs.max_gsvs_emit_size >> 2; - - radeon_set_context_reg_seq(cmd_buffer->cs, R_028A60_VGT_GSVS_RING_OFFSET_1, 3); - radeon_emit(cmd_buffer->cs, gsvs_itemsize); - radeon_emit(cmd_buffer->cs, gsvs_itemsize); - radeon_emit(cmd_buffer->cs, gsvs_itemsize); - - radeon_set_context_reg(cmd_buffer->cs, R_028AB0_VGT_GSVS_RING_ITEMSIZE, gsvs_itemsize); - - radeon_set_context_reg(cmd_buffer->cs, R_028B38_VGT_GS_MAX_VERT_OUT, gs->info.gs.vertices_out); - - uint32_t gs_vert_itemsize = gs->info.gs.gsvs_vertex_size; - radeon_set_context_reg_seq(cmd_buffer->cs, R_028B5C_VGT_GS_VERT_ITEMSIZE, 4); - radeon_emit(cmd_buffer->cs, gs_vert_itemsize >> 2); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, 0); - - uint32_t gs_num_invocations = gs->info.gs.invocations; - radeon_set_context_reg(cmd_buffer->cs, R_028B90_VGT_GS_INSTANCE_CNT, - S_028B90_CNT(MIN2(gs_num_invocations, 127)) | - S_028B90_ENABLE(gs_num_invocations > 0)); - - va = ws->buffer_get_va(gs->bo); - ws->cs_add_buffer(cmd_buffer->cs, gs->bo, 8); - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B220_SPI_SHADER_PGM_LO_GS, 4); - radeon_emit(cmd_buffer->cs, va >> 8); - radeon_emit(cmd_buffer->cs, va >> 40); - radeon_emit(cmd_buffer->cs, gs->rsrc1); - radeon_emit(cmd_buffer->cs, gs->rsrc2); - - radv_emit_hw_vs(cmd_buffer, pipeline, pipeline->gs_copy_shader, &pipeline->gs_copy_shader->info.vs.outinfo); - - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.pipeline, MESA_SHADER_GEOMETRY, - AC_UD_GS_VS_RING_STRIDE_ENTRIES); - if (loc->sgpr_idx != -1) { - uint32_t stride = gs->info.gs.max_gsvs_emit_size; - uint32_t num_entries = 64; - bool is_vi = cmd_buffer->device->physical_device->rad_info.chip_class >= VI; - - if (is_vi) - num_entries *= stride; - stride = S_008F04_STRIDE(stride); - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B230_SPI_SHADER_USER_DATA_GS_0 + loc->sgpr_idx * 4, 2); - radeon_emit(cmd_buffer->cs, stride); - radeon_emit(cmd_buffer->cs, num_entries); - } -} static void radv_emit_fragment_shader(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline) { struct radeon_winsys *ws = cmd_buffer->device->ws; - struct radv_shader_variant *ps; + struct radv_shader_variant *ps, *vs; uint64_t va; unsigned spi_baryc_cntl = S_0286E0_FRONT_FACE_ALL_BITS(1); struct radv_blend_state *blend = &pipeline->graphics.blend; + unsigned ps_offset = 0; + unsigned z_order; assert (pipeline->shaders[MESA_SHADER_FRAGMENT]); ps = pipeline->shaders[MESA_SHADER_FRAGMENT]; - + vs = pipeline->shaders[MESA_SHADER_VERTEX]; va = ws->buffer_get_va(ps->bo); ws->cs_add_buffer(cmd_buffer->cs, ps->bo, 8); @@ -790,8 +442,20 @@ radv_emit_fragment_shader(struct radv_cmd_buffer *cmd_buffer, radeon_emit(cmd_buffer->cs, ps->rsrc1); radeon_emit(cmd_buffer->cs, ps->rsrc2); + if (ps->info.fs.early_fragment_test || !ps->info.fs.writes_memory) + z_order = V_02880C_EARLY_Z_THEN_LATE_Z; + else + z_order = V_02880C_LATE_Z; + + radeon_set_context_reg(cmd_buffer->cs, R_02880C_DB_SHADER_CONTROL, - pipeline->graphics.db_shader_control); + S_02880C_Z_EXPORT_ENABLE(ps->info.fs.writes_z) | + S_02880C_STENCIL_TEST_VAL_EXPORT_ENABLE(ps->info.fs.writes_stencil) | + S_02880C_KILL_ENABLE(!!ps->info.fs.can_discard) | + S_02880C_Z_ORDER(z_order) | + S_02880C_DEPTH_BEFORE_SHADER(ps->info.fs.early_fragment_test) | + S_02880C_EXEC_ON_HIER_FAIL(ps->info.fs.writes_memory) | + S_02880C_EXEC_ON_NOOP(ps->info.fs.writes_memory)); radeon_set_context_reg(cmd_buffer->cs, R_0286CC_SPI_PS_INPUT_ENA, ps->config.spi_ps_input_ena); @@ -799,43 +463,51 @@ radv_emit_fragment_shader(struct radv_cmd_buffer *cmd_buffer, radeon_set_context_reg(cmd_buffer->cs, R_0286D0_SPI_PS_INPUT_ADDR, ps->config.spi_ps_input_addr); - if (ps->info.fs.force_persample) - spi_baryc_cntl |= S_0286E0_POS_FLOAT_LOCATION(2); - + spi_baryc_cntl |= S_0286E0_POS_FLOAT_LOCATION(2); radeon_set_context_reg(cmd_buffer->cs, R_0286D8_SPI_PS_IN_CONTROL, S_0286D8_NUM_INTERP(ps->info.fs.num_interp)); radeon_set_context_reg(cmd_buffer->cs, R_0286E0_SPI_BARYC_CNTL, spi_baryc_cntl); radeon_set_context_reg(cmd_buffer->cs, R_028710_SPI_SHADER_Z_FORMAT, - pipeline->graphics.shader_z_format); + ps->info.fs.writes_stencil ? V_028710_SPI_SHADER_32_GR : + ps->info.fs.writes_z ? V_028710_SPI_SHADER_32_R : + V_028710_SPI_SHADER_ZERO); radeon_set_context_reg(cmd_buffer->cs, R_028714_SPI_SHADER_COL_FORMAT, blend->spi_shader_col_format); radeon_set_context_reg(cmd_buffer->cs, R_028238_CB_TARGET_MASK, blend->cb_target_mask); radeon_set_context_reg(cmd_buffer->cs, R_02823C_CB_SHADER_MASK, blend->cb_shader_mask); - if (pipeline->graphics.ps_input_cntl_num) { - radeon_set_context_reg_seq(cmd_buffer->cs, R_028644_SPI_PS_INPUT_CNTL_0, pipeline->graphics.ps_input_cntl_num); - for (unsigned i = 0; i < pipeline->graphics.ps_input_cntl_num; i++) { - radeon_emit(cmd_buffer->cs, pipeline->graphics.ps_input_cntl[i]); - } + if (ps->info.fs.has_pcoord) { + unsigned val; + val = S_028644_PT_SPRITE_TEX(1) | S_028644_OFFSET(0x20); + radeon_set_context_reg(cmd_buffer->cs, R_028644_SPI_PS_INPUT_CNTL_0 + 4 * ps_offset, val); + ps_offset = 1; } -} -static void polaris_set_vgt_vertex_reuse(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline) -{ - uint32_t vtx_reuse_depth = 30; - if (cmd_buffer->device->physical_device->rad_info.family < CHIP_POLARIS10) - return; + for (unsigned i = 0; i < 32 && (1u << i) <= ps->info.fs.input_mask; ++i) { + unsigned vs_offset, flat_shade; + unsigned val; + + if (!(ps->info.fs.input_mask & (1u << i))) + continue; + + + if (!(vs->info.vs.export_mask & (1u << i))) { + radeon_set_context_reg(cmd_buffer->cs, R_028644_SPI_PS_INPUT_CNTL_0 + 4 * ps_offset, + S_028644_OFFSET(0x20)); + ++ps_offset; + continue; + } + + vs_offset = util_bitcount(vs->info.vs.export_mask & ((1u << i) - 1)); + flat_shade = !!(ps->info.fs.flat_shaded_mask & (1u << ps_offset)); - if (pipeline->shaders[MESA_SHADER_TESS_EVAL]) { - if (pipeline->shaders[MESA_SHADER_TESS_EVAL]->info.tes.spacing == TESS_SPACING_FRACTIONAL_ODD) - vtx_reuse_depth = 14; + val = S_028644_OFFSET(vs_offset) | S_028644_FLAT_SHADE(flat_shade); + radeon_set_context_reg(cmd_buffer->cs, R_028644_SPI_PS_INPUT_CNTL_0 + 4 * ps_offset, val); + ++ps_offset; } - radeon_set_context_reg(cmd_buffer->cs, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL, - vtx_reuse_depth); } static void @@ -850,23 +522,11 @@ radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer, radv_emit_graphics_raster_state(cmd_buffer, pipeline); radv_update_multisample_state(cmd_buffer, pipeline); radv_emit_vertex_shader(cmd_buffer, pipeline); - radv_emit_tess_shaders(cmd_buffer, pipeline); - radv_emit_geometry_shader(cmd_buffer, pipeline); radv_emit_fragment_shader(cmd_buffer, pipeline); - polaris_set_vgt_vertex_reuse(cmd_buffer, pipeline); - - cmd_buffer->scratch_size_needed = - MAX2(cmd_buffer->scratch_size_needed, - pipeline->max_waves * pipeline->scratch_bytes_per_wave); - radeon_set_context_reg(cmd_buffer->cs, R_0286E8_SPI_TMPRING_SIZE, - S_0286E8_WAVES(pipeline->max_waves) | - S_0286E8_WAVESIZE(pipeline->scratch_bytes_per_wave >> 10)); + radeon_set_context_reg(cmd_buffer->cs, R_028A94_VGT_MULTI_PRIM_IB_RESET_EN, + pipeline->graphics.prim_restart_enable); - if (!cmd_buffer->state.emitted_pipeline || - cmd_buffer->state.emitted_pipeline->graphics.can_use_guardband != - pipeline->graphics.can_use_guardband) - cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_SCISSOR; cmd_buffer->state.emitted_pipeline = pipeline; } @@ -882,9 +542,7 @@ radv_emit_scissor(struct radv_cmd_buffer *cmd_buffer) { uint32_t count = cmd_buffer->state.dynamic.scissor.count; si_write_scissors(cmd_buffer->cs, 0, count, - cmd_buffer->state.dynamic.scissor.scissors, - cmd_buffer->state.dynamic.viewport.viewports, - cmd_buffer->state.emitted_pipeline->graphics.can_use_guardband); + cmd_buffer->state.dynamic.scissor.scissors); radeon_set_context_reg(cmd_buffer->cs, R_028A48_PA_SC_MODE_CNTL_0, cmd_buffer->state.pipeline->graphics.ms.pa_sc_mode_cntl_0 | S_028A48_VPORT_SCISSOR_ENABLE(count ? 1 : 0)); } @@ -894,7 +552,7 @@ radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer, int index, struct radv_color_buffer_info *cb) { - bool is_vi = cmd_buffer->device->physical_device->rad_info.chip_class >= VI; + bool is_vi = cmd_buffer->device->instance->physicalDevice.rad_info.chip_class >= VI; radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); radeon_emit(cmd_buffer->cs, cb->cb_color_base); radeon_emit(cmd_buffer->cs, cb->cb_color_pitch); @@ -986,7 +644,7 @@ radv_set_depth_clear_regs(struct radv_cmd_buffer *cmd_buffer, va += image->offset + image->clear_value_offset; unsigned reg_offset = 0, reg_count = 0; - if (!image->surface.htile_size || !aspects) + if (!image->htile.size || !aspects) return; if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) { @@ -1025,7 +683,7 @@ radv_load_depth_clear_regs(struct radv_cmd_buffer *cmd_buffer, uint64_t va = cmd_buffer->device->ws->buffer_get_va(image->bo); va += image->offset + image->clear_value_offset; - if (!image->surface.htile_size) + if (!image->htile.size) return; cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, image->bo, 8); @@ -1160,13 +818,13 @@ void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer) uint32_t db_count_control; if(!cmd_buffer->state.active_occlusion_queries) { - if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) { + if (cmd_buffer->device->instance->physicalDevice.rad_info.chip_class >= CIK) { db_count_control = 0; } else { db_count_control = S_028004_ZPASS_INCREMENT_DISABLE(1); } } else { - if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) { + if (cmd_buffer->device->instance->physicalDevice.rad_info.chip_class >= CIK) { db_count_control = S_028004_PERFECT_ZPASS_COUNTS(1) | S_028004_SAMPLE_RATE(0) | /* TODO: set this to the number of samples of the current framebuffer */ S_028004_ZPASS_ENABLE(1) | @@ -1186,15 +844,6 @@ radv_cmd_buffer_flush_dynamic_state(struct radv_cmd_buffer *cmd_buffer) { struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; - if (G_028810_DX_RASTERIZATION_KILL(cmd_buffer->state.pipeline->graphics.raster.pa_cl_clip_cntl)) - return; - - if (cmd_buffer->state.dirty & (RADV_CMD_DIRTY_DYNAMIC_VIEWPORT)) - radv_emit_viewport(cmd_buffer); - - if (cmd_buffer->state.dirty & (RADV_CMD_DIRTY_DYNAMIC_SCISSOR | RADV_CMD_DIRTY_DYNAMIC_VIEWPORT)) - radv_emit_scissor(cmd_buffer); - if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH) { unsigned width = cmd_buffer->state.dynamic.line_width * 8; radeon_set_context_reg(cmd_buffer->cs, R_028A08_PA_SU_LINE_CNTL, @@ -1246,118 +895,9 @@ radv_cmd_buffer_flush_dynamic_state(struct radv_cmd_buffer *cmd_buffer) } static void -emit_stage_descriptor_set_userdata(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline, - int idx, - uint64_t va, - gl_shader_stage stage) -{ - struct ac_userdata_info *desc_set_loc = &pipeline->shaders[stage]->info.user_sgprs_locs.descriptor_sets[idx]; - uint32_t base_reg = shader_stage_to_user_data_0(stage, radv_pipeline_has_gs(pipeline), radv_pipeline_has_tess(pipeline)); - - if (desc_set_loc->sgpr_idx == -1) - return; - - assert(!desc_set_loc->indirect); - assert(desc_set_loc->num_sgprs == 2); - radeon_set_sh_reg_seq(cmd_buffer->cs, - base_reg + desc_set_loc->sgpr_idx * 4, 2); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); -} - -static void -radv_emit_descriptor_set_userdata(struct radv_cmd_buffer *cmd_buffer, - VkShaderStageFlags stages, - struct radv_descriptor_set *set, - unsigned idx) -{ - if (cmd_buffer->state.pipeline) { - if (stages & VK_SHADER_STAGE_FRAGMENT_BIT) - emit_stage_descriptor_set_userdata(cmd_buffer, cmd_buffer->state.pipeline, - idx, set->va, - MESA_SHADER_FRAGMENT); - - if (stages & VK_SHADER_STAGE_VERTEX_BIT) - emit_stage_descriptor_set_userdata(cmd_buffer, cmd_buffer->state.pipeline, - idx, set->va, - MESA_SHADER_VERTEX); - - if ((stages & VK_SHADER_STAGE_GEOMETRY_BIT) && radv_pipeline_has_gs(cmd_buffer->state.pipeline)) - emit_stage_descriptor_set_userdata(cmd_buffer, cmd_buffer->state.pipeline, - idx, set->va, - MESA_SHADER_GEOMETRY); - - if ((stages & VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT) && radv_pipeline_has_tess(cmd_buffer->state.pipeline)) - emit_stage_descriptor_set_userdata(cmd_buffer, cmd_buffer->state.pipeline, - idx, set->va, - MESA_SHADER_TESS_CTRL); - - if ((stages & VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT) && radv_pipeline_has_tess(cmd_buffer->state.pipeline)) - emit_stage_descriptor_set_userdata(cmd_buffer, cmd_buffer->state.pipeline, - idx, set->va, - MESA_SHADER_TESS_EVAL); - } - - if (cmd_buffer->state.compute_pipeline && (stages & VK_SHADER_STAGE_COMPUTE_BIT)) - emit_stage_descriptor_set_userdata(cmd_buffer, cmd_buffer->state.compute_pipeline, - idx, set->va, - MESA_SHADER_COMPUTE); -} - -static void -radv_flush_push_descriptors(struct radv_cmd_buffer *cmd_buffer) -{ - struct radv_descriptor_set *set = &cmd_buffer->push_descriptors.set; - uint32_t *ptr = NULL; - unsigned bo_offset; - - if (!radv_cmd_buffer_upload_alloc(cmd_buffer, set->size, 32, - &bo_offset, - (void**) &ptr)) - return; - - set->va = cmd_buffer->device->ws->buffer_get_va(cmd_buffer->upload.upload_bo); - set->va += bo_offset; - - memcpy(ptr, set->mapped_ptr, set->size); -} - -static void -radv_flush_descriptors(struct radv_cmd_buffer *cmd_buffer, - VkShaderStageFlags stages) -{ - unsigned i; - if (!cmd_buffer->state.descriptors_dirty) - return; - - if (cmd_buffer->state.push_descriptors_dirty) - radv_flush_push_descriptors(cmd_buffer); - - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, - cmd_buffer->cs, - MAX_SETS * MESA_SHADER_STAGES * 4); - - for (i = 0; i < MAX_SETS; i++) { - if (!(cmd_buffer->state.descriptors_dirty & (1 << i))) - continue; - struct radv_descriptor_set *set = cmd_buffer->state.descriptors[i]; - if (!set) - continue; - - radv_emit_descriptor_set_userdata(cmd_buffer, stages, set, i); - } - cmd_buffer->state.descriptors_dirty = 0; - cmd_buffer->state.push_descriptors_dirty = false; - assert(cmd_buffer->cs->cdw <= cdw_max); -} - -static void radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, - struct radv_pipeline *pipeline, - VkShaderStageFlags stages) -{ - struct radv_pipeline_layout *layout = pipeline->layout; + struct radv_pipeline_layout *layout, + VkShaderStageFlags stages) { unsigned offset; void *ptr; uint64_t va; @@ -1366,10 +906,9 @@ radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, if (!stages || !layout || (!layout->push_constant_size && !layout->dynamic_offset_count)) return; - if (!radv_cmd_buffer_upload_alloc(cmd_buffer, layout->push_constant_size + - 16 * layout->dynamic_offset_count, - 256, &offset, &ptr)) - return; + radv_cmd_buffer_upload_alloc(cmd_buffer, layout->push_constant_size + + 16 * layout->dynamic_offset_count, + 256, &offset, &ptr); memcpy(ptr, cmd_buffer->push_constants, layout->push_constant_size); memcpy((char*)ptr + layout->push_constant_size, cmd_buffer->dynamic_buffers, @@ -1378,70 +917,40 @@ radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, va = cmd_buffer->device->ws->buffer_get_va(cmd_buffer->upload.upload_bo); va += offset; - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, - cmd_buffer->cs, MESA_SHADER_STAGES * 4); - if (stages & VK_SHADER_STAGE_VERTEX_BIT) - radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_VERTEX, - AC_UD_PUSH_CONSTANTS, va); - - if (stages & VK_SHADER_STAGE_FRAGMENT_BIT) - radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_FRAGMENT, - AC_UD_PUSH_CONSTANTS, va); - - if ((stages & VK_SHADER_STAGE_GEOMETRY_BIT) && radv_pipeline_has_gs(pipeline)) - radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_GEOMETRY, - AC_UD_PUSH_CONSTANTS, va); - - if ((stages & VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT) && radv_pipeline_has_tess(pipeline)) - radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_TESS_CTRL, - AC_UD_PUSH_CONSTANTS, va); - - if ((stages & VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT) && radv_pipeline_has_tess(pipeline)) - radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_TESS_EVAL, - AC_UD_PUSH_CONSTANTS, va); - - if (stages & VK_SHADER_STAGE_COMPUTE_BIT) - radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_COMPUTE, - AC_UD_PUSH_CONSTANTS, va); - - cmd_buffer->push_constant_stages &= ~stages; - assert(cmd_buffer->cs->cdw <= cdw_max); -} - -static void radv_emit_primitive_reset_state(struct radv_cmd_buffer *cmd_buffer, - bool indexed_draw) -{ - int32_t primitive_reset_en = indexed_draw && cmd_buffer->state.pipeline->graphics.prim_restart_enable; - - if (primitive_reset_en != cmd_buffer->state.last_primitive_reset_en) { - cmd_buffer->state.last_primitive_reset_en = primitive_reset_en; - radeon_set_context_reg(cmd_buffer->cs, R_028A94_VGT_MULTI_PRIM_IB_RESET_EN, - primitive_reset_en); + if (stages & VK_SHADER_STAGE_VERTEX_BIT) { + radeon_set_sh_reg_seq(cmd_buffer->cs, + R_00B130_SPI_SHADER_USER_DATA_VS_0 + AC_USERDATA_PUSH_CONST_DYN * 4, 2); + radeon_emit(cmd_buffer->cs, va); + radeon_emit(cmd_buffer->cs, va >> 32); } - if (primitive_reset_en) { - uint32_t primitive_reset_index = cmd_buffer->state.index_type ? 0xffffffffu : 0xffffu; + if (stages & VK_SHADER_STAGE_FRAGMENT_BIT) { + radeon_set_sh_reg_seq(cmd_buffer->cs, + R_00B030_SPI_SHADER_USER_DATA_PS_0 + AC_USERDATA_PUSH_CONST_DYN * 4, 2); + radeon_emit(cmd_buffer->cs, va); + radeon_emit(cmd_buffer->cs, va >> 32); + } - if (primitive_reset_index != cmd_buffer->state.last_primitive_reset_index) { - cmd_buffer->state.last_primitive_reset_index = primitive_reset_index; - radeon_set_context_reg(cmd_buffer->cs, R_02840C_VGT_MULTI_PRIM_IB_RESET_INDX, - primitive_reset_index); - } + if (stages & VK_SHADER_STAGE_COMPUTE_BIT) { + radeon_set_sh_reg_seq(cmd_buffer->cs, + R_00B900_COMPUTE_USER_DATA_0 + AC_USERDATA_PUSH_CONST_DYN * 4, 2); + radeon_emit(cmd_buffer->cs, va); + radeon_emit(cmd_buffer->cs, va >> 32); } + + cmd_buffer->push_constant_stages &= ~stages; } static void -radv_cmd_buffer_flush_state(struct radv_cmd_buffer *cmd_buffer, - bool indexed_draw, bool instanced_draw, - bool indirect_draw, - uint32_t draw_vertex_count) +radv_cmd_buffer_flush_state(struct radv_cmd_buffer *cmd_buffer) { struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; struct radv_device *device = cmd_buffer->device; uint32_t ia_multi_vgt_param; + uint32_t ls_hs_config = 0; - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, - cmd_buffer->cs, 4096); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, + 4096); if ((cmd_buffer->state.vertex_descriptors_dirty || cmd_buffer->state.vb_dirty) && cmd_buffer->state.pipeline->num_vertex_attribs) { @@ -1469,7 +978,7 @@ radv_cmd_buffer_flush_state(struct radv_cmd_buffer *cmd_buffer, va += offset + buffer->offset; desc[0] = va; desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) | S_008F04_STRIDE(stride); - if (cmd_buffer->device->physical_device->rad_info.chip_class <= CIK && stride) + if (cmd_buffer->device->instance->physicalDevice.rad_info.chip_class <= CIK && stride) desc[2] = (buffer->size - offset - cmd_buffer->state.pipeline->va_format_size[i]) / stride + 1; else desc[2] = buffer->size - offset; @@ -1478,9 +987,11 @@ radv_cmd_buffer_flush_state(struct radv_cmd_buffer *cmd_buffer, va = device->ws->buffer_get_va(cmd_buffer->upload.upload_bo); va += vb_offset; + radeon_set_sh_reg_seq(cmd_buffer->cs, + R_00B130_SPI_SHADER_USER_DATA_VS_0 + AC_USERDATA_VS_VERTEX_BUFFERS * 4, 2); + radeon_emit(cmd_buffer->cs, va); + radeon_emit(cmd_buffer->cs, va >> 32); - radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_VERTEX, - AC_UD_VS_VERTEX_BUFFERS, va); } cmd_buffer->state.vertex_descriptors_dirty = false; @@ -1491,32 +1002,31 @@ radv_cmd_buffer_flush_state(struct radv_cmd_buffer *cmd_buffer, if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_RENDER_TARGETS) radv_emit_framebuffer_state(cmd_buffer); - ia_multi_vgt_param = si_get_ia_multi_vgt_param(cmd_buffer, instanced_draw, indirect_draw, draw_vertex_count); - if (cmd_buffer->state.last_ia_multi_vgt_param != ia_multi_vgt_param) { - if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) - radeon_set_context_reg_idx(cmd_buffer->cs, R_028AA8_IA_MULTI_VGT_PARAM, 1, ia_multi_vgt_param); - else - radeon_set_context_reg(cmd_buffer->cs, R_028AA8_IA_MULTI_VGT_PARAM, ia_multi_vgt_param); - cmd_buffer->state.last_ia_multi_vgt_param = ia_multi_vgt_param; - } + if (cmd_buffer->state.dirty & (RADV_CMD_DIRTY_DYNAMIC_VIEWPORT)) + radv_emit_viewport(cmd_buffer); + + if (cmd_buffer->state.dirty & (RADV_CMD_DIRTY_DYNAMIC_SCISSOR)) + radv_emit_scissor(cmd_buffer); if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE) { - radeon_set_context_reg(cmd_buffer->cs, R_028B54_VGT_SHADER_STAGES_EN, pipeline->graphics.vgt_shader_stages_en); + radeon_set_context_reg(cmd_buffer->cs, R_028B54_VGT_SHADER_STAGES_EN, 0); + ia_multi_vgt_param = si_get_ia_multi_vgt_param(cmd_buffer); - if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) { + if (cmd_buffer->device->instance->physicalDevice.rad_info.chip_class >= CIK) { + radeon_set_context_reg_idx(cmd_buffer->cs, R_028AA8_IA_MULTI_VGT_PARAM, 1, ia_multi_vgt_param); + radeon_set_context_reg_idx(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, 2, ls_hs_config); radeon_set_uconfig_reg_idx(cmd_buffer->cs, R_030908_VGT_PRIMITIVE_TYPE, 1, cmd_buffer->state.pipeline->graphics.prim); } else { radeon_set_config_reg(cmd_buffer->cs, R_008958_VGT_PRIMITIVE_TYPE, cmd_buffer->state.pipeline->graphics.prim); + radeon_set_context_reg(cmd_buffer->cs, R_028AA8_IA_MULTI_VGT_PARAM, ia_multi_vgt_param); + radeon_set_context_reg(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, ls_hs_config); } radeon_set_context_reg(cmd_buffer->cs, R_028A6C_VGT_GS_OUT_PRIM_TYPE, cmd_buffer->state.pipeline->graphics.gs_out); } radv_cmd_buffer_flush_dynamic_state(cmd_buffer); - radv_emit_primitive_reset_state(cmd_buffer, indexed_draw); - - radv_flush_descriptors(cmd_buffer, VK_SHADER_STAGE_ALL_GRAPHICS); - radv_flush_constants(cmd_buffer, cmd_buffer->state.pipeline, + radv_flush_constants(cmd_buffer, cmd_buffer->state.pipeline->layout, VK_SHADER_STAGE_ALL_GRAPHICS); assert(cmd_buffer->cs->cdw <= cdw_max); @@ -1554,86 +1064,11 @@ static void radv_stage_flush(struct radv_cmd_buffer *cmd_buffer, } } -static enum radv_cmd_flush_bits -radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, - VkAccessFlags src_flags) -{ - enum radv_cmd_flush_bits flush_bits = 0; - uint32_t b; - for_each_bit(b, src_flags) { - switch ((VkAccessFlagBits)(1 << b)) { - case VK_ACCESS_SHADER_WRITE_BIT: - flush_bits |= RADV_CMD_FLAG_WRITEBACK_GLOBAL_L2; - break; - case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT: - flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB | - RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; - break; - case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT: - flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB | - RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; - break; - case VK_ACCESS_TRANSFER_WRITE_BIT: - flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB | - RADV_CMD_FLAG_FLUSH_AND_INV_CB_META | - RADV_CMD_FLAG_FLUSH_AND_INV_DB | - RADV_CMD_FLAG_FLUSH_AND_INV_DB_META | - RADV_CMD_FLAG_INV_GLOBAL_L2; - break; - default: - break; - } - } - return flush_bits; -} - -static enum radv_cmd_flush_bits -radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, - VkAccessFlags dst_flags, - struct radv_image *image) -{ - enum radv_cmd_flush_bits flush_bits = 0; - uint32_t b; - for_each_bit(b, dst_flags) { - switch ((VkAccessFlagBits)(1 << b)) { - case VK_ACCESS_INDIRECT_COMMAND_READ_BIT: - case VK_ACCESS_INDEX_READ_BIT: - case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT: - break; - case VK_ACCESS_UNIFORM_READ_BIT: - flush_bits |= RADV_CMD_FLAG_INV_VMEM_L1 | RADV_CMD_FLAG_INV_SMEM_L1; - break; - case VK_ACCESS_SHADER_READ_BIT: - case VK_ACCESS_TRANSFER_READ_BIT: - case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT: - flush_bits |= RADV_CMD_FLAG_INV_VMEM_L1 | - RADV_CMD_FLAG_INV_GLOBAL_L2; - break; - case VK_ACCESS_COLOR_ATTACHMENT_READ_BIT: - /* TODO: change to image && when the image gets passed - * through from the subpass. */ - if (!image || (image->usage & VK_IMAGE_USAGE_STORAGE_BIT)) - flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB | - RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; - break; - case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT: - if (!image || (image->usage & VK_IMAGE_USAGE_STORAGE_BIT)) - flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB | - RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; - break; - default: - break; - } - } - return flush_bits; -} - static void radv_subpass_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_subpass_barrier *barrier) { - cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, barrier->src_access_mask); radv_stage_flush(cmd_buffer, barrier->src_stage_mask); - cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dst_access_mask, - NULL); + + /* TODO: actual cache flushes */ } static void radv_handle_subpass_image_transition(struct radv_cmd_buffer *cmd_buffer, @@ -1651,7 +1086,7 @@ static void radv_handle_subpass_image_transition(struct radv_cmd_buffer *cmd_buf radv_handle_image_transition(cmd_buffer, view->image, cmd_buffer->state.attachments[idx].current_layout, - att.layout, 0, 0, &range, + att.layout, range, cmd_buffer->state.attachments[idx].pending_clear_aspects); cmd_buffer->state.attachments[idx].current_layout = att.layout; @@ -1751,27 +1186,9 @@ VkResult radv_AllocateCommandBuffers( VkResult result = VK_SUCCESS; uint32_t i; - memset(pCommandBuffers, 0, - sizeof(*pCommandBuffers)*pAllocateInfo->commandBufferCount); - for (i = 0; i < pAllocateInfo->commandBufferCount; i++) { - - if (!list_empty(&pool->free_cmd_buffers)) { - struct radv_cmd_buffer *cmd_buffer = list_first_entry(&pool->free_cmd_buffers, struct radv_cmd_buffer, pool_link); - - list_del(&cmd_buffer->pool_link); - list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers); - - radv_reset_cmd_buffer(cmd_buffer); - cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC; - cmd_buffer->level = pAllocateInfo->level; - - pCommandBuffers[i] = radv_cmd_buffer_to_handle(cmd_buffer); - result = VK_SUCCESS; - } else { - result = radv_create_cmd_buffer(device, pool, pAllocateInfo->level, - &pCommandBuffers[i]); - } + result = radv_create_cmd_buffer(device, pool, pAllocateInfo->level, + &pCommandBuffers[i]); if (result != VK_SUCCESS) break; } @@ -1783,6 +1200,24 @@ VkResult radv_AllocateCommandBuffers( return result; } +static void +radv_cmd_buffer_destroy(struct radv_cmd_buffer *cmd_buffer) +{ + list_del(&cmd_buffer->pool_link); + + list_for_each_entry_safe(struct radv_cmd_buffer_upload, up, + &cmd_buffer->upload.list, list) { + cmd_buffer->device->ws->buffer_destroy(up->upload_bo); + list_del(&up->list); + free(up); + } + + if (cmd_buffer->upload.upload_bo) + cmd_buffer->device->ws->buffer_destroy(cmd_buffer->upload.upload_bo); + cmd_buffer->device->ws->cs_destroy(cmd_buffer->cs); + vk_free(&cmd_buffer->pool->alloc, cmd_buffer); +} + void radv_FreeCommandBuffers( VkDevice device, VkCommandPool commandPool, @@ -1792,15 +1227,29 @@ void radv_FreeCommandBuffers( for (uint32_t i = 0; i < commandBufferCount; i++) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, pCommandBuffers[i]); - if (cmd_buffer) { - if (cmd_buffer->pool) { - list_del(&cmd_buffer->pool_link); - list_addtail(&cmd_buffer->pool_link, &cmd_buffer->pool->free_cmd_buffers); - } else - radv_cmd_buffer_destroy(cmd_buffer); + if (cmd_buffer) + radv_cmd_buffer_destroy(cmd_buffer); + } +} - } +static void radv_reset_cmd_buffer(struct radv_cmd_buffer *cmd_buffer) +{ + + cmd_buffer->device->ws->cs_reset(cmd_buffer->cs); + + list_for_each_entry_safe(struct radv_cmd_buffer_upload, up, + &cmd_buffer->upload.list, list) { + cmd_buffer->device->ws->buffer_destroy(up->upload_bo); + list_del(&up->list); + free(up); } + + if (cmd_buffer->upload.upload_bo) + cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, + cmd_buffer->upload.upload_bo, 8); + cmd_buffer->upload.offset = 0; + + cmd_buffer->record_fail = false; } VkResult radv_ResetCommandBuffer( @@ -1812,20 +1261,6 @@ VkResult radv_ResetCommandBuffer( return VK_SUCCESS; } -static void emit_gfx_buffer_state(struct radv_cmd_buffer *cmd_buffer) -{ - struct radv_device *device = cmd_buffer->device; - if (device->gfx_init) { - uint64_t va = device->ws->buffer_get_va(device->gfx_init); - device->ws->cs_add_buffer(cmd_buffer->cs, device->gfx_init, 8); - radeon_emit(cmd_buffer->cs, PKT3(PKT3_INDIRECT_BUFFER_CIK, 2, 0)); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, (va >> 32) & 0xffff); - radeon_emit(cmd_buffer->cs, device->gfx_init_size_dw & 0xffff); - } else - si_init_config(cmd_buffer); -} - VkResult radv_BeginCommandBuffer( VkCommandBuffer commandBuffer, const VkCommandBufferBeginInfo *pBeginInfo) @@ -1834,22 +1269,20 @@ VkResult radv_BeginCommandBuffer( radv_reset_cmd_buffer(cmd_buffer); memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state)); - cmd_buffer->state.last_primitive_reset_en = -1; /* setup initial configuration into command buffer */ if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) { - switch (cmd_buffer->queue_family_index) { - case RADV_QUEUE_GENERAL: - emit_gfx_buffer_state(cmd_buffer); - radv_set_db_count_control(cmd_buffer); - break; - case RADV_QUEUE_COMPUTE: - si_init_compute(cmd_buffer); - break; - case RADV_QUEUE_TRANSFER: - default: - break; - } + /* Flush read caches at the beginning of CS not flushed by the kernel. */ + cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_INV_ICACHE | + RADV_CMD_FLAG_PS_PARTIAL_FLUSH | + RADV_CMD_FLAG_CS_PARTIAL_FLUSH | + RADV_CMD_FLAG_INV_VMEM_L1 | + RADV_CMD_FLAG_INV_SMEM_L1 | + RADV_CMD_FLUSH_AND_INV_FRAMEBUFFER | + RADV_CMD_FLAG_INV_GLOBAL_L2; + si_init_config(&cmd_buffer->device->instance->physicalDevice, cmd_buffer); + radv_set_db_count_control(cmd_buffer); + si_emit_cache_flush(cmd_buffer); } if (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) { @@ -1863,7 +1296,6 @@ VkResult radv_BeginCommandBuffer( radv_cmd_buffer_set_subpass(cmd_buffer, subpass, false); } - radv_cmd_buffer_trace_emit(cmd_buffer); return VK_SUCCESS; } @@ -1910,10 +1342,8 @@ void radv_bind_descriptor_set(struct radv_cmd_buffer *cmd_buffer, { struct radeon_winsys *ws = cmd_buffer->device->ws; - assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR)); - cmd_buffer->state.descriptors[idx] = set; - cmd_buffer->state.descriptors_dirty |= (1 << idx); + if (!set) return; @@ -1921,6 +1351,21 @@ void radv_bind_descriptor_set(struct radv_cmd_buffer *cmd_buffer, if (set->descriptors[j]) ws->cs_add_buffer(cmd_buffer->cs, set->descriptors[j], 7); + radeon_set_sh_reg_seq(cmd_buffer->cs, + R_00B030_SPI_SHADER_USER_DATA_PS_0 + 8 * idx, 2); + radeon_emit(cmd_buffer->cs, set->va); + radeon_emit(cmd_buffer->cs, set->va >> 32); + + radeon_set_sh_reg_seq(cmd_buffer->cs, + R_00B130_SPI_SHADER_USER_DATA_VS_0 + 8 * idx, 2); + radeon_emit(cmd_buffer->cs, set->va); + radeon_emit(cmd_buffer->cs, set->va >> 32); + + radeon_set_sh_reg_seq(cmd_buffer->cs, + R_00B900_COMPUTE_USER_DATA_0 + 8 * idx, 2); + radeon_emit(cmd_buffer->cs, set->va); + radeon_emit(cmd_buffer->cs, set->va >> 32); + if(set->bo) ws->cs_add_buffer(cmd_buffer->cs, set->bo, 8); } @@ -1939,13 +1384,16 @@ void radv_CmdBindDescriptorSets( RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout); unsigned dyn_idx = 0; + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, + MAX_SETS * 4 * 6); + for (unsigned i = 0; i < descriptorSetCount; ++i) { unsigned idx = i + firstSet; RADV_FROM_HANDLE(radv_descriptor_set, set, pDescriptorSets[i]); radv_bind_descriptor_set(cmd_buffer, set, idx); for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) { - unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start; + unsigned idx = j + layout->set[i].dynamic_offset_start; uint32_t *dst = cmd_buffer->dynamic_buffers + idx * 4; assert(dyn_idx < dynamicOffsetCount); @@ -1964,116 +1412,8 @@ void radv_CmdBindDescriptorSets( set->layout->dynamic_shader_stages; } } -} - -static bool radv_init_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, - struct radv_descriptor_set *set, - struct radv_descriptor_set_layout *layout) -{ - set->size = layout->size; - set->layout = layout; - - if (cmd_buffer->push_descriptors.capacity < set->size) { - size_t new_size = MAX2(set->size, 1024); - new_size = MAX2(new_size, 2 * cmd_buffer->push_descriptors.capacity); - new_size = MIN2(new_size, 96 * MAX_PUSH_DESCRIPTORS); - - free(set->mapped_ptr); - set->mapped_ptr = malloc(new_size); - - if (!set->mapped_ptr) { - cmd_buffer->push_descriptors.capacity = 0; - cmd_buffer->record_fail = true; - return false; - } - - cmd_buffer->push_descriptors.capacity = new_size; - } - - return true; -} - -void radv_meta_push_descriptor_set( - struct radv_cmd_buffer* cmd_buffer, - VkPipelineBindPoint pipelineBindPoint, - VkPipelineLayout _layout, - uint32_t set, - uint32_t descriptorWriteCount, - const VkWriteDescriptorSet* pDescriptorWrites) -{ - RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout); - struct radv_descriptor_set *push_set = &cmd_buffer->meta_push_descriptors; - unsigned bo_offset; - - assert(layout->set[set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR); - - push_set->size = layout->set[set].layout->size; - push_set->layout = layout->set[set].layout; - - if (!radv_cmd_buffer_upload_alloc(cmd_buffer, push_set->size, 32, - &bo_offset, - (void**) &push_set->mapped_ptr)) - return; - - push_set->va = cmd_buffer->device->ws->buffer_get_va(cmd_buffer->upload.upload_bo); - push_set->va += bo_offset; - - radv_update_descriptor_sets(cmd_buffer->device, cmd_buffer, - radv_descriptor_set_to_handle(push_set), - descriptorWriteCount, pDescriptorWrites, 0, NULL); - - cmd_buffer->state.descriptors[set] = push_set; - cmd_buffer->state.descriptors_dirty |= (1 << set); -} - -void radv_CmdPushDescriptorSetKHR( - VkCommandBuffer commandBuffer, - VkPipelineBindPoint pipelineBindPoint, - VkPipelineLayout _layout, - uint32_t set, - uint32_t descriptorWriteCount, - const VkWriteDescriptorSet* pDescriptorWrites) -{ - RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout); - struct radv_descriptor_set *push_set = &cmd_buffer->push_descriptors.set; - - assert(layout->set[set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR); - - if (!radv_init_push_descriptor_set(cmd_buffer, push_set, layout->set[set].layout)) - return; - - radv_update_descriptor_sets(cmd_buffer->device, cmd_buffer, - radv_descriptor_set_to_handle(push_set), - descriptorWriteCount, pDescriptorWrites, 0, NULL); - - cmd_buffer->state.descriptors[set] = push_set; - cmd_buffer->state.descriptors_dirty |= (1 << set); - cmd_buffer->state.push_descriptors_dirty = true; -} - -void radv_CmdPushDescriptorSetWithTemplateKHR( - VkCommandBuffer commandBuffer, - VkDescriptorUpdateTemplateKHR descriptorUpdateTemplate, - VkPipelineLayout _layout, - uint32_t set, - const void* pData) -{ - RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout); - struct radv_descriptor_set *push_set = &cmd_buffer->push_descriptors.set; - - assert(layout->set[set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR); - - if (!radv_init_push_descriptor_set(cmd_buffer, push_set, layout->set[set].layout)) - return; - - radv_update_descriptor_set_with_template(cmd_buffer->device, cmd_buffer, push_set, - descriptorUpdateTemplate, pData); - cmd_buffer->state.descriptors[set] = push_set; - cmd_buffer->state.descriptors_dirty |= (1 << set); - cmd_buffer->state.push_descriptors_dirty = true; + assert(cmd_buffer->cs->cdw <= cdw_max); } void radv_CmdPushConstants(VkCommandBuffer commandBuffer, @@ -2093,9 +1433,7 @@ VkResult radv_EndCommandBuffer( { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - if (cmd_buffer->queue_family_index != RADV_QUEUE_TRANSFER) - si_emit_cache_flush(cmd_buffer); - + si_emit_cache_flush(cmd_buffer); if (!cmd_buffer->device->ws->cs_finalize(cmd_buffer->cs) || cmd_buffer->record_fail) return VK_ERROR_OUT_OF_DEVICE_MEMORY; @@ -2120,8 +1458,7 @@ radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer) ws->cs_add_buffer(cmd_buffer->cs, compute_shader->bo, 8); - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, - cmd_buffer->cs, 16); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 16); radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B830_COMPUTE_PGM_LO, 2); radeon_emit(cmd_buffer->cs, va >> 8); @@ -2131,15 +1468,9 @@ radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer) radeon_emit(cmd_buffer->cs, compute_shader->rsrc1); radeon_emit(cmd_buffer->cs, compute_shader->rsrc2); - - cmd_buffer->compute_scratch_size_needed = - MAX2(cmd_buffer->compute_scratch_size_needed, - pipeline->max_waves * pipeline->scratch_bytes_per_wave); - /* change these once we have scratch support */ radeon_set_sh_reg(cmd_buffer->cs, R_00B860_COMPUTE_TMPRING_SIZE, - S_00B860_WAVES(pipeline->max_waves) | - S_00B860_WAVESIZE(pipeline->scratch_bytes_per_wave >> 10)); + S_00B860_WAVES(32) | S_00B860_WAVESIZE(0)); radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); radeon_emit(cmd_buffer->cs, @@ -2152,13 +1483,6 @@ radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer) assert(cmd_buffer->cs->cdw <= cdw_max); } -static void radv_mark_descriptor_sets_dirty(struct radv_cmd_buffer *cmd_buffer) -{ - for (unsigned i = 0; i < MAX_SETS; i++) { - if (cmd_buffer->state.descriptors[i]) - cmd_buffer->state.descriptors_dirty |= (1u << i); - } -} void radv_CmdBindPipeline( VkCommandBuffer commandBuffer, @@ -2168,8 +1492,6 @@ void radv_CmdBindPipeline( RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); RADV_FROM_HANDLE(radv_pipeline, pipeline, _pipeline); - radv_mark_descriptor_sets_dirty(cmd_buffer); - switch (pipelineBindPoint) { case VK_PIPELINE_BIND_POINT_COMPUTE: cmd_buffer->state.compute_pipeline = pipeline; @@ -2177,9 +1499,6 @@ void radv_CmdBindPipeline( break; case VK_PIPELINE_BIND_POINT_GRAPHICS: cmd_buffer->state.pipeline = pipeline; - if (!pipeline) - break; - cmd_buffer->state.vertex_descriptors_dirty = true; cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE; cmd_buffer->push_constant_stages |= pipeline->active_stages; @@ -2189,23 +1508,6 @@ void radv_CmdBindPipeline( radv_dynamic_state_copy(&cmd_buffer->state.dynamic, &pipeline->dynamic_state, pipeline->dynamic_state_mask); - - if (pipeline->graphics.esgs_ring_size > cmd_buffer->esgs_ring_size_needed) - cmd_buffer->esgs_ring_size_needed = pipeline->graphics.esgs_ring_size; - if (pipeline->graphics.gsvs_ring_size > cmd_buffer->gsvs_ring_size_needed) - cmd_buffer->gsvs_ring_size_needed = pipeline->graphics.gsvs_ring_size; - - if (radv_pipeline_has_tess(pipeline)) - cmd_buffer->tess_rings_needed = true; - - if (radv_pipeline_has_gs(pipeline)) { - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.pipeline, MESA_SHADER_GEOMETRY, - AC_UD_SCRATCH_RING_OFFSETS); - if (cmd_buffer->ring_offsets_idx == -1) - cmd_buffer->ring_offsets_idx = loc->sgpr_idx; - else if (loc->sgpr_idx != -1) - assert(loc->sgpr_idx == cmd_buffer->ring_offsets_idx); - } break; default: assert(!"invalid bind point"); @@ -2342,6 +1644,7 @@ void radv_CmdSetStencilReference( cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE; } + void radv_CmdExecuteCommands( VkCommandBuffer commandBuffer, uint32_t commandBufferCount, @@ -2349,44 +1652,17 @@ void radv_CmdExecuteCommands( { RADV_FROM_HANDLE(radv_cmd_buffer, primary, commandBuffer); - /* Emit pending flushes on primary prior to executing secondary */ - si_emit_cache_flush(primary); - for (uint32_t i = 0; i < commandBufferCount; i++) { RADV_FROM_HANDLE(radv_cmd_buffer, secondary, pCmdBuffers[i]); - primary->scratch_size_needed = MAX2(primary->scratch_size_needed, - secondary->scratch_size_needed); - primary->compute_scratch_size_needed = MAX2(primary->compute_scratch_size_needed, - secondary->compute_scratch_size_needed); - - if (secondary->esgs_ring_size_needed > primary->esgs_ring_size_needed) - primary->esgs_ring_size_needed = secondary->esgs_ring_size_needed; - if (secondary->gsvs_ring_size_needed > primary->gsvs_ring_size_needed) - primary->gsvs_ring_size_needed = secondary->gsvs_ring_size_needed; - if (secondary->tess_rings_needed) - primary->tess_rings_needed = true; - if (secondary->sample_positions_needed) - primary->sample_positions_needed = true; - - if (secondary->ring_offsets_idx != -1) { - if (primary->ring_offsets_idx == -1) - primary->ring_offsets_idx = secondary->ring_offsets_idx; - else - assert(secondary->ring_offsets_idx == primary->ring_offsets_idx); - } primary->device->ws->cs_execute_secondary(primary->cs, secondary->cs); } /* if we execute secondary we need to re-emit out pipelines */ if (commandBufferCount) { primary->state.emitted_pipeline = NULL; - primary->state.emitted_compute_pipeline = NULL; primary->state.dirty |= RADV_CMD_DIRTY_PIPELINE; primary->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_ALL; - primary->state.last_primitive_reset_en = -1; - primary->state.last_primitive_reset_index = 0; - radv_mark_descriptor_sets_dirty(primary); } } @@ -2410,9 +1686,6 @@ VkResult radv_CreateCommandPool( pool->alloc = device->alloc; list_inithead(&pool->cmd_buffers); - list_inithead(&pool->free_cmd_buffers); - - pool->queue_family_index = pCreateInfo->queueFamilyIndex; *pCmdPool = radv_cmd_pool_to_handle(pool); @@ -2436,11 +1709,6 @@ void radv_DestroyCommandPool( radv_cmd_buffer_destroy(cmd_buffer); } - list_for_each_entry_safe(struct radv_cmd_buffer, cmd_buffer, - &pool->free_cmd_buffers, pool_link) { - radv_cmd_buffer_destroy(cmd_buffer); - } - vk_free2(&device->alloc, pAllocator, pool); } @@ -2459,22 +1727,6 @@ VkResult radv_ResetCommandPool( return VK_SUCCESS; } -void radv_TrimCommandPoolKHR( - VkDevice device, - VkCommandPool commandPool, - VkCommandPoolTrimFlagsKHR flags) -{ - RADV_FROM_HANDLE(radv_cmd_pool, pool, commandPool); - - if (!pool) - return; - - list_for_each_entry_safe(struct radv_cmd_buffer, cmd_buffer, - &pool->free_cmd_buffers, pool_link) { - radv_cmd_buffer_destroy(cmd_buffer); - } -} - void radv_CmdBeginRenderPass( VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo* pRenderPassBegin, @@ -2484,14 +1736,16 @@ void radv_CmdBeginRenderPass( RADV_FROM_HANDLE(radv_render_pass, pass, pRenderPassBegin->renderPass); RADV_FROM_HANDLE(radv_framebuffer, framebuffer, pRenderPassBegin->framebuffer); - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, - cmd_buffer->cs, 2048); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, + 2048); cmd_buffer->state.framebuffer = framebuffer; cmd_buffer->state.pass = pass; cmd_buffer->state.render_area = pRenderPassBegin->renderArea; radv_cmd_state_setup_attachments(cmd_buffer, pass, pRenderPassBegin); + si_emit_cache_flush(cmd_buffer); + radv_cmd_buffer_set_subpass(cmd_buffer, pass->subpasses, true); assert(cmd_buffer->cs->cdw <= cdw_max); @@ -2504,6 +1758,7 @@ void radv_CmdNextSubpass( { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + si_emit_cache_flush(cmd_buffer); radv_cmd_buffer_resolve_subpass(cmd_buffer); radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, @@ -2521,21 +1776,13 @@ void radv_CmdDraw( uint32_t firstInstance) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + radv_cmd_buffer_flush_state(cmd_buffer); - radv_cmd_buffer_flush_state(cmd_buffer, false, (instanceCount > 1), false, vertexCount); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 9); - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 10); - - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.pipeline, MESA_SHADER_VERTEX, - AC_UD_VS_BASE_VERTEX_START_INSTANCE); - if (loc->sgpr_idx != -1) { - uint32_t base_reg = shader_stage_to_user_data_0(MESA_SHADER_VERTEX, radv_pipeline_has_gs(cmd_buffer->state.pipeline), - radv_pipeline_has_tess(cmd_buffer->state.pipeline)); - radeon_set_sh_reg_seq(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, 3); - radeon_emit(cmd_buffer->cs, firstVertex); - radeon_emit(cmd_buffer->cs, firstInstance); - radeon_emit(cmd_buffer->cs, 0); - } + radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B130_SPI_SHADER_USER_DATA_VS_0 + AC_USERDATA_VS_BASE_VERTEX * 4, 2); + radeon_emit(cmd_buffer->cs, firstVertex); + radeon_emit(cmd_buffer->cs, firstInstance); radeon_emit(cmd_buffer->cs, PKT3(PKT3_NUM_INSTANCES, 0, 0)); radeon_emit(cmd_buffer->cs, instanceCount); @@ -2545,8 +1792,18 @@ void radv_CmdDraw( S_0287F0_USE_OPAQUE(0)); assert(cmd_buffer->cs->cdw <= cdw_max); +} - radv_cmd_buffer_trace_emit(cmd_buffer); +static void radv_emit_primitive_reset_index(struct radv_cmd_buffer *cmd_buffer) +{ + uint32_t primitive_reset_index = cmd_buffer->state.last_primitive_reset_index ? 0xffffffffu : 0xffffu; + + if (cmd_buffer->state.pipeline->graphics.prim_restart_enable && + primitive_reset_index != cmd_buffer->state.last_primitive_reset_index) { + cmd_buffer->state.last_primitive_reset_index = primitive_reset_index; + radeon_set_context_reg(cmd_buffer->cs, R_02840C_VGT_MULTI_PRIM_IB_RESET_INDX, + primitive_reset_index); + } } void radv_CmdDrawIndexed( @@ -2562,23 +1819,17 @@ void radv_CmdDrawIndexed( uint32_t index_max_size = (cmd_buffer->state.index_buffer->size - cmd_buffer->state.index_offset) / index_size; uint64_t index_va; - radv_cmd_buffer_flush_state(cmd_buffer, true, (instanceCount > 1), false, indexCount); + radv_cmd_buffer_flush_state(cmd_buffer); + radv_emit_primitive_reset_index(cmd_buffer); - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 15); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 14); radeon_emit(cmd_buffer->cs, PKT3(PKT3_INDEX_TYPE, 0, 0)); radeon_emit(cmd_buffer->cs, cmd_buffer->state.index_type); - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.pipeline, MESA_SHADER_VERTEX, - AC_UD_VS_BASE_VERTEX_START_INSTANCE); - if (loc->sgpr_idx != -1) { - uint32_t base_reg = shader_stage_to_user_data_0(MESA_SHADER_VERTEX, radv_pipeline_has_gs(cmd_buffer->state.pipeline), - radv_pipeline_has_tess(cmd_buffer->state.pipeline)); - radeon_set_sh_reg_seq(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, 3); - radeon_emit(cmd_buffer->cs, vertexOffset); - radeon_emit(cmd_buffer->cs, firstInstance); - radeon_emit(cmd_buffer->cs, 0); - } + radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B130_SPI_SHADER_USER_DATA_VS_0 + AC_USERDATA_VS_BASE_VERTEX * 4, 2); + radeon_emit(cmd_buffer->cs, vertexOffset); + radeon_emit(cmd_buffer->cs, firstInstance); radeon_emit(cmd_buffer->cs, PKT3(PKT3_NUM_INSTANCES, 0, 0)); radeon_emit(cmd_buffer->cs, instanceCount); @@ -2592,43 +1843,28 @@ void radv_CmdDrawIndexed( radeon_emit(cmd_buffer->cs, V_0287F0_DI_SRC_SEL_DMA); assert(cmd_buffer->cs->cdw <= cdw_max); - radv_cmd_buffer_trace_emit(cmd_buffer); } static void radv_emit_indirect_draw(struct radv_cmd_buffer *cmd_buffer, VkBuffer _buffer, VkDeviceSize offset, - VkBuffer _count_buffer, - VkDeviceSize count_offset, uint32_t draw_count, uint32_t stride, bool indexed) { RADV_FROM_HANDLE(radv_buffer, buffer, _buffer); - RADV_FROM_HANDLE(radv_buffer, count_buffer, _count_buffer); struct radeon_winsys_cs *cs = cmd_buffer->cs; unsigned di_src_sel = indexed ? V_0287F0_DI_SRC_SEL_DMA : V_0287F0_DI_SRC_SEL_AUTO_INDEX; uint64_t indirect_va = cmd_buffer->device->ws->buffer_get_va(buffer->bo); indirect_va += offset + buffer->offset; - uint64_t count_va = 0; - - if (count_buffer) { - count_va = cmd_buffer->device->ws->buffer_get_va(count_buffer->bo); - count_va += count_offset + count_buffer->offset; - } if (!draw_count) return; cmd_buffer->device->ws->cs_add_buffer(cs, buffer->bo, 8); - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.pipeline, MESA_SHADER_VERTEX, - AC_UD_VS_BASE_VERTEX_START_INSTANCE); - uint32_t base_reg = shader_stage_to_user_data_0(MESA_SHADER_VERTEX, radv_pipeline_has_gs(cmd_buffer->state.pipeline), - radv_pipeline_has_tess(cmd_buffer->state.pipeline)); - assert(loc->sgpr_idx != -1); radeon_emit(cs, PKT3(PKT3_SET_BASE, 2, 0)); radeon_emit(cs, 1); radeon_emit(cs, indirect_va); @@ -2638,60 +1874,51 @@ radv_emit_indirect_draw(struct radv_cmd_buffer *cmd_buffer, PKT3_DRAW_INDIRECT_MULTI, 8, false)); radeon_emit(cs, 0); - radeon_emit(cs, ((base_reg + loc->sgpr_idx * 4) - SI_SH_REG_OFFSET) >> 2); - radeon_emit(cs, ((base_reg + (loc->sgpr_idx + 1) * 4) - SI_SH_REG_OFFSET) >> 2); - radeon_emit(cs, (((base_reg + (loc->sgpr_idx + 2) * 4) - SI_SH_REG_OFFSET) >> 2) | - S_2C3_DRAW_INDEX_ENABLE(1) | - S_2C3_COUNT_INDIRECT_ENABLE(!!count_va)); + radeon_emit(cs, ((R_00B130_SPI_SHADER_USER_DATA_VS_0 + AC_USERDATA_VS_BASE_VERTEX * 4) - SI_SH_REG_OFFSET) >> 2); + radeon_emit(cs, ((R_00B130_SPI_SHADER_USER_DATA_VS_0 + AC_USERDATA_VS_START_INSTANCE * 4) - SI_SH_REG_OFFSET) >> 2); + radeon_emit(cs, 0); /* draw_index */ radeon_emit(cs, draw_count); /* count */ - radeon_emit(cs, count_va); /* count_addr */ - radeon_emit(cs, count_va >> 32); + radeon_emit(cs, 0); /* count_addr -- disabled */ + radeon_emit(cs, 0); radeon_emit(cs, stride); /* stride */ radeon_emit(cs, di_src_sel); - radv_cmd_buffer_trace_emit(cmd_buffer); } -static void -radv_cmd_draw_indirect_count(VkCommandBuffer commandBuffer, - VkBuffer buffer, - VkDeviceSize offset, - VkBuffer countBuffer, - VkDeviceSize countBufferOffset, - uint32_t maxDrawCount, - uint32_t stride) +void radv_CmdDrawIndirect( + VkCommandBuffer commandBuffer, + VkBuffer _buffer, + VkDeviceSize offset, + uint32_t drawCount, + uint32_t stride) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - radv_cmd_buffer_flush_state(cmd_buffer, false, false, true, 0); + radv_cmd_buffer_flush_state(cmd_buffer); - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, - cmd_buffer->cs, 14); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 14); - radv_emit_indirect_draw(cmd_buffer, buffer, offset, - countBuffer, countBufferOffset, maxDrawCount, stride, false); + radv_emit_indirect_draw(cmd_buffer, _buffer, offset, drawCount, stride, false); assert(cmd_buffer->cs->cdw <= cdw_max); } -static void -radv_cmd_draw_indexed_indirect_count( +void radv_CmdDrawIndexedIndirect( VkCommandBuffer commandBuffer, - VkBuffer buffer, + VkBuffer _buffer, VkDeviceSize offset, - VkBuffer countBuffer, - VkDeviceSize countBufferOffset, - uint32_t maxDrawCount, + uint32_t drawCount, uint32_t stride) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); int index_size = cmd_buffer->state.index_type ? 4 : 2; uint32_t index_max_size = (cmd_buffer->state.index_buffer->size - cmd_buffer->state.index_offset) / index_size; uint64_t index_va; - radv_cmd_buffer_flush_state(cmd_buffer, true, false, true, 0); + radv_cmd_buffer_flush_state(cmd_buffer); + radv_emit_primitive_reset_index(cmd_buffer); index_va = cmd_buffer->device->ws->buffer_get_va(cmd_buffer->state.index_buffer->bo); index_va += cmd_buffer->state.index_buffer->offset + cmd_buffer->state.index_offset; - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 21); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 21); radeon_emit(cmd_buffer->cs, PKT3(PKT3_INDEX_TYPE, 0, 0)); radeon_emit(cmd_buffer->cs, cmd_buffer->state.index_type); @@ -2703,72 +1930,11 @@ radv_cmd_draw_indexed_indirect_count( radeon_emit(cmd_buffer->cs, PKT3(PKT3_INDEX_BUFFER_SIZE, 0, 0)); radeon_emit(cmd_buffer->cs, index_max_size); - radv_emit_indirect_draw(cmd_buffer, buffer, offset, - countBuffer, countBufferOffset, maxDrawCount, stride, true); + radv_emit_indirect_draw(cmd_buffer, _buffer, offset, drawCount, stride, true); assert(cmd_buffer->cs->cdw <= cdw_max); } -void radv_CmdDrawIndirect( - VkCommandBuffer commandBuffer, - VkBuffer buffer, - VkDeviceSize offset, - uint32_t drawCount, - uint32_t stride) -{ - radv_cmd_draw_indirect_count(commandBuffer, buffer, offset, - VK_NULL_HANDLE, 0, drawCount, stride); -} - -void radv_CmdDrawIndexedIndirect( - VkCommandBuffer commandBuffer, - VkBuffer buffer, - VkDeviceSize offset, - uint32_t drawCount, - uint32_t stride) -{ - radv_cmd_draw_indexed_indirect_count(commandBuffer, buffer, offset, - VK_NULL_HANDLE, 0, drawCount, stride); -} - -void radv_CmdDrawIndirectCountAMD( - VkCommandBuffer commandBuffer, - VkBuffer buffer, - VkDeviceSize offset, - VkBuffer countBuffer, - VkDeviceSize countBufferOffset, - uint32_t maxDrawCount, - uint32_t stride) -{ - radv_cmd_draw_indirect_count(commandBuffer, buffer, offset, - countBuffer, countBufferOffset, - maxDrawCount, stride); -} - -void radv_CmdDrawIndexedIndirectCountAMD( - VkCommandBuffer commandBuffer, - VkBuffer buffer, - VkDeviceSize offset, - VkBuffer countBuffer, - VkDeviceSize countBufferOffset, - uint32_t maxDrawCount, - uint32_t stride) -{ - radv_cmd_draw_indexed_indirect_count(commandBuffer, buffer, offset, - countBuffer, countBufferOffset, - maxDrawCount, stride); -} - -static void -radv_flush_compute_state(struct radv_cmd_buffer *cmd_buffer) -{ - radv_emit_compute_pipeline(cmd_buffer); - radv_flush_descriptors(cmd_buffer, VK_SHADER_STAGE_COMPUTE_BIT); - radv_flush_constants(cmd_buffer, cmd_buffer->state.compute_pipeline, - VK_SHADER_STAGE_COMPUTE_BIT); - si_emit_cache_flush(cmd_buffer); -} - void radv_CmdDispatch( VkCommandBuffer commandBuffer, uint32_t x, @@ -2777,20 +1943,16 @@ void radv_CmdDispatch( { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - radv_flush_compute_state(cmd_buffer); - - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 10); + radv_emit_compute_pipeline(cmd_buffer); + radv_flush_constants(cmd_buffer, cmd_buffer->state.compute_pipeline->layout, + VK_SHADER_STAGE_COMPUTE_BIT); + si_emit_cache_flush(cmd_buffer); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 10); - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.compute_pipeline, - MESA_SHADER_COMPUTE, AC_UD_CS_GRID_SIZE); - if (loc->sgpr_idx != -1) { - assert(!loc->indirect); - assert(loc->num_sgprs == 3); - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B900_COMPUTE_USER_DATA_0 + loc->sgpr_idx * 4, 3); - radeon_emit(cmd_buffer->cs, x); - radeon_emit(cmd_buffer->cs, y); - radeon_emit(cmd_buffer->cs, z); - } + radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B900_COMPUTE_USER_DATA_0 + AC_USERDATA_CS_GRID_SIZE * 4, 3); + radeon_emit(cmd_buffer->cs, x); + radeon_emit(cmd_buffer->cs, y); + radeon_emit(cmd_buffer->cs, z); radeon_emit(cmd_buffer->cs, PKT3(PKT3_DISPATCH_DIRECT, 3, 0) | PKT3_SHADER_TYPE_S(1)); @@ -2800,7 +1962,6 @@ void radv_CmdDispatch( radeon_emit(cmd_buffer->cs, 1); assert(cmd_buffer->cs->cdw <= cdw_max); - radv_cmd_buffer_trace_emit(cmd_buffer); } void radv_CmdDispatchIndirect( @@ -2815,44 +1976,35 @@ void radv_CmdDispatchIndirect( cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, buffer->bo, 8); - radv_flush_compute_state(cmd_buffer); - - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 25); - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.compute_pipeline, - MESA_SHADER_COMPUTE, AC_UD_CS_GRID_SIZE); - if (loc->sgpr_idx != -1) { - for (unsigned i = 0; i < 3; ++i) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_COPY_DATA, 4, 0)); - radeon_emit(cmd_buffer->cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) | - COPY_DATA_DST_SEL(COPY_DATA_REG)); - radeon_emit(cmd_buffer->cs, (va + 4 * i)); - radeon_emit(cmd_buffer->cs, (va + 4 * i) >> 32); - radeon_emit(cmd_buffer->cs, ((R_00B900_COMPUTE_USER_DATA_0 + loc->sgpr_idx * 4) >> 2) + i); - radeon_emit(cmd_buffer->cs, 0); - } - } + radv_emit_compute_pipeline(cmd_buffer); + radv_flush_constants(cmd_buffer, cmd_buffer->state.compute_pipeline->layout, + VK_SHADER_STAGE_COMPUTE_BIT); + si_emit_cache_flush(cmd_buffer); - if (radv_cmd_buffer_uses_mec(cmd_buffer)) { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_DISPATCH_INDIRECT, 2, 0) | - PKT3_SHADER_TYPE_S(1)); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); - radeon_emit(cmd_buffer->cs, 1); - } else { - radeon_emit(cmd_buffer->cs, PKT3(PKT3_SET_BASE, 2, 0) | - PKT3_SHADER_TYPE_S(1)); - radeon_emit(cmd_buffer->cs, 1); - radeon_emit(cmd_buffer->cs, va); - radeon_emit(cmd_buffer->cs, va >> 32); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 25); - radeon_emit(cmd_buffer->cs, PKT3(PKT3_DISPATCH_INDIRECT, 1, 0) | - PKT3_SHADER_TYPE_S(1)); + for (unsigned i = 0; i < 3; ++i) { + radeon_emit(cmd_buffer->cs, PKT3(PKT3_COPY_DATA, 4, 0)); + radeon_emit(cmd_buffer->cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) | + COPY_DATA_DST_SEL(COPY_DATA_REG)); + radeon_emit(cmd_buffer->cs, (va + 4 * i)); + radeon_emit(cmd_buffer->cs, (va + 4 * i) >> 32); + radeon_emit(cmd_buffer->cs, ((R_00B900_COMPUTE_USER_DATA_0 + AC_USERDATA_CS_GRID_SIZE * 4) >> 2) + i); radeon_emit(cmd_buffer->cs, 0); - radeon_emit(cmd_buffer->cs, 1); } + radeon_emit(cmd_buffer->cs, PKT3(PKT3_SET_BASE, 2, 0) | + PKT3_SHADER_TYPE_S(1)); + radeon_emit(cmd_buffer->cs, 1); + radeon_emit(cmd_buffer->cs, va); + radeon_emit(cmd_buffer->cs, va >> 32); + + radeon_emit(cmd_buffer->cs, PKT3(PKT3_DISPATCH_INDIRECT, 1, 0) | + PKT3_SHADER_TYPE_S(1)); + radeon_emit(cmd_buffer->cs, 0); + radeon_emit(cmd_buffer->cs, 1); + assert(cmd_buffer->cs->cdw <= cdw_max); - radv_cmd_buffer_trace_emit(cmd_buffer); } void radv_unaligned_dispatch( @@ -2874,9 +2026,11 @@ void radv_unaligned_dispatch( remainder[1] = y + compute_shader->info.cs.block_size[1] - align_u32_npot(y, compute_shader->info.cs.block_size[1]); remainder[2] = z + compute_shader->info.cs.block_size[2] - align_u32_npot(z, compute_shader->info.cs.block_size[2]); - radv_flush_compute_state(cmd_buffer); - - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 15); + radv_emit_compute_pipeline(cmd_buffer); + radv_flush_constants(cmd_buffer, cmd_buffer->state.compute_pipeline->layout, + VK_SHADER_STAGE_COMPUTE_BIT); + si_emit_cache_flush(cmd_buffer); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 15); radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3); radeon_emit(cmd_buffer->cs, @@ -2889,14 +2043,11 @@ void radv_unaligned_dispatch( S_00B81C_NUM_THREAD_FULL(compute_shader->info.cs.block_size[2]) | S_00B81C_NUM_THREAD_PARTIAL(remainder[2])); - struct ac_userdata_info *loc = radv_lookup_user_sgpr(cmd_buffer->state.compute_pipeline, - MESA_SHADER_COMPUTE, AC_UD_CS_GRID_SIZE); - if (loc->sgpr_idx != -1) { - radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B900_COMPUTE_USER_DATA_0 + loc->sgpr_idx * 4, 3); - radeon_emit(cmd_buffer->cs, blocks[0]); - radeon_emit(cmd_buffer->cs, blocks[1]); - radeon_emit(cmd_buffer->cs, blocks[2]); - } + radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B900_COMPUTE_USER_DATA_0 + AC_USERDATA_CS_GRID_SIZE * 4, 3); + radeon_emit(cmd_buffer->cs, blocks[0]); + radeon_emit(cmd_buffer->cs, blocks[1]); + radeon_emit(cmd_buffer->cs, blocks[2]); + radeon_emit(cmd_buffer->cs, PKT3(PKT3_DISPATCH_DIRECT, 3, 0) | PKT3_SHADER_TYPE_S(1)); radeon_emit(cmd_buffer->cs, blocks[0]); @@ -2906,7 +2057,6 @@ void radv_unaligned_dispatch( S_00B800_PARTIAL_TG_EN(1)); assert(cmd_buffer->cs->cdw <= cdw_max); - radv_cmd_buffer_trace_emit(cmd_buffer); } void radv_CmdEndRenderPass( @@ -2916,6 +2066,7 @@ void radv_CmdEndRenderPass( radv_subpass_barrier(cmd_buffer, &cmd_buffer->state.pass->end_barrier); + si_emit_cache_flush(cmd_buffer); radv_cmd_buffer_resolve_subpass(cmd_buffer); for (unsigned i = 0; i < cmd_buffer->state.framebuffer->attachment_count; ++i) { @@ -2934,32 +2085,26 @@ void radv_CmdEndRenderPass( static void radv_initialize_htile(struct radv_cmd_buffer *cmd_buffer, - struct radv_image *image, - const VkImageSubresourceRange *range) + struct radv_image *image) { - assert(range->baseMipLevel == 0); - assert(range->levelCount == 1 || range->levelCount == VK_REMAINING_ARRAY_LAYERS); - unsigned layer_count = radv_get_layerCount(image, range); - uint64_t size = image->surface.htile_slice_size * layer_count; - uint64_t offset = image->offset + image->htile_offset + - image->surface.htile_slice_size * range->baseArrayLayer; cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB | RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; - radv_fill_buffer(cmd_buffer, image->bo, offset, size, 0xffffffff); + radv_fill_buffer(cmd_buffer, image->bo, image->offset + image->htile.offset, + image->htile.size, 0xffffffff); cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META | RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VMEM_L1 | - RADV_CMD_FLAG_WRITEBACK_GLOBAL_L2; + RADV_CMD_FLAG_INV_GLOBAL_L2; } static void radv_handle_depth_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout src_layout, VkImageLayout dst_layout, - const VkImageSubresourceRange *range, + VkImageSubresourceRange range, VkImageAspectFlags pending_clears) { if (dst_layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL && @@ -2972,26 +2117,20 @@ static void radv_handle_depth_image_transition(struct radv_cmd_buffer *cmd_buffe } else if (src_layout == VK_IMAGE_LAYOUT_UNDEFINED && radv_layout_has_htile(image, dst_layout)) { /* TODO: merge with the clear if applicable */ - radv_initialize_htile(cmd_buffer, image, range); + radv_initialize_htile(cmd_buffer, image); } else if (!radv_layout_has_htile(image, src_layout) && radv_layout_has_htile(image, dst_layout)) { - radv_initialize_htile(cmd_buffer, image, range); + radv_initialize_htile(cmd_buffer, image); } else if ((radv_layout_has_htile(image, src_layout) && !radv_layout_has_htile(image, dst_layout)) || (radv_layout_is_htile_compressed(image, src_layout) && !radv_layout_is_htile_compressed(image, dst_layout))) { - VkImageSubresourceRange local_range = *range; - local_range.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT; - local_range.baseMipLevel = 0; - local_range.levelCount = 1; - - cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB | - RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; - radv_decompress_depth_image_inplace(cmd_buffer, image, &local_range); + range.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT; + range.baseMipLevel = 0; + range.levelCount = 1; - cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB | - RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; + radv_decompress_depth_image_inplace(cmd_buffer, image, &range); } } @@ -3007,16 +2146,14 @@ void radv_initialise_cmask(struct radv_cmd_buffer *cmd_buffer, cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB_META | RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VMEM_L1 | - RADV_CMD_FLAG_WRITEBACK_GLOBAL_L2; + RADV_CMD_FLAG_INV_GLOBAL_L2; } static void radv_handle_cmask_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout src_layout, VkImageLayout dst_layout, - unsigned src_queue_mask, - unsigned dst_queue_mask, - const VkImageSubresourceRange *range, + VkImageSubresourceRange range, VkImageAspectFlags pending_clears) { if (src_layout == VK_IMAGE_LAYOUT_UNDEFINED) { @@ -3024,9 +2161,9 @@ static void radv_handle_cmask_image_transition(struct radv_cmd_buffer *cmd_buffe radv_initialise_cmask(cmd_buffer, image, 0xccccccccu); else radv_initialise_cmask(cmd_buffer, image, 0xffffffffu); - } else if (radv_layout_can_fast_clear(image, src_layout, src_queue_mask) && - !radv_layout_can_fast_clear(image, dst_layout, dst_queue_mask)) { - radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); + } else if (radv_layout_has_cmask(image, src_layout) && + !radv_layout_has_cmask(image, dst_layout)) { + radv_fast_clear_flush_image_inplace(cmd_buffer, image); } } @@ -3044,23 +2181,21 @@ void radv_initialize_dcc(struct radv_cmd_buffer *cmd_buffer, RADV_CMD_FLAG_FLUSH_AND_INV_CB_META | RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VMEM_L1 | - RADV_CMD_FLAG_WRITEBACK_GLOBAL_L2; + RADV_CMD_FLAG_INV_GLOBAL_L2; } static void radv_handle_dcc_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout src_layout, VkImageLayout dst_layout, - unsigned src_queue_mask, - unsigned dst_queue_mask, - const VkImageSubresourceRange *range, + VkImageSubresourceRange range, VkImageAspectFlags pending_clears) { if (src_layout == VK_IMAGE_LAYOUT_UNDEFINED) { radv_initialize_dcc(cmd_buffer, image, 0x20202020u); - } else if (radv_layout_can_fast_clear(image, src_layout, src_queue_mask) && - !radv_layout_can_fast_clear(image, dst_layout, dst_queue_mask)) { - radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); + } else if(src_layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL && + dst_layout != VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL) { + radv_fast_clear_flush_image_inplace(cmd_buffer, image); } } @@ -3068,46 +2203,20 @@ static void radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout src_layout, VkImageLayout dst_layout, - uint32_t src_family, - uint32_t dst_family, - const VkImageSubresourceRange *range, + VkImageSubresourceRange range, VkImageAspectFlags pending_clears) { - if (image->exclusive && src_family != dst_family) { - /* This is an acquire or a release operation and there will be - * a corresponding release/acquire. Do the transition in the - * most flexible queue. */ - - assert(src_family == cmd_buffer->queue_family_index || - dst_family == cmd_buffer->queue_family_index); - - if (cmd_buffer->queue_family_index == RADV_QUEUE_TRANSFER) - return; - - if (cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE && - (src_family == RADV_QUEUE_GENERAL || - dst_family == RADV_QUEUE_GENERAL)) - return; - } - - unsigned src_queue_mask = radv_image_queue_family_mask(image, src_family, cmd_buffer->queue_family_index); - unsigned dst_queue_mask = radv_image_queue_family_mask(image, dst_family, cmd_buffer->queue_family_index); - - if (image->surface.htile_size) + if (image->htile.size) radv_handle_depth_image_transition(cmd_buffer, image, src_layout, dst_layout, range, pending_clears); if (image->cmask.size) radv_handle_cmask_image_transition(cmd_buffer, image, src_layout, - dst_layout, src_queue_mask, - dst_queue_mask, range, - pending_clears); + dst_layout, range, pending_clears); if (image->surface.dcc_size) radv_handle_dcc_image_transition(cmd_buffer, image, src_layout, - dst_layout, src_queue_mask, - dst_queue_mask, range, - pending_clears); + dst_layout, range, pending_clears); } void radv_CmdPipelineBarrier( @@ -3123,43 +2232,76 @@ void radv_CmdPipelineBarrier( const VkImageMemoryBarrier* pImageMemoryBarriers) { RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); - enum radv_cmd_flush_bits src_flush_bits = 0; - enum radv_cmd_flush_bits dst_flush_bits = 0; - + VkAccessFlags src_flags = 0; + VkAccessFlags dst_flags = 0; + uint32_t b; for (uint32_t i = 0; i < memoryBarrierCount; i++) { - src_flush_bits |= radv_src_access_flush(cmd_buffer, pMemoryBarriers[i].srcAccessMask); - dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pMemoryBarriers[i].dstAccessMask, - NULL); + src_flags |= pMemoryBarriers[i].srcAccessMask; + dst_flags |= pMemoryBarriers[i].dstAccessMask; } for (uint32_t i = 0; i < bufferMemoryBarrierCount; i++) { - src_flush_bits |= radv_src_access_flush(cmd_buffer, pBufferMemoryBarriers[i].srcAccessMask); - dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pBufferMemoryBarriers[i].dstAccessMask, - NULL); + src_flags |= pBufferMemoryBarriers[i].srcAccessMask; + dst_flags |= pBufferMemoryBarriers[i].dstAccessMask; } for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) { RADV_FROM_HANDLE(radv_image, image, pImageMemoryBarriers[i].image); - src_flush_bits |= radv_src_access_flush(cmd_buffer, pImageMemoryBarriers[i].srcAccessMask); - dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pImageMemoryBarriers[i].dstAccessMask, - image); - } - - radv_stage_flush(cmd_buffer, srcStageMask); - cmd_buffer->state.flush_bits |= src_flush_bits; + src_flags |= pImageMemoryBarriers[i].srcAccessMask; + dst_flags |= pImageMemoryBarriers[i].dstAccessMask; - for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) { - RADV_FROM_HANDLE(radv_image, image, pImageMemoryBarriers[i].image); radv_handle_image_transition(cmd_buffer, image, pImageMemoryBarriers[i].oldLayout, pImageMemoryBarriers[i].newLayout, - pImageMemoryBarriers[i].srcQueueFamilyIndex, - pImageMemoryBarriers[i].dstQueueFamilyIndex, - &pImageMemoryBarriers[i].subresourceRange, + pImageMemoryBarriers[i].subresourceRange, 0); } - cmd_buffer->state.flush_bits |= dst_flush_bits; + enum radv_cmd_flush_bits flush_bits = 0; + + for_each_bit(b, src_flags) { + switch ((VkAccessFlagBits)(1 << b)) { + case VK_ACCESS_SHADER_WRITE_BIT: + flush_bits |= RADV_CMD_FLAG_INV_GLOBAL_L2; + break; + case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT: + flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB; + break; + case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT: + flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB; + break; + case VK_ACCESS_TRANSFER_WRITE_BIT: + flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB; + break; + default: + break; + } + } + + for_each_bit(b, dst_flags) { + switch ((VkAccessFlagBits)(1 << b)) { + case VK_ACCESS_INDIRECT_COMMAND_READ_BIT: + case VK_ACCESS_INDEX_READ_BIT: + case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT: + case VK_ACCESS_UNIFORM_READ_BIT: + flush_bits |= RADV_CMD_FLAG_INV_VMEM_L1; + break; + case VK_ACCESS_SHADER_READ_BIT: + flush_bits |= RADV_CMD_FLAG_INV_GLOBAL_L2; + break; + case VK_ACCESS_COLOR_ATTACHMENT_READ_BIT: + case VK_ACCESS_TRANSFER_READ_BIT: + case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT: + flush_bits |= RADV_CMD_FLUSH_AND_INV_FRAMEBUFFER | RADV_CMD_FLAG_INV_GLOBAL_L2; + default: + break; + } + } + + flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | + RADV_CMD_FLAG_PS_PARTIAL_FLUSH; + + cmd_buffer->state.flush_bits |= flush_bits; } @@ -3173,12 +2315,12 @@ static void write_event(struct radv_cmd_buffer *cmd_buffer, cmd_buffer->device->ws->cs_add_buffer(cs, event->bo, 8); - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 12); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 12); /* TODO: this is overkill. Probably should figure something out from * the stage mask. */ - if (cmd_buffer->device->physical_device->rad_info.chip_class == CIK) { + if (cmd_buffer->device->instance->physicalDevice.rad_info.chip_class == CIK) { radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0)); radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_BOTTOM_OF_PIPE_TS) | EVENT_INDEX(5)); @@ -3240,7 +2382,7 @@ void radv_CmdWaitEvents(VkCommandBuffer commandBuffer, cmd_buffer->device->ws->cs_add_buffer(cs, event->bo, 8); - MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 7); + unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 7); radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); radeon_emit(cs, WAIT_REG_MEM_EQUAL | WAIT_REG_MEM_MEM_SPACE(1)); @@ -3260,9 +2402,7 @@ void radv_CmdWaitEvents(VkCommandBuffer commandBuffer, radv_handle_image_transition(cmd_buffer, image, pImageMemoryBarriers[i].oldLayout, pImageMemoryBarriers[i].newLayout, - pImageMemoryBarriers[i].srcQueueFamilyIndex, - pImageMemoryBarriers[i].dstQueueFamilyIndex, - &pImageMemoryBarriers[i].subresourceRange, + pImageMemoryBarriers[i].subresourceRange, 0); } |