dri/i965/brw_binding_tables.c

/*
 * Copyright  2013 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/**
 * \file brw_binding_tables.c
 *
 * State atoms which upload the "binding table" for each shader stage.
 *
 * Binding tables map a numeric "surface index" to the SURFACE_STATE structure
 * for a currently bound surface.  This allows SEND messages (such as sampler
 * or data port messages) to refer to a particular surface by number, rather
 * than by pointer.
 *
 * The binding table is stored as a (sparse) array of SURFACE_STATE entries;
 * surface indexes are simply indexes into the array.  The ordering of the
 * entries is entirely left up to software; see the SURF_INDEX_* macros in
 * brw_context.h to see our current layout.
 */

#include "main/mtypes.h"

#include "brw_context.h"
#include "brw_defines.h"
#include "brw_state.h"
#include "intel_batchbuffer.h"

static const GLuint stage_to_bt_edit[] = {
   [MESA_SHADER_VERTEX] = _3DSTATE_BINDING_TABLE_EDIT_VS,
   [MESA_SHADER_GEOMETRY] = _3DSTATE_BINDING_TABLE_EDIT_GS,
   [MESA_SHADER_FRAGMENT] = _3DSTATE_BINDING_TABLE_EDIT_PS,
};

static uint32_t
reserve_hw_bt_space(struct brw_context *brw, unsigned bytes)
{
   /* From the Broadwell PRM, Volume 16, "Workarounds",
    * WaStateBindingTableOverfetch:
    * "HW over-fetches two cache lines of binding table indices.  When
    *  using the resource streamer, SW needs to pad binding table pointer
    *  updates with an additional two cache lines."
    *
    * Cache lines are 64 bytes, so we subtract 128 bytes from the size of
    * the binding table pool buffer.
    */
   if (brw->hw_bt_pool.next_offset + bytes >= brw->hw_bt_pool.bo->size - 128) {
      gen7_reset_hw_bt_pool_offsets(brw);
   }

   uint32_t offset = brw->hw_bt_pool.next_offset;

   /* From the Haswell PRM, Volume 2b: Command Reference: Instructions,
    * 3DSTATE_BINDING_TABLE_POINTERS_xS:
    *
    * "If HW Binding Table is enabled, the offset is relative to the
    *  Binding Table Pool Base Address and the alignment is 64 bytes."
    */
   brw->hw_bt_pool.next_offset += ALIGN(bytes, 64);

   return offset;
}

/**
 * Upload a shader stage's binding table as indirect state.
 *
 * This copies brw_stage_state::surf_offset[] into the indirect state section
 * of the batchbuffer (allocated by brw_state_batch()).
 */
void
brw_upload_binding_table(struct brw_context *brw,
                         uint32_t packet_name,
                         const struct brw_stage_prog_data *prog_data,
                         struct brw_stage_state *stage_state)
{
   if (prog_data->binding_table.size_bytes == 0) {
      /* There are no surfaces; skip making the binding table altogether. */
      if (stage_state->bind_bo_offset == 0 && brw->gen < 9)
         return;

      stage_state->bind_bo_offset = 0;
   } else {
      /* Upload a new binding table. */
      if (INTEL_DEBUG & DEBUG_SHADER_TIME) {
         brw_emit_buffer_surface_state(
            brw, &stage_state->surf_offset[
                    prog_data->binding_table.shader_time_start],
            brw->shader_time.bo, 0, BRW_SURFACEFORMAT_RAW,
            brw->shader_time.bo->size, 1, true);
      }
      /* When RS is enabled use hw-binding table uploads, otherwise fallback to
       * software-uploads.
       */
      if (brw->use_resource_streamer) {
         gen7_update_binding_table_from_array(brw, stage_state->stage,
                                              stage_state->surf_offset,
                                              prog_data->binding_table
                                              .size_bytes / 4);
      } else {
         uint32_t *bind = brw_state_batch(brw, AUB_TRACE_BINDING_TABLE,
                                          prog_data->binding_table.size_bytes,
                                          32,
                                          &stage_state->bind_bo_offset);

         /* BRW_NEW_SURFACES and BRW_NEW_*_CONSTBUF */
         memcpy(bind, stage_state->surf_offset,
                prog_data->binding_table.size_bytes);
      }
   }

   brw->ctx.NewDriverState |= BRW_NEW_BINDING_TABLE_POINTERS;

   if (brw->gen >= 7) {
      if (brw->use_resource_streamer) {
         stage_state->bind_bo_offset =
            reserve_hw_bt_space(brw, prog_data->binding_table.size_bytes);
      }
      BEGIN_BATCH(2);
      OUT_BATCH(packet_name << 16 | (2 - 2));
      /* Align SurfaceStateOffset[16:6] format to [15:5] PS Binding Table field
       * when hw-generated binding table is enabled.
       */
      OUT_BATCH(brw->use_resource_streamer ?
                (stage_state->bind_bo_offset >> 1) :
                stage_state->bind_bo_offset);
      ADVANCE_BATCH();
   }
}

/**
 * State atoms which upload the binding table for a particular shader stage.
 *  @{
 */

/** Upload the VS binding table. */
static void
brw_vs_upload_binding_table(struct brw_context *brw)
{
   /* BRW_NEW_VS_PROG_DATA */
   const struct brw_stage_prog_data *prog_data = brw->vs.base.prog_data;
   brw_upload_binding_table(brw,
                            _3DSTATE_BINDING_TABLE_POINTERS_VS,
                            prog_data,
                            &brw->vs.base);
}

const struct brw_tracked_state brw_vs_binding_table = {
   .dirty = {
      .mesa = 0,
      .brw = BRW_NEW_BATCH |
             BRW_NEW_BLORP |
             BRW_NEW_VS_CONSTBUF |
             BRW_NEW_VS_PROG_DATA |
             BRW_NEW_SURFACES,
   },
   .emit = brw_vs_upload_binding_table,
};


/** Upload the PS binding table. */
static void
brw_upload_wm_binding_table(struct brw_context *brw)
{
   /* BRW_NEW_FS_PROG_DATA */
   const struct brw_stage_prog_data *prog_data = brw->wm.base.prog_data;
   brw_upload_binding_table(brw,
                            _3DSTATE_BINDING_TABLE_POINTERS_PS,
                            prog_data,
                            &brw->wm.base);
}

const struct brw_tracked_state brw_wm_binding_table = {
   .dirty = {
      .mesa = 0,
      .brw = BRW_NEW_BATCH |
             BRW_NEW_BLORP |
             BRW_NEW_FS_PROG_DATA |
             BRW_NEW_SURFACES,
   },
   .emit = brw_upload_wm_binding_table,
};

/** Upload the TCS binding table (if tessellation stages are active). */
static void
brw_tcs_upload_binding_table(struct brw_context *brw)
{
   /* Skip if the tessellation stages are disabled. */
   if (brw->tess_eval_program == NULL)
      return;

   /* BRW_NEW_TCS_PROG_DATA */
   const struct brw_stage_prog_data *prog_data = brw->tcs.base.prog_data;
   brw_upload_binding_table(brw,
                            _3DSTATE_BINDING_TABLE_POINTERS_HS,
                            prog_data,
                            &brw->tcs.base);
}

const struct brw_tracked_state brw_tcs_binding_table = {
   .dirty = {
      .mesa = 0,
      .brw = BRW_NEW_BATCH |
             BRW_NEW_BLORP |
             BRW_NEW_DEFAULT_TESS_LEVELS |
             BRW_NEW_SURFACES |
             BRW_NEW_TCS_CONSTBUF |
             BRW_NEW_TCS_PROG_DATA,
   },
   .emit = brw_tcs_upload_binding_table,
};

/** Upload the TES binding table (if TES is active). */
static void
brw_tes_upload_binding_table(struct brw_context *brw)
{
   /* If there's no TES, skip changing anything. */
   if (brw->tess_eval_program == NULL)
      return;

   /* BRW_NEW_TES_PROG_DATA */
   const struct brw_stage_prog_data *prog_data = brw->tes.base.prog_data;
   brw_upload_binding_table(brw,
                            _3DSTATE_BINDING_TABLE_POINTERS_DS,
                            prog_data,
                            &brw->tes.base);
}

const struct brw_tracked_state brw_tes_binding_table = {
   .dirty = {
      .mesa = 0,
      .brw = BRW_NEW_BATCH |
             BRW_NEW_BLORP |
             BRW_NEW_SURFACES |
             BRW_NEW_TES_CONSTBUF |
             BRW_NEW_TES_PROG_DATA,
   },
   .emit = brw_tes_upload_binding_table,
};

/** Upload the GS binding table (if GS is active). */
static void
brw_gs_upload_binding_table(struct brw_context *brw)
{
   /* If there's no GS, skip changing anything. */
   if (brw->geometry_program == NULL)
      return;

   /* BRW_NEW_GS_PROG_DATA */
   const struct brw_stage_prog_data *prog_data = brw->gs.base.prog_data;
   brw_upload_binding_table(brw,
                            _3DSTATE_BINDING_TABLE_POINTERS_GS,
                            prog_data,
                            &brw->gs.base);
}

const struct brw_tracked_state brw_gs_binding_table = {
   .dirty = {
      .mesa = 0,
      .brw = BRW_NEW_BATCH |
             BRW_NEW_BLORP |
             BRW_NEW_GS_CONSTBUF |
             BRW_NEW_GS_PROG_DATA |
             BRW_NEW_SURFACES,
   },
   .emit = brw_gs_upload_binding_table,
};

/**
 * Edit a single entry in a hardware-generated binding table
 */
void
gen7_edit_hw_binding_table_entry(struct brw_context *brw,
                                 gl_shader_stage stage,
                                 uint32_t index,
                                 uint32_t surf_offset)
{
   assert(stage < ARRAY_SIZE(stage_to_bt_edit));
   assert(stage_to_bt_edit[stage]);

   uint32_t dw2 = SET_FIELD(index, BRW_BINDING_TABLE_INDEX) |
      (brw->gen >= 8 ? GEN8_SURFACE_STATE_EDIT(surf_offset) :
       HSW_SURFACE_STATE_EDIT(surf_offset));

   BEGIN_BATCH(3);
   OUT_BATCH(stage_to_bt_edit[stage] << 16 | (3 - 2));
   OUT_BATCH(BRW_BINDING_TABLE_EDIT_TARGET_ALL);
   OUT_BATCH(dw2);
   ADVANCE_BATCH();
}

/**
 * Upload a whole hardware binding table for the given stage.
 *
 * Takes an array of surface offsets and the number of binding table
 * entries.
 */
void
gen7_update_binding_table_from_array(struct brw_context *brw,
                                     gl_shader_stage stage,
                                     const uint32_t* binding_table,
                                     int num_surfaces)
{
   uint32_t dw2 = 0;

   assert(stage < ARRAY_SIZE(stage_to_bt_edit));
   assert(stage_to_bt_edit[stage]);

   BEGIN_BATCH(num_surfaces + 2);
   OUT_BATCH(stage_to_bt_edit[stage] << 16 | num_surfaces);
   OUT_BATCH(BRW_BINDING_TABLE_EDIT_TARGET_ALL);
   for (int i = 0; i < num_surfaces; i++) {
      dw2 = SET_FIELD(i, BRW_BINDING_TABLE_INDEX) |
         (brw->gen >= 8 ? GEN8_SURFACE_STATE_EDIT(binding_table[i]) :
          HSW_SURFACE_STATE_EDIT(binding_table[i]));
      OUT_BATCH(dw2);
   }
   ADVANCE_BATCH();
}

/**
 * Disable hardware binding table support, falling back to the
 * older software-generated binding table mechanism.
 */
void
gen7_disable_hw_binding_tables(struct brw_context *brw)
{
   if (!brw->use_resource_streamer)
      return;
   /* From the Haswell PRM, Volume 7: 3D Media GPGPU,
    * 3DSTATE_BINDING_TABLE_POOL_ALLOC > Programming Note:
    *
    * "When switching between HW and SW binding table generation, SW must
    * issue a state cache invalidate."
    */
   brw_emit_pipe_control_flush(brw, PIPE_CONTROL_STATE_CACHE_INVALIDATE);

   int pkt_len = brw->gen >= 8 ? 4 : 3;

   BEGIN_BATCH(pkt_len);
   OUT_BATCH(_3DSTATE_BINDING_TABLE_POOL_ALLOC << 16 | (pkt_len - 2));
   if (brw->gen >= 8) {
      OUT_BATCH(0);
      OUT_BATCH(0);
      OUT_BATCH(0);
   } else {
      OUT_BATCH(HSW_BT_POOL_ALLOC_MUST_BE_ONE);
      OUT_BATCH(0);
   }
   ADVANCE_BATCH();
}

/**
 * Enable hardware binding tables and set up the binding table pool.
 */
void
gen7_enable_hw_binding_tables(struct brw_context *brw)
{
   if (!brw->use_resource_streamer)
      return;

   if (!brw->hw_bt_pool.bo) {
      /* We use a single re-usable buffer object for the lifetime of the
       * context and size it to maximum allowed binding tables that can be
       * programmed per batch:
       *
       * From the Haswell PRM, Volume 7: 3D Media GPGPU,
       * 3DSTATE_BINDING_TABLE_POOL_ALLOC > Programming Note:
       * "A maximum of 16,383 Binding tables are allowed in any batch buffer"
       */
      static const int max_size = 16383 * 4;
      brw->hw_bt_pool.bo = drm_intel_bo_alloc(brw->bufmgr, "hw_bt",
                                              max_size, 64);
      brw->hw_bt_pool.next_offset = 0;
   }

   /* From the Haswell PRM, Volume 7: 3D Media GPGPU,
    * 3DSTATE_BINDING_TABLE_POOL_ALLOC > Programming Note:
    *
    * "When switching between HW and SW binding table generation, SW must
    * issue a state cache invalidate."
    */
   brw_emit_pipe_control_flush(brw, PIPE_CONTROL_STATE_CACHE_INVALIDATE);

   int pkt_len = brw->gen >= 8 ? 4 : 3;
   uint32_t dw1 = BRW_HW_BINDING_TABLE_ENABLE;
   if (brw->is_haswell) {
      dw1 |= SET_FIELD(GEN7_MOCS_L3, GEN7_HW_BT_POOL_MOCS) |
             HSW_BT_POOL_ALLOC_MUST_BE_ONE;
   } else if (brw->gen >= 8) {
      dw1 |= BDW_MOCS_WB;
   }

   BEGIN_BATCH(pkt_len);
   OUT_BATCH(_3DSTATE_BINDING_TABLE_POOL_ALLOC << 16 | (pkt_len - 2));
   if (brw->gen >= 8) {
      OUT_RELOC64(brw->hw_bt_pool.bo, I915_GEM_DOMAIN_SAMPLER, 0, dw1);
      OUT_BATCH(brw->hw_bt_pool.bo->size);
   } else {
      OUT_RELOC(brw->hw_bt_pool.bo, I915_GEM_DOMAIN_SAMPLER, 0, dw1);
      OUT_RELOC(brw->hw_bt_pool.bo, I915_GEM_DOMAIN_SAMPLER, 0,
             brw->hw_bt_pool.bo->size);
   }
   ADVANCE_BATCH();
}

void
gen7_reset_hw_bt_pool_offsets(struct brw_context *brw)
{
   brw->hw_bt_pool.next_offset = 0;
}

const struct brw_tracked_state gen7_hw_binding_tables = {
   .dirty = {
      .mesa = 0,
      .brw = BRW_NEW_BATCH |
             BRW_NEW_BLORP,
   },
   .emit = gen7_enable_hw_binding_tables
};

/** @} */

/**
 * State atoms which emit 3DSTATE packets to update the binding table pointers.
 *  @{
 */

/**
 * (Gen4-5) Upload the binding table pointers for all shader stages.
 *
 * The binding table pointers are relative to the surface state base address,
 * which points at the batchbuffer containing the streamed batch state.
 */
static void
gen4_upload_binding_table_pointers(struct brw_context *brw)
{
   BEGIN_BATCH(6);
   OUT_BATCH(_3DSTATE_BINDING_TABLE_POINTERS << 16 | (6 - 2));
   OUT_BATCH(brw->vs.base.bind_bo_offset);
   OUT_BATCH(0); /* gs */
   OUT_BATCH(0); /* clip */
   OUT_BATCH(0); /* sf */
   OUT_BATCH(brw->wm.base.bind_bo_offset);
   ADVANCE_BATCH();
}

const struct brw_tracked_state brw_binding_table_pointers = {
   .dirty = {
      .mesa = 0,
      .brw = BRW_NEW_BATCH |
             BRW_NEW_BLORP |
             BRW_NEW_BINDING_TABLE_POINTERS |
             BRW_NEW_STATE_BASE_ADDRESS,
   },
   .emit = gen4_upload_binding_table_pointers,
};

/**
 * (Sandybridge Only) Upload the binding table pointers for all shader stages.
 *
 * The binding table pointers are relative to the surface state base address,
 * which points at the batchbuffer containing the streamed batch state.
 */
static void
gen6_upload_binding_table_pointers(struct brw_context *brw)
{
   BEGIN_BATCH(4);
   OUT_BATCH(_3DSTATE_BINDING_TABLE_POINTERS << 16 |
             GEN6_BINDING_TABLE_MODIFY_VS |
             GEN6_BINDING_TABLE_MODIFY_GS |
             GEN6_BINDING_TABLE_MODIFY_PS |
             (4 - 2));
   OUT_BATCH(brw->vs.base.bind_bo_offset); /* vs */
   if (brw->ff_gs.prog_active)
      OUT_BATCH(brw->ff_gs.bind_bo_offset); /* gs */
   else
      OUT_BATCH(brw->gs.base.bind_bo_offset); /* gs */
   OUT_BATCH(brw->wm.base.bind_bo_offset); /* wm/ps */
   ADVANCE_BATCH();
}

const struct brw_tracked_state gen6_binding_table_pointers = {
   .dirty = {
      .mesa = 0,
      .brw = BRW_NEW_BATCH |
             BRW_NEW_BLORP |
             BRW_NEW_BINDING_TABLE_POINTERS |
             BRW_NEW_STATE_BASE_ADDRESS,
   },
   .emit = gen6_upload_binding_table_pointers,
};

/** @} */