1 /* 2 * Copyright 2011 Marek Olk <maraeo (at) gmail.com> 3 * Copyright 2015 Advanced Micro Devices, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining 7 * a copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 16 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS 18 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 21 * USE OR OTHER DEALINGS IN THE SOFTWARE. 22 * 23 * The above copyright notice and this permission notice (including the 24 * next paragraph) shall be included in all copies or substantial portions 25 * of the Software. 26 */ 27 28 #ifndef AMDGPU_CS_H 29 #define AMDGPU_CS_H 30 31 #include "amdgpu_bo.h" 32 #include "util/u_memory.h" 33 #include <amdgpu_drm.h> 34 35 struct amdgpu_ctx { 36 struct amdgpu_winsys *ws; 37 amdgpu_context_handle ctx; 38 amdgpu_bo_handle user_fence_bo; 39 uint64_t *user_fence_cpu_address_base; 40 int refcount; 41 unsigned initial_num_total_rejected_cs; 42 unsigned num_rejected_cs; 43 }; 44 45 struct amdgpu_cs_buffer { 46 struct amdgpu_winsys_bo *bo; 47 union { 48 struct { 49 uint64_t priority_usage; 50 } real; 51 struct { 52 uint32_t real_idx; /* index of underlying real BO */ 53 } slab; 54 } u; 55 enum radeon_bo_usage usage; 56 }; 57 58 enum ib_type { 59 IB_MAIN, 60 IB_NUM, 61 }; 62 63 struct amdgpu_ib { 64 struct radeon_winsys_cs base; 65 66 /* A buffer out of which new IBs are allocated. */ 67 struct pb_buffer *big_ib_buffer; 68 uint8_t *ib_mapped; 69 unsigned used_ib_space; 70 unsigned max_ib_size; 71 uint32_t *ptr_ib_size; 72 bool ptr_ib_size_inside_ib; 73 enum ib_type ib_type; 74 }; 75 76 struct amdgpu_cs_context { 77 struct drm_amdgpu_cs_chunk_ib ib[IB_NUM]; 78 79 /* Buffers. */ 80 unsigned max_real_buffers; 81 unsigned num_real_buffers; 82 struct amdgpu_cs_buffer *real_buffers; 83 84 unsigned max_real_submit; 85 amdgpu_bo_handle *handles; 86 uint8_t *flags; 87 88 unsigned num_slab_buffers; 89 unsigned max_slab_buffers; 90 struct amdgpu_cs_buffer *slab_buffers; 91 92 unsigned num_sparse_buffers; 93 unsigned max_sparse_buffers; 94 struct amdgpu_cs_buffer *sparse_buffers; 95 96 int buffer_indices_hashlist[4096]; 97 98 struct amdgpu_winsys_bo *last_added_bo; 99 unsigned last_added_bo_index; 100 unsigned last_added_bo_usage; 101 uint64_t last_added_bo_priority_usage; 102 103 struct pipe_fence_handle **fence_dependencies; 104 unsigned num_fence_dependencies; 105 unsigned max_fence_dependencies; 106 107 struct pipe_fence_handle *fence; 108 109 /* the error returned from cs_flush for non-async submissions */ 110 int error_code; 111 }; 112 113 struct amdgpu_cs { 114 struct amdgpu_ib main; /* must be first because this is inherited */ 115 struct amdgpu_ctx *ctx; 116 enum ring_type ring_type; 117 struct drm_amdgpu_cs_chunk_fence fence_chunk; 118 119 /* We flip between these two CS. While one is being consumed 120 * by the kernel in another thread, the other one is being filled 121 * by the pipe driver. */ 122 struct amdgpu_cs_context csc1; 123 struct amdgpu_cs_context csc2; 124 /* The currently-used CS. */ 125 struct amdgpu_cs_context *csc; 126 /* The CS being currently-owned by the other thread. */ 127 struct amdgpu_cs_context *cst; 128 129 /* Flush CS. */ 130 void (*flush_cs)(void *ctx, unsigned flags, struct pipe_fence_handle **fence); 131 void *flush_data; 132 133 struct util_queue_fence flush_completed; 134 struct pipe_fence_handle *next_fence; 135 }; 136 137 struct amdgpu_fence { 138 struct pipe_reference reference; 139 /* If ctx == NULL, this fence is syncobj-based. */ 140 uint32_t syncobj; 141 142 struct amdgpu_winsys *ws; 143 struct amdgpu_ctx *ctx; /* submission context */ 144 struct amdgpu_cs_fence fence; 145 uint64_t *user_fence_cpu_address; 146 147 /* If the fence has been submitted. This is unsignalled for deferred fences 148 * (cs->next_fence) and while an IB is still being submitted in the submit 149 * thread. */ 150 struct util_queue_fence submitted; 151 152 volatile int signalled; /* bool (int for atomicity) */ 153 }; 154 155 static inline bool amdgpu_fence_is_syncobj(struct amdgpu_fence *fence) 156 { 157 return fence->ctx == NULL; 158 } 159 160 static inline void amdgpu_ctx_unref(struct amdgpu_ctx *ctx) 161 { 162 if (p_atomic_dec_zero(&ctx->refcount)) { 163 amdgpu_cs_ctx_free(ctx->ctx); 164 amdgpu_bo_free(ctx->user_fence_bo); 165 FREE(ctx); 166 } 167 } 168 169 static inline void amdgpu_fence_reference(struct pipe_fence_handle **dst, 170 struct pipe_fence_handle *src) 171 { 172 struct amdgpu_fence **rdst = (struct amdgpu_fence **)dst; 173 struct amdgpu_fence *rsrc = (struct amdgpu_fence *)src; 174 175 if (pipe_reference(&(*rdst)->reference, &rsrc->reference)) { 176 struct amdgpu_fence *fence = *rdst; 177 178 if (amdgpu_fence_is_syncobj(fence)) 179 amdgpu_cs_destroy_syncobj(fence->ws->dev, fence->syncobj); 180 else 181 amdgpu_ctx_unref(fence->ctx); 182 183 util_queue_fence_destroy(&fence->submitted); 184 FREE(fence); 185 } 186 *rdst = rsrc; 187 } 188 189 int amdgpu_lookup_buffer(struct amdgpu_cs_context *cs, struct amdgpu_winsys_bo *bo); 190 191 static inline struct amdgpu_ib * 192 amdgpu_ib(struct radeon_winsys_cs *base) 193 { 194 return (struct amdgpu_ib *)base; 195 } 196 197 static inline struct amdgpu_cs * 198 amdgpu_cs(struct radeon_winsys_cs *base) 199 { 200 assert(amdgpu_ib(base)->ib_type == IB_MAIN); 201 return (struct amdgpu_cs*)base; 202 } 203 204 #define get_container(member_ptr, container_type, container_member) \ 205 (container_type *)((char *)(member_ptr) - offsetof(container_type, container_member)) 206 207 static inline struct amdgpu_cs * 208 amdgpu_cs_from_ib(struct amdgpu_ib *ib) 209 { 210 switch (ib->ib_type) { 211 case IB_MAIN: 212 return get_container(ib, struct amdgpu_cs, main); 213 default: 214 unreachable("bad ib_type"); 215 } 216 } 217 218 static inline bool 219 amdgpu_bo_is_referenced_by_cs(struct amdgpu_cs *cs, 220 struct amdgpu_winsys_bo *bo) 221 { 222 int num_refs = bo->num_cs_references; 223 return num_refs == bo->ws->num_cs || 224 (num_refs && amdgpu_lookup_buffer(cs->csc, bo) != -1); 225 } 226 227 static inline bool 228 amdgpu_bo_is_referenced_by_cs_with_usage(struct amdgpu_cs *cs, 229 struct amdgpu_winsys_bo *bo, 230 enum radeon_bo_usage usage) 231 { 232 int index; 233 struct amdgpu_cs_buffer *buffer; 234 235 if (!bo->num_cs_references) 236 return false; 237 238 index = amdgpu_lookup_buffer(cs->csc, bo); 239 if (index == -1) 240 return false; 241 242 buffer = bo->bo ? &cs->csc->real_buffers[index] : 243 bo->sparse ? &cs->csc->sparse_buffers[index] : 244 &cs->csc->slab_buffers[index]; 245 246 return (buffer->usage & usage) != 0; 247 } 248 249 static inline bool 250 amdgpu_bo_is_referenced_by_any_cs(struct amdgpu_winsys_bo *bo) 251 { 252 return bo->num_cs_references != 0; 253 } 254 255 bool amdgpu_fence_wait(struct pipe_fence_handle *fence, uint64_t timeout, 256 bool absolute); 257 void amdgpu_add_fences(struct amdgpu_winsys_bo *bo, 258 unsigned num_fences, 259 struct pipe_fence_handle **fences); 260 void amdgpu_cs_sync_flush(struct radeon_winsys_cs *rcs); 261 void amdgpu_cs_init_functions(struct amdgpu_winsys *ws); 262 void amdgpu_cs_submit_ib(void *job, int thread_index); 263 264 #endif 265
