1 2 #include "util/u_format.h" 3 4 #include "nvc0_context.h" 5 6 #include "nv50/nv50_defs.xml.h" 7 8 struct nvc0_transfer { 9 struct pipe_transfer base; 10 struct nv50_m2mf_rect rect[2]; 11 uint32_t nblocksx; 12 uint16_t nblocksy; 13 uint16_t nlayers; 14 }; 15 16 static void 17 nvc0_m2mf_transfer_rect(struct nvc0_context *nvc0, 18 const struct nv50_m2mf_rect *dst, 19 const struct nv50_m2mf_rect *src, 20 uint32_t nblocksx, uint32_t nblocksy) 21 { 22 struct nouveau_pushbuf *push = nvc0->base.pushbuf; 23 struct nouveau_bufctx *bctx = nvc0->bufctx; 24 const int cpp = dst->cpp; 25 uint32_t src_ofst = src->base; 26 uint32_t dst_ofst = dst->base; 27 uint32_t height = nblocksy; 28 uint32_t sy = src->y; 29 uint32_t dy = dst->y; 30 uint32_t exec = (1 << 20); 31 32 assert(dst->cpp == src->cpp); 33 34 nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD); 35 nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR); 36 nouveau_pushbuf_bufctx(push, bctx); 37 nouveau_pushbuf_validate(push); 38 39 if (nouveau_bo_memtype(src->bo)) { 40 BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_IN), 5); 41 PUSH_DATA (push, src->tile_mode); 42 PUSH_DATA (push, src->width * cpp); 43 PUSH_DATA (push, src->height); 44 PUSH_DATA (push, src->depth); 45 PUSH_DATA (push, src->z); 46 } else { 47 src_ofst += src->y * src->pitch + src->x * cpp; 48 49 BEGIN_NVC0(push, NVC0_M2MF(PITCH_IN), 1); 50 PUSH_DATA (push, src->width * cpp); 51 52 exec |= NVC0_M2MF_EXEC_LINEAR_IN; 53 } 54 55 if (nouveau_bo_memtype(dst->bo)) { 56 BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_OUT), 5); 57 PUSH_DATA (push, dst->tile_mode); 58 PUSH_DATA (push, dst->width * cpp); 59 PUSH_DATA (push, dst->height); 60 PUSH_DATA (push, dst->depth); 61 PUSH_DATA (push, dst->z); 62 } else { 63 dst_ofst += dst->y * dst->pitch + dst->x * cpp; 64 65 BEGIN_NVC0(push, NVC0_M2MF(PITCH_OUT), 1); 66 PUSH_DATA (push, dst->width * cpp); 67 68 exec |= NVC0_M2MF_EXEC_LINEAR_OUT; 69 } 70 71 while (height) { 72 int line_count = height > 2047 ? 2047 : height; 73 74 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2); 75 PUSH_DATAh(push, src->bo->offset + src_ofst); 76 PUSH_DATA (push, src->bo->offset + src_ofst); 77 78 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2); 79 PUSH_DATAh(push, dst->bo->offset + dst_ofst); 80 PUSH_DATA (push, dst->bo->offset + dst_ofst); 81 82 if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) { 83 BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_IN_X), 2); 84 PUSH_DATA (push, src->x * cpp); 85 PUSH_DATA (push, sy); 86 } else { 87 src_ofst += line_count * src->pitch; 88 } 89 if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) { 90 BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_OUT_X), 2); 91 PUSH_DATA (push, dst->x * cpp); 92 PUSH_DATA (push, dy); 93 } else { 94 dst_ofst += line_count * dst->pitch; 95 } 96 97 BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2); 98 PUSH_DATA (push, nblocksx * cpp); 99 PUSH_DATA (push, line_count); 100 BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1); 101 PUSH_DATA (push, exec); 102 103 height -= line_count; 104 sy += line_count; 105 dy += line_count; 106 } 107 108 nouveau_bufctx_reset(bctx, 0); 109 } 110 111 static void 112 nve4_m2mf_transfer_rect(struct nvc0_context *nvc0, 113 const struct nv50_m2mf_rect *dst, 114 const struct nv50_m2mf_rect *src, 115 uint32_t nblocksx, uint32_t nblocksy) 116 { 117 struct nouveau_pushbuf *push = nvc0->base.pushbuf; 118 struct nouveau_bufctx *bctx = nvc0->bufctx; 119 uint32_t exec; 120 uint32_t src_base = src->base; 121 uint32_t dst_base = dst->base; 122 const int cpp = dst->cpp; 123 124 assert(dst->cpp == src->cpp); 125 126 nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR); 127 nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD); 128 nouveau_pushbuf_bufctx(push, bctx); 129 nouveau_pushbuf_validate(push); 130 131 exec = 0x200 /* 2D_ENABLE */ | 0x6 /* UNK */; 132 133 if (!nouveau_bo_memtype(dst->bo)) { 134 assert(!dst->z); 135 dst_base += dst->y * dst->pitch + dst->x * cpp; 136 exec |= 0x100; /* DST_MODE_2D_LINEAR */ 137 } 138 if (!nouveau_bo_memtype(src->bo)) { 139 assert(!src->z); 140 src_base += src->y * src->pitch + src->x * cpp; 141 exec |= 0x080; /* SRC_MODE_2D_LINEAR */ 142 } 143 144 BEGIN_NVC0(push, SUBC_COPY(0x070c), 6); 145 PUSH_DATA (push, 0x1000 | dst->tile_mode); 146 PUSH_DATA (push, dst->pitch); 147 PUSH_DATA (push, dst->height); 148 PUSH_DATA (push, dst->depth); 149 PUSH_DATA (push, dst->z); 150 PUSH_DATA (push, (dst->y << 16) | (dst->x * cpp)); 151 152 BEGIN_NVC0(push, SUBC_COPY(0x0728), 6); 153 PUSH_DATA (push, 0x1000 | src->tile_mode); 154 PUSH_DATA (push, src->pitch); 155 PUSH_DATA (push, src->height); 156 PUSH_DATA (push, src->depth); 157 PUSH_DATA (push, src->z); 158 PUSH_DATA (push, (src->y << 16) | (src->x * cpp)); 159 160 BEGIN_NVC0(push, SUBC_COPY(0x0400), 8); 161 PUSH_DATAh(push, src->bo->offset + src_base); 162 PUSH_DATA (push, src->bo->offset + src_base); 163 PUSH_DATAh(push, dst->bo->offset + dst_base); 164 PUSH_DATA (push, dst->bo->offset + dst_base); 165 PUSH_DATA (push, src->pitch); 166 PUSH_DATA (push, dst->pitch); 167 PUSH_DATA (push, nblocksx * cpp); 168 PUSH_DATA (push, nblocksy); 169 170 BEGIN_NVC0(push, SUBC_COPY(0x0300), 1); 171 PUSH_DATA (push, exec); 172 173 nouveau_bufctx_reset(bctx, 0); 174 } 175 176 void 177 nvc0_m2mf_push_linear(struct nouveau_context *nv, 178 struct nouveau_bo *dst, unsigned offset, unsigned domain, 179 unsigned size, const void *data) 180 { 181 struct nvc0_context *nvc0 = nvc0_context(&nv->pipe); 182 struct nouveau_pushbuf *push = nv->pushbuf; 183 uint32_t *src = (uint32_t *)data; 184 unsigned count = (size + 3) / 4; 185 186 nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR); 187 nouveau_pushbuf_bufctx(push, nvc0->bufctx); 188 nouveau_pushbuf_validate(push); 189 190 while (count) { 191 unsigned nr; 192 193 if (!PUSH_SPACE(push, 16)) 194 break; 195 nr = PUSH_AVAIL(push); 196 assert(nr >= 16); 197 nr = MIN2(count, nr - 9); 198 nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN); 199 200 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2); 201 PUSH_DATAh(push, dst->offset + offset); 202 PUSH_DATA (push, dst->offset + offset); 203 BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2); 204 PUSH_DATA (push, nr * 4); 205 PUSH_DATA (push, 1); 206 BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1); 207 PUSH_DATA (push, 0x100111); 208 209 /* must not be interrupted (trap on QUERY fence, 0x50 works however) */ 210 BEGIN_NIC0(push, NVC0_M2MF(DATA), nr); 211 PUSH_DATAp(push, src, nr); 212 213 count -= nr; 214 src += nr; 215 offset += nr * 4; 216 } 217 218 nouveau_bufctx_reset(nvc0->bufctx, 0); 219 } 220 221 void 222 nve4_p2mf_push_linear(struct nouveau_context *nv, 223 struct nouveau_bo *dst, unsigned offset, unsigned domain, 224 unsigned size, const void *data) 225 { 226 struct nvc0_context *nvc0 = nvc0_context(&nv->pipe); 227 struct nouveau_pushbuf *push = nv->pushbuf; 228 uint32_t *src = (uint32_t *)data; 229 unsigned count = (size + 3) / 4; 230 231 nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR); 232 nouveau_pushbuf_bufctx(push, nvc0->bufctx); 233 nouveau_pushbuf_validate(push); 234 235 while (count) { 236 unsigned nr; 237 238 if (!PUSH_SPACE(push, 16)) 239 break; 240 nr = PUSH_AVAIL(push); 241 assert(nr >= 16); 242 nr = MIN2(count, nr - 8); 243 nr = MIN2(nr, (NV04_PFIFO_MAX_PACKET_LEN - 1)); 244 245 BEGIN_NVC0(push, NVE4_P2MF(DST_ADDRESS_HIGH), 2); 246 PUSH_DATAh(push, dst->offset + offset); 247 PUSH_DATA (push, dst->offset + offset); 248 BEGIN_NVC0(push, NVE4_P2MF(LINE_LENGTH_IN), 2); 249 PUSH_DATA (push, nr * 4); 250 PUSH_DATA (push, 1); 251 /* must not be interrupted (trap on QUERY fence, 0x50 works however) */ 252 BEGIN_1IC0(push, NVE4_P2MF(EXEC), nr + 1); 253 PUSH_DATA (push, 0x1001); 254 PUSH_DATAp(push, src, nr); 255 256 count -= nr; 257 src += nr; 258 offset += nr * 4; 259 } 260 261 nouveau_bufctx_reset(nvc0->bufctx, 0); 262 } 263 264 static void 265 nvc0_m2mf_copy_linear(struct nouveau_context *nv, 266 struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom, 267 struct nouveau_bo *src, unsigned srcoff, unsigned srcdom, 268 unsigned size) 269 { 270 struct nouveau_pushbuf *push = nv->pushbuf; 271 struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx; 272 273 nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD); 274 nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR); 275 nouveau_pushbuf_bufctx(push, bctx); 276 nouveau_pushbuf_validate(push); 277 278 while (size) { 279 unsigned bytes = MIN2(size, 1 << 17); 280 281 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2); 282 PUSH_DATAh(push, dst->offset + dstoff); 283 PUSH_DATA (push, dst->offset + dstoff); 284 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2); 285 PUSH_DATAh(push, src->offset + srcoff); 286 PUSH_DATA (push, src->offset + srcoff); 287 BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2); 288 PUSH_DATA (push, bytes); 289 PUSH_DATA (push, 1); 290 BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1); 291 PUSH_DATA (push, (1 << NVC0_M2MF_EXEC_INC__SHIFT) | 292 NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT); 293 294 srcoff += bytes; 295 dstoff += bytes; 296 size -= bytes; 297 } 298 299 nouveau_bufctx_reset(bctx, 0); 300 } 301 302 static void 303 nve4_m2mf_copy_linear(struct nouveau_context *nv, 304 struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom, 305 struct nouveau_bo *src, unsigned srcoff, unsigned srcdom, 306 unsigned size) 307 { 308 struct nouveau_pushbuf *push = nv->pushbuf; 309 struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx; 310 311 nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD); 312 nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR); 313 nouveau_pushbuf_bufctx(push, bctx); 314 nouveau_pushbuf_validate(push); 315 316 BEGIN_NVC0(push, SUBC_COPY(0x0400), 4); 317 PUSH_DATAh(push, src->offset + srcoff); 318 PUSH_DATA (push, src->offset + srcoff); 319 PUSH_DATAh(push, dst->offset + dstoff); 320 PUSH_DATA (push, dst->offset + dstoff); 321 BEGIN_NVC0(push, SUBC_COPY(0x0418), 1); 322 PUSH_DATA (push, size); 323 BEGIN_NVC0(push, SUBC_COPY(0x0300), 1); 324 PUSH_DATA (push, 0x186); 325 326 nouveau_bufctx_reset(bctx, 0); 327 } 328 329 struct pipe_transfer * 330 nvc0_miptree_transfer_new(struct pipe_context *pctx, 331 struct pipe_resource *res, 332 unsigned level, 333 unsigned usage, 334 const struct pipe_box *box) 335 { 336 struct nvc0_context *nvc0 = nvc0_context(pctx); 337 struct nouveau_device *dev = nvc0->screen->base.device; 338 struct nv50_miptree *mt = nv50_miptree(res); 339 struct nvc0_transfer *tx; 340 uint32_t size; 341 int ret; 342 343 if (usage & PIPE_TRANSFER_MAP_DIRECTLY) 344 return NULL; 345 346 tx = CALLOC_STRUCT(nvc0_transfer); 347 if (!tx) 348 return NULL; 349 350 pipe_resource_reference(&tx->base.resource, res); 351 352 tx->base.level = level; 353 tx->base.usage = usage; 354 tx->base.box = *box; 355 356 if (util_format_is_plain(res->format)) { 357 tx->nblocksx = box->width << mt->ms_x; 358 tx->nblocksy = box->height << mt->ms_y; 359 } else { 360 tx->nblocksx = util_format_get_nblocksx(res->format, box->width); 361 tx->nblocksy = util_format_get_nblocksy(res->format, box->height); 362 } 363 tx->nlayers = box->depth; 364 365 tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format); 366 tx->base.layer_stride = tx->nblocksy * tx->base.stride; 367 368 nv50_m2mf_rect_setup(&tx->rect[0], res, level, box->x, box->y, box->z); 369 370 size = tx->base.layer_stride; 371 372 ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0, 373 size * tx->nlayers, NULL, &tx->rect[1].bo); 374 if (ret) { 375 FREE(tx); 376 return NULL; 377 } 378 379 tx->rect[1].cpp = tx->rect[0].cpp; 380 tx->rect[1].width = tx->nblocksx; 381 tx->rect[1].height = tx->nblocksy; 382 tx->rect[1].depth = 1; 383 tx->rect[1].pitch = tx->base.stride; 384 tx->rect[1].domain = NOUVEAU_BO_GART; 385 386 if (usage & PIPE_TRANSFER_READ) { 387 unsigned base = tx->rect[0].base; 388 unsigned z = tx->rect[0].z; 389 unsigned i; 390 for (i = 0; i < tx->nlayers; ++i) { 391 nvc0->m2mf_copy_rect(nvc0, &tx->rect[1], &tx->rect[0], 392 tx->nblocksx, tx->nblocksy); 393 if (mt->layout_3d) 394 tx->rect[0].z++; 395 else 396 tx->rect[0].base += mt->layer_stride; 397 tx->rect[1].base += size; 398 } 399 tx->rect[0].z = z; 400 tx->rect[0].base = base; 401 tx->rect[1].base = 0; 402 } 403 404 return &tx->base; 405 } 406 407 void 408 nvc0_miptree_transfer_del(struct pipe_context *pctx, 409 struct pipe_transfer *transfer) 410 { 411 struct nvc0_context *nvc0 = nvc0_context(pctx); 412 struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer; 413 struct nv50_miptree *mt = nv50_miptree(tx->base.resource); 414 unsigned i; 415 416 if (tx->base.usage & PIPE_TRANSFER_WRITE) { 417 for (i = 0; i < tx->nlayers; ++i) { 418 nvc0->m2mf_copy_rect(nvc0, &tx->rect[0], &tx->rect[1], 419 tx->nblocksx, tx->nblocksy); 420 if (mt->layout_3d) 421 tx->rect[0].z++; 422 else 423 tx->rect[0].base += mt->layer_stride; 424 tx->rect[1].base += tx->nblocksy * tx->base.stride; 425 } 426 } 427 428 nouveau_bo_ref(NULL, &tx->rect[1].bo); 429 pipe_resource_reference(&transfer->resource, NULL); 430 431 FREE(tx); 432 } 433 434 void * 435 nvc0_miptree_transfer_map(struct pipe_context *pctx, 436 struct pipe_transfer *transfer) 437 { 438 struct nvc0_context *nvc0 = nvc0_context(pctx); 439 struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer; 440 int ret; 441 unsigned flags = 0; 442 443 if (tx->rect[1].bo->map) 444 return tx->rect[1].bo->map; 445 446 if (transfer->usage & PIPE_TRANSFER_READ) 447 flags = NOUVEAU_BO_RD; 448 if (transfer->usage & PIPE_TRANSFER_WRITE) 449 flags |= NOUVEAU_BO_WR; 450 451 ret = nouveau_bo_map(tx->rect[1].bo, flags, nvc0->screen->base.client); 452 if (ret) 453 return NULL; 454 return tx->rect[1].bo->map; 455 } 456 457 void 458 nvc0_miptree_transfer_unmap(struct pipe_context *pctx, 459 struct pipe_transfer *transfer) 460 { 461 } 462 463 void 464 nvc0_cb_push(struct nouveau_context *nv, 465 struct nouveau_bo *bo, unsigned domain, 466 unsigned base, unsigned size, 467 unsigned offset, unsigned words, const uint32_t *data) 468 { 469 struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx; 470 struct nouveau_pushbuf *push = nv->pushbuf; 471 472 assert(!(offset & 3)); 473 size = align(size, 0x100); 474 475 nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain); 476 nouveau_pushbuf_bufctx(push, bctx); 477 nouveau_pushbuf_validate(push); 478 479 BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3); 480 PUSH_DATA (push, size); 481 PUSH_DATAh(push, bo->offset + base); 482 PUSH_DATA (push, bo->offset + base); 483 484 while (words) { 485 unsigned nr = PUSH_AVAIL(push); 486 nr = MIN2(nr, words); 487 nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1); 488 489 BEGIN_1IC0(push, NVC0_3D(CB_POS), nr + 1); 490 PUSH_DATA (push, offset); 491 PUSH_DATAp(push, data, nr); 492 493 words -= nr; 494 data += nr; 495 offset += nr * 4; 496 } 497 498 nouveau_bufctx_reset(bctx, 0); 499 } 500 501 void 502 nvc0_init_transfer_functions(struct nvc0_context *nvc0) 503 { 504 if (nvc0->screen->base.class_3d >= NVE4_3D_CLASS) { 505 nvc0->m2mf_copy_rect = nve4_m2mf_transfer_rect; 506 nvc0->base.copy_data = nve4_m2mf_copy_linear; 507 nvc0->base.push_data = nve4_p2mf_push_linear; 508 } else { 509 nvc0->m2mf_copy_rect = nvc0_m2mf_transfer_rect; 510 nvc0->base.copy_data = nvc0_m2mf_copy_linear; 511 nvc0->base.push_data = nvc0_m2mf_push_linear; 512 } 513 nvc0->base.push_cb = nvc0_cb_push; 514 } 515
