Home | History | Annotate | Download | only in driver 
      1 /*
      2  * Copyright (C) 2011-2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include "rsdCore.h"
     18 #include "rsdBcc.h"
     19 #include "rsdRuntime.h"
     20 #include "rsdAllocation.h"
     21 #include "rsdIntrinsics.h"
     22 
     23 #include "rsContext.h"
     24 #include "rsElement.h"
     25 #include "rsScriptC.h"
     26 
     27 #include "utils/Vector.h"
     28 #include "utils/Timers.h"
     29 #include "utils/StopWatch.h"
     30 #include "utils/String8.h"
     31 
     32 #include <dlfcn.h>
     33 #include <stdio.h>
     34 #include <string.h>
     35 
     36 using namespace android;
     37 using namespace android::renderscript;
     38 
     39 #define MAXLINE 500
     40 #define MAKE_STR_HELPER(S) #S
     41 #define MAKE_STR(S) MAKE_STR_HELPER(S)
     42 #define EXPORT_VAR_STR "exportVarCount: "
     43 #define EXPORT_VAR_STR_LEN strlen(EXPORT_VAR_STR)
     44 #define EXPORT_FUNC_STR "exportFuncCount: "
     45 #define EXPORT_FUNC_STR_LEN strlen(EXPORT_FUNC_STR)
     46 #define EXPORT_FOREACH_STR "exportForEachCount: "
     47 #define EXPORT_FOREACH_STR_LEN strlen(EXPORT_FOREACH_STR)
     48 #define OBJECT_SLOT_STR "objectSlotCount: "
     49 #define OBJECT_SLOT_STR_LEN strlen(OBJECT_SLOT_STR)
     50 
     51 
     52 
     53 static Script * setTLS(Script *sc) {
     54     ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(rsdgThreadTLSKey);
     55     rsAssert(tls);
     56     Script *old = tls->mScript;
     57     tls->mScript = sc;
     58     return old;
     59 }
     60 
     61 
     62 bool rsdScriptInit(const Context *rsc,
     63                      ScriptC *script,
     64                      char const *resName,
     65                      char const *cacheDir,
     66                      uint8_t const *bitcode,
     67                      size_t bitcodeSize,
     68                      uint32_t flags) {
     69     //ALOGE("rsdScriptCreate %p %p %p %p %i %i %p", rsc, resName, cacheDir, bitcode, bitcodeSize, flags, lookupFunc);
     70     //ALOGE("rsdScriptInit %p %p", rsc, script);
     71 
     72     pthread_mutex_lock(&rsdgInitMutex);
     73 
     74     String8 scriptSOName(cacheDir);
     75     scriptSOName = scriptSOName.getPathDir();
     76     scriptSOName.appendPath("lib");
     77     scriptSOName.append("/lib");
     78     scriptSOName.append(resName);
     79     scriptSOName.append(".so");
     80 
     81     String8 scriptInfoName(cacheDir);
     82     scriptInfoName = scriptInfoName.getPathDir();
     83     scriptInfoName.appendPath("lib/");
     84     scriptInfoName.append(resName);
     85     scriptInfoName.append(".bcinfo");
     86 
     87     void *scriptSO = NULL;
     88     FILE *fp = NULL;
     89     DrvScript *drv = (DrvScript *)calloc(1, sizeof(DrvScript));
     90     if (drv == NULL) {
     91         goto error;
     92     }
     93     script->mHal.drv = drv;
     94 
     95     ALOGE("Opening up info object: %s", scriptInfoName.string());
     96 
     97     fp = fopen(scriptInfoName.string(), "r");
     98     if (!fp) {
     99         ALOGE("Unable to open info file: %s", scriptInfoName.string());
    100         goto error;
    101     }
    102 
    103     ALOGE("Opening up shared object: %s", scriptSOName.string());
    104     scriptSO = dlopen(scriptSOName.string(), RTLD_NOW | RTLD_LOCAL);
    105     if (scriptSO == NULL) {
    106         ALOGE("Unable to open shared library (%s): %s",
    107               scriptSOName.string(), dlerror());
    108         goto error;
    109     }
    110     drv->mScriptSO = scriptSO;
    111 
    112     if (scriptSO) {
    113         char line[MAXLINE];
    114         drv->mScriptSO = scriptSO;
    115         drv->mRoot = (RootFunc_t) dlsym(scriptSO, "root");
    116         if (drv->mRoot) {
    117             ALOGE("Found root(): %p", drv->mRoot);
    118         }
    119         drv->mRootExpand = (RootFunc_t) dlsym(scriptSO, "root.expand");
    120         if (drv->mRootExpand) {
    121             ALOGE("Found root.expand(): %p", drv->mRootExpand);
    122         }
    123         drv->mInit = (InvokeFunc_t) dlsym(scriptSO, "init");
    124         if (drv->mInit) {
    125             ALOGE("Found init(): %p", drv->mInit);
    126         }
    127         drv->mFreeChildren = (InvokeFunc_t) dlsym(scriptSO, ".rs.dtor");
    128         if (drv->mFreeChildren) {
    129             ALOGE("Found .rs.dtor(): %p", drv->mFreeChildren);
    130         }
    131 
    132         size_t varCount = 0;
    133         if (fgets(line, MAXLINE, fp) == NULL) {
    134             goto error;
    135         }
    136         if (sscanf(line, EXPORT_VAR_STR "%zu", &varCount) != 1) {
    137             ALOGE("Invalid export var count!: %s", line);
    138             goto error;
    139         }
    140 
    141         script->mHal.info.exportedVariableCount = varCount;
    142         ALOGE("varCount: %zu", varCount);
    143         if (varCount > 0) {
    144             // Start by creating/zeroing this member, since we don't want to
    145             // accidentally clean up invalid pointers later (if we error out).
    146             drv->mFieldIsObject = new bool[varCount];
    147             if (drv->mFieldIsObject == NULL) {
    148                 goto error;
    149             }
    150             memset(drv->mFieldIsObject, 0,
    151                    varCount * sizeof(*drv->mFieldIsObject));
    152             drv->mFieldAddress = new void*[varCount];
    153             if (drv->mFieldAddress == NULL) {
    154                 goto error;
    155             }
    156             for (size_t i = 0; i < varCount; ++i) {
    157                 if (fgets(line, MAXLINE, fp) == NULL) {
    158                     goto error;
    159                 }
    160                 char *c = strrchr(line, '\n');
    161                 if (c) {
    162                     *c = '\0';
    163                 }
    164                 drv->mFieldAddress[i] = dlsym(scriptSO, line);
    165                 if (drv->mFieldAddress[i] == NULL) {
    166                     ALOGE("Failed to find variable address for %s: %s",
    167                           line, dlerror());
    168                     // Not a critical error if we don't find a global variable.
    169                 }
    170                 else {
    171                     ALOGE("Found variable %s at %p", line,
    172                           drv->mFieldAddress[i]);
    173                 }
    174             }
    175         }
    176 
    177         size_t funcCount = 0;
    178         if (fgets(line, MAXLINE, fp) == NULL) {
    179             goto error;
    180         }
    181         if (sscanf(line, EXPORT_FUNC_STR "%zu", &funcCount) != 1) {
    182             ALOGE("Invalid export func count!: %s", line);
    183             goto error;
    184         }
    185 
    186         script->mHal.info.exportedFunctionCount = funcCount;
    187         ALOGE("funcCount: %zu", funcCount);
    188 
    189         if (funcCount > 0) {
    190             drv->mInvokeFunctions = new InvokeFunc_t[funcCount];
    191             if (drv->mInvokeFunctions == NULL) {
    192                 goto error;
    193             }
    194             for (size_t i = 0; i < funcCount; ++i) {
    195                 if (fgets(line, MAXLINE, fp) == NULL) {
    196                     goto error;
    197                 }
    198                 char *c = strrchr(line, '\n');
    199                 if (c) {
    200                     *c = '\0';
    201                 }
    202 
    203                 drv->mInvokeFunctions[i] =
    204                         (InvokeFunc_t) dlsym(scriptSO, line);
    205                 if (drv->mInvokeFunctions[i] == NULL) {
    206                     ALOGE("Failed to get function address for %s(): %s",
    207                           line, dlerror());
    208                     goto error;
    209                 }
    210                 else {
    211                     ALOGE("Found InvokeFunc_t %s at %p", line,
    212                           drv->mInvokeFunctions[i]);
    213                 }
    214             }
    215         }
    216 
    217         size_t forEachCount = 0;
    218         if (fgets(line, MAXLINE, fp) == NULL) {
    219             goto error;
    220         }
    221         if (sscanf(line, EXPORT_FOREACH_STR "%zu", &forEachCount) != 1) {
    222             ALOGE("Invalid export forEach count!: %s", line);
    223             goto error;
    224         }
    225 
    226         if (forEachCount > 0) {
    227 
    228             drv->mForEachSignatures = new uint32_t[forEachCount];
    229             if (drv->mForEachSignatures == NULL) {
    230                 goto error;
    231             }
    232             drv->mForEachFunctions = new ForEachFunc_t[forEachCount];
    233             if (drv->mForEachFunctions == NULL) {
    234                 goto error;
    235             }
    236             for (size_t i = 0; i < forEachCount; ++i) {
    237                 unsigned int tmpSig = 0;
    238                 char tmpName[MAXLINE];
    239 
    240                 if (fgets(line, MAXLINE, fp) == NULL) {
    241                     goto error;
    242                 }
    243                 if (sscanf(line, "%u - %" MAKE_STR(MAXLINE) "s",
    244                            &tmpSig, tmpName) != 2) {
    245                     ALOGE("Invalid export forEach!: %s", line);
    246                     goto error;
    247                 }
    248 
    249                 drv->mForEachSignatures[i] = tmpSig;
    250                 drv->mForEachFunctions[i] =
    251                         (ForEachFunc_t) dlsym(scriptSO, tmpName);
    252                 if (drv->mForEachFunctions[i] == NULL) {
    253                     ALOGE("Failed to find forEach function address for %s: %s",
    254                           tmpName, dlerror());
    255                     goto error;
    256                 }
    257                 else {
    258                     // TODO - Maybe add ForEachExpandPass to .so creation and
    259                     // then lookup the ".expand" version of these kernels
    260                     // instead.
    261                     ALOGE("Found forEach %s at %p", tmpName,
    262                           drv->mForEachFunctions[i]);
    263                 }
    264             }
    265         }
    266 
    267         size_t objectSlotCount = 0;
    268         if (fgets(line, MAXLINE, fp) == NULL) {
    269             goto error;
    270         }
    271         if (sscanf(line, OBJECT_SLOT_STR "%zu", &objectSlotCount) != 1) {
    272             ALOGE("Invalid object slot count!: %s", line);
    273             goto error;
    274         }
    275 
    276         if (objectSlotCount > 0) {
    277             rsAssert(varCount > 0);
    278             for (size_t i = 0; i < objectSlotCount; ++i) {
    279                 uint32_t varNum = 0;
    280                 if (fgets(line, MAXLINE, fp) == NULL) {
    281                     goto error;
    282                 }
    283                 if (sscanf(line, "%u", &varNum) != 1) {
    284                     ALOGE("Invalid object slot!: %s", line);
    285                     goto error;
    286                 }
    287 
    288                 if (varNum < varCount) {
    289                     drv->mFieldIsObject[varNum] = true;
    290                 }
    291             }
    292         }
    293 
    294         script->mHal.info.exportedPragmaCount = 0;
    295 
    296         if (drv->mRootExpand) {
    297             script->mHal.info.root = drv->mRootExpand;
    298         } else {
    299             script->mHal.info.root = drv->mRoot;
    300         }
    301 
    302         if (varCount > 0) {
    303             drv->mBoundAllocs = new Allocation *[varCount];
    304             memset(drv->mBoundAllocs, 0, varCount * sizeof(*drv->mBoundAllocs));
    305         }
    306 
    307         script->mHal.info.isThreadable = true;
    308 
    309         if (scriptSO == (void*)1) {
    310             rsdLookupRuntimeStub(script, "acos");
    311         }
    312     }
    313 
    314     fclose(fp);
    315     pthread_mutex_unlock(&rsdgInitMutex);
    316     return true;
    317 
    318 error:
    319 
    320     fclose(fp);
    321     pthread_mutex_unlock(&rsdgInitMutex);
    322     if (drv) {
    323         delete[] drv->mInvokeFunctions;
    324         delete[] drv->mForEachFunctions;
    325         delete[] drv->mFieldAddress;
    326         delete[] drv->mFieldIsObject;
    327         delete[] drv->mForEachSignatures;
    328         delete[] drv->mBoundAllocs;
    329         if (drv->mScriptSO) {
    330             dlclose(drv->mScriptSO);
    331         }
    332         free(drv);
    333     }
    334     script->mHal.drv = NULL;
    335     return false;
    336 
    337 }
    338 
    339 bool rsdInitIntrinsic(const Context *rsc, Script *s, RsScriptIntrinsicID iid, Element *e) {
    340     pthread_mutex_lock(&rsdgInitMutex);
    341 
    342     DrvScript *drv = (DrvScript *)calloc(1, sizeof(DrvScript));
    343     if (drv == NULL) {
    344         goto error;
    345     }
    346     s->mHal.drv = drv;
    347     drv->mIntrinsicID = iid;
    348     drv->mIntrinsicData = rsdIntrinsic_Init(rsc, s, iid, &drv->mIntrinsicFuncs);
    349     s->mHal.info.isThreadable = true;
    350 
    351     pthread_mutex_unlock(&rsdgInitMutex);
    352     return true;
    353 
    354 error:
    355     pthread_mutex_unlock(&rsdgInitMutex);
    356     return false;
    357 }
    358 
    359 typedef void (*rs_t)(const void *, void *, const void *, uint32_t, uint32_t, uint32_t, uint32_t);
    360 
    361 static void wc_xy(void *usr, uint32_t idx) {
    362     MTLaunchStruct *mtls = (MTLaunchStruct *)usr;
    363     RsForEachStubParamStruct p;
    364     memcpy(&p, &mtls->fep, sizeof(p));
    365     RsdHal * dc = (RsdHal *)mtls->rsc->mHal.drv;
    366     uint32_t sig = mtls->sig;
    367 
    368     rs_t bare_fn = (rs_t) mtls->kernel;
    369     while (1) {
    370         uint32_t slice = (uint32_t)android_atomic_inc(&mtls->mSliceNum);
    371         uint32_t yStart = mtls->yStart + slice * mtls->mSliceSize;
    372         uint32_t yEnd = yStart + mtls->mSliceSize;
    373         yEnd = rsMin(yEnd, mtls->yEnd);
    374         if (yEnd <= yStart) {
    375             return;
    376         }
    377 
    378         //ALOGE("usr idx %i, x %i,%i  y %i,%i", idx, mtls->xStart, mtls->xEnd, yStart, yEnd);
    379         //ALOGE("usr ptr in %p,  out %p", mtls->ptrIn, mtls->ptrOut);
    380         for (p.y = yStart; p.y < yEnd; p.y++) {
    381             p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * p.y);
    382             p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * p.y);
    383             for (uint32_t x = mtls->xStart; x < mtls->xEnd; ++x) {
    384                 bare_fn(p.in, p.out, p.usr, x, p.y, 0, 0);
    385                 p.in = (char *)(p.in) + mtls->fep.eStrideIn;
    386                 p.out = (char *)(p.out) + mtls->fep.eStrideOut;
    387             }
    388         }
    389     }
    390 }
    391 
    392 static void wc_x(void *usr, uint32_t idx) {
    393     MTLaunchStruct *mtls = (MTLaunchStruct *)usr;
    394     RsForEachStubParamStruct p;
    395     memcpy(&p, &mtls->fep, sizeof(p));
    396     RsdHal * dc = (RsdHal *)mtls->rsc->mHal.drv;
    397     uint32_t sig = mtls->sig;
    398 
    399     rs_t bare_fn = (rs_t) mtls->kernel;
    400     while (1) {
    401         uint32_t slice = (uint32_t)android_atomic_inc(&mtls->mSliceNum);
    402         uint32_t xStart = mtls->xStart + slice * mtls->mSliceSize;
    403         uint32_t xEnd = xStart + mtls->mSliceSize;
    404         xEnd = rsMin(xEnd, mtls->xEnd);
    405         if (xEnd <= xStart) {
    406             return;
    407         }
    408 
    409         //ALOGE("usr slice %i idx %i, x %i,%i", slice, idx, xStart, xEnd);
    410         //ALOGE("usr ptr in %p,  out %p", mtls->ptrIn, mtls->ptrOut);
    411 
    412         p.out = mtls->fep.ptrOut + (mtls->fep.eStrideOut * xStart);
    413         p.in = mtls->fep.ptrIn + (mtls->fep.eStrideIn * xStart);
    414         for (uint32_t x = mtls->xStart; x < mtls->xEnd; ++x) {
    415             bare_fn(p.in, p.out, p.usr, x, 0, 0, 0);
    416             p.in = (char *)(p.in) + mtls->fep.eStrideIn;
    417             p.out = (char *)(p.out) + mtls->fep.eStrideOut;
    418         }
    419     }
    420 }
    421 
    422 void rsdScriptInvokeForEachMtlsSetup(const Context *rsc,
    423                                      const Allocation * ain,
    424                                      Allocation * aout,
    425                                      const void * usr,
    426                                      uint32_t usrLen,
    427                                      const RsScriptCall *sc,
    428                                      MTLaunchStruct *mtls) {
    429 
    430     memset(mtls, 0, sizeof(MTLaunchStruct));
    431 
    432     if (ain) {
    433         mtls->fep.dimX = ain->getType()->getDimX();
    434         mtls->fep.dimY = ain->getType()->getDimY();
    435         mtls->fep.dimZ = ain->getType()->getDimZ();
    436         //mtls->dimArray = ain->getType()->getDimArray();
    437     } else if (aout) {
    438         mtls->fep.dimX = aout->getType()->getDimX();
    439         mtls->fep.dimY = aout->getType()->getDimY();
    440         mtls->fep.dimZ = aout->getType()->getDimZ();
    441         //mtls->dimArray = aout->getType()->getDimArray();
    442     } else {
    443         rsc->setError(RS_ERROR_BAD_SCRIPT, "rsForEach called with null allocations");
    444         return;
    445     }
    446 
    447     if (!sc || (sc->xEnd == 0)) {
    448         mtls->xEnd = mtls->fep.dimX;
    449     } else {
    450         rsAssert(sc->xStart < mtls->fep.dimX);
    451         rsAssert(sc->xEnd <= mtls->fep.dimX);
    452         rsAssert(sc->xStart < sc->xEnd);
    453         mtls->xStart = rsMin(mtls->fep.dimX, sc->xStart);
    454         mtls->xEnd = rsMin(mtls->fep.dimX, sc->xEnd);
    455         if (mtls->xStart >= mtls->xEnd) return;
    456     }
    457 
    458     if (!sc || (sc->yEnd == 0)) {
    459         mtls->yEnd = mtls->fep.dimY;
    460     } else {
    461         rsAssert(sc->yStart < mtls->fep.dimY);
    462         rsAssert(sc->yEnd <= mtls->fep.dimY);
    463         rsAssert(sc->yStart < sc->yEnd);
    464         mtls->yStart = rsMin(mtls->fep.dimY, sc->yStart);
    465         mtls->yEnd = rsMin(mtls->fep.dimY, sc->yEnd);
    466         if (mtls->yStart >= mtls->yEnd) return;
    467     }
    468 
    469     mtls->xEnd = rsMax((uint32_t)1, mtls->xEnd);
    470     mtls->yEnd = rsMax((uint32_t)1, mtls->yEnd);
    471     mtls->zEnd = rsMax((uint32_t)1, mtls->zEnd);
    472     mtls->arrayEnd = rsMax((uint32_t)1, mtls->arrayEnd);
    473 
    474     rsAssert(!ain || (ain->getType()->getDimZ() == 0));
    475 
    476     Context *mrsc = (Context *)rsc;
    477     mtls->rsc = mrsc;
    478     mtls->ain = ain;
    479     mtls->aout = aout;
    480     mtls->fep.usr = usr;
    481     mtls->fep.usrLen = usrLen;
    482     mtls->mSliceSize = 10;
    483     mtls->mSliceNum = 0;
    484 
    485     mtls->fep.ptrIn = NULL;
    486     mtls->fep.eStrideIn = 0;
    487 
    488     if (ain) {
    489         DrvAllocation *aindrv = (DrvAllocation *)ain->mHal.drv;
    490         mtls->fep.ptrIn = (const uint8_t *)aindrv->lod[0].mallocPtr;
    491         mtls->fep.eStrideIn = ain->getType()->getElementSizeBytes();
    492         mtls->fep.yStrideIn = aindrv->lod[0].stride;
    493     }
    494 
    495     mtls->fep.ptrOut = NULL;
    496     mtls->fep.eStrideOut = 0;
    497     if (aout) {
    498         DrvAllocation *aoutdrv = (DrvAllocation *)aout->mHal.drv;
    499         mtls->fep.ptrOut = (uint8_t *)aoutdrv->lod[0].mallocPtr;
    500         mtls->fep.eStrideOut = aout->getType()->getElementSizeBytes();
    501         mtls->fep.yStrideOut = aoutdrv->lod[0].stride;
    502     }
    503 }
    504 
    505 void rsdScriptLaunchThreads(const Context *rsc,
    506                             Script *s,
    507                             uint32_t slot,
    508                             const Allocation * ain,
    509                             Allocation * aout,
    510                             const void * usr,
    511                             uint32_t usrLen,
    512                             const RsScriptCall *sc,
    513                             MTLaunchStruct *mtls) {
    514 
    515     Script * oldTLS = setTLS(s);
    516     Context *mrsc = (Context *)rsc;
    517     RsdHal * dc = (RsdHal *)mtls->rsc->mHal.drv;
    518 
    519     if ((dc->mWorkers.mCount > 1) && s->mHal.info.isThreadable && !dc->mInForEach) {
    520         dc->mInForEach = true;
    521         if (mtls->fep.dimY > 1) {
    522             mtls->mSliceSize = mtls->fep.dimY / (dc->mWorkers.mCount * 4);
    523             if(mtls->mSliceSize < 1) {
    524                 mtls->mSliceSize = 1;
    525             }
    526 
    527             rsdLaunchThreads(mrsc, wc_xy, mtls);
    528         } else {
    529             mtls->mSliceSize = mtls->fep.dimX / (dc->mWorkers.mCount * 4);
    530             if(mtls->mSliceSize < 1) {
    531                 mtls->mSliceSize = 1;
    532             }
    533 
    534             rsdLaunchThreads(mrsc, wc_x, mtls);
    535         }
    536         dc->mInForEach = false;
    537 
    538         //ALOGE("launch 1");
    539     } else {
    540         RsForEachStubParamStruct p;
    541         memcpy(&p, &mtls->fep, sizeof(p));
    542         uint32_t sig = mtls->sig;
    543 
    544         //ALOGE("launch 3");
    545         outer_foreach_t fn = (outer_foreach_t) mtls->kernel;
    546         for (p.ar[0] = mtls->arrayStart; p.ar[0] < mtls->arrayEnd; p.ar[0]++) {
    547             for (p.z = mtls->zStart; p.z < mtls->zEnd; p.z++) {
    548                 for (p.y = mtls->yStart; p.y < mtls->yEnd; p.y++) {
    549                     uint32_t offset = mtls->fep.dimY * mtls->fep.dimZ * p.ar[0] +
    550                                       mtls->fep.dimY * p.z + p.y;
    551                     p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * offset);
    552                     p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * offset);
    553                     fn(&p, mtls->xStart, mtls->xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut);
    554                 }
    555             }
    556         }
    557     }
    558 
    559     setTLS(oldTLS);
    560 }
    561 
    562 void rsdScriptInvokeForEach(const Context *rsc,
    563                             Script *s,
    564                             uint32_t slot,
    565                             const Allocation * ain,
    566                             Allocation * aout,
    567                             const void * usr,
    568                             uint32_t usrLen,
    569                             const RsScriptCall *sc) {
    570 
    571     RsdHal * dc = (RsdHal *)rsc->mHal.drv;
    572 
    573     MTLaunchStruct mtls;
    574     rsdScriptInvokeForEachMtlsSetup(rsc, ain, aout, usr, usrLen, sc, &mtls);
    575     mtls.script = s;
    576     mtls.fep.slot = slot;
    577 
    578     DrvScript *drv = (DrvScript *)s->mHal.drv;
    579     if (drv->mIntrinsicID) {
    580         mtls.kernel = (void (*)())drv->mIntrinsicFuncs.root;
    581         mtls.fep.usr = drv->mIntrinsicData;
    582     } else {
    583 		mtls.kernel = drv->mForEachFunctions[slot];
    584         rsAssert(mtls.kernel != NULL);
    585         mtls.sig = drv->mForEachSignatures[slot];
    586     }
    587 
    588 
    589     rsdScriptLaunchThreads(rsc, s, slot, ain, aout, usr, usrLen, sc, &mtls);
    590 }
    591 
    592 
    593 int rsdScriptInvokeRoot(const Context *dc, Script *script) {
    594     DrvScript *drv = (DrvScript *)script->mHal.drv;
    595 
    596     Script * oldTLS = setTLS(script);
    597     int ret = drv->mRoot();
    598     setTLS(oldTLS);
    599 
    600     return ret;
    601 }
    602 
    603 void rsdScriptInvokeInit(const Context *dc, Script *script) {
    604     DrvScript *drv = (DrvScript *)script->mHal.drv;
    605 
    606     if (drv->mInit) {
    607         drv->mInit();
    608     }
    609 }
    610 
    611 void rsdScriptInvokeFreeChildren(const Context *dc, Script *script) {
    612     DrvScript *drv = (DrvScript *)script->mHal.drv;
    613 
    614     if (drv->mFreeChildren) {
    615         drv->mFreeChildren();
    616     }
    617 }
    618 
    619 void rsdScriptInvokeFunction(const Context *dc, Script *script,
    620                             uint32_t slot,
    621                             const void *params,
    622                             size_t paramLength) {
    623     DrvScript *drv = (DrvScript *)script->mHal.drv;
    624     //ALOGE("invoke %p %p %i %p %i", dc, script, slot, params, paramLength);
    625 
    626     Script * oldTLS = setTLS(script);
    627     reinterpret_cast<void (*)(const void *, uint32_t)>(
    628         drv->mInvokeFunctions[slot])(params, paramLength);
    629     setTLS(oldTLS);
    630 }
    631 
    632 void rsdScriptSetGlobalVar(const Context *dc, const Script *script,
    633                            uint32_t slot, void *data, size_t dataLength) {
    634     DrvScript *drv = (DrvScript *)script->mHal.drv;
    635     //rsAssert(!script->mFieldIsObject[slot]);
    636     //ALOGE("setGlobalVar %p %p %i %p %i", dc, script, slot, data, dataLength);
    637 
    638     if (drv->mIntrinsicID) {
    639         drv->mIntrinsicFuncs.setVar(dc, script, drv->mIntrinsicData, slot, data, dataLength);
    640         return;
    641     }
    642 
    643     int32_t *destPtr = reinterpret_cast<int32_t *>(drv->mFieldAddress[slot]);
    644     if (!destPtr) {
    645         //ALOGV("Calling setVar on slot = %i which is null", slot);
    646         return;
    647     }
    648 
    649     memcpy(destPtr, data, dataLength);
    650 }
    651 
    652 void rsdScriptSetGlobalVarWithElemDims(
    653         const android::renderscript::Context *dc,
    654         const android::renderscript::Script *script,
    655         uint32_t slot, void *data, size_t dataLength,
    656         const android::renderscript::Element *elem,
    657         const size_t *dims, size_t dimLength) {
    658     DrvScript *drv = (DrvScript *)script->mHal.drv;
    659 
    660     int32_t *destPtr = reinterpret_cast<int32_t *>(drv->mFieldAddress[slot]);
    661     if (!destPtr) {
    662         //ALOGV("Calling setVar on slot = %i which is null", slot);
    663         return;
    664     }
    665 
    666     // We want to look at dimension in terms of integer components,
    667     // but dimLength is given in terms of bytes.
    668     dimLength /= sizeof(int);
    669 
    670     // Only a single dimension is currently supported.
    671     rsAssert(dimLength == 1);
    672     if (dimLength == 1) {
    673         // First do the increment loop.
    674         size_t stride = elem->getSizeBytes();
    675         char *cVal = reinterpret_cast<char *>(data);
    676         for (size_t i = 0; i < dims[0]; i++) {
    677             elem->incRefs(cVal);
    678             cVal += stride;
    679         }
    680 
    681         // Decrement loop comes after (to prevent race conditions).
    682         char *oldVal = reinterpret_cast<char *>(destPtr);
    683         for (size_t i = 0; i < dims[0]; i++) {
    684             elem->decRefs(oldVal);
    685             oldVal += stride;
    686         }
    687     }
    688 
    689     memcpy(destPtr, data, dataLength);
    690 }
    691 
    692 void rsdScriptSetGlobalBind(const Context *dc, const Script *script, uint32_t slot, Allocation *data) {
    693     DrvScript *drv = (DrvScript *)script->mHal.drv;
    694 
    695     //rsAssert(!script->mFieldIsObject[slot]);
    696     //ALOGE("setGlobalBind %p %p %i %p", dc, script, slot, data);
    697 
    698     if (drv->mIntrinsicID) {
    699         drv->mIntrinsicFuncs.bind(dc, script, drv->mIntrinsicData, slot, data);
    700         return;
    701     }
    702 
    703     int32_t *destPtr = reinterpret_cast<int32_t *>(drv->mFieldAddress[slot]);
    704     if (!destPtr) {
    705         //ALOGV("Calling setVar on slot = %i which is null", slot);
    706         return;
    707     }
    708 
    709     void *ptr = NULL;
    710     drv->mBoundAllocs[slot] = data;
    711     if(data) {
    712         DrvAllocation *allocDrv = (DrvAllocation *)data->mHal.drv;
    713         ptr = allocDrv->lod[0].mallocPtr;
    714     }
    715     memcpy(destPtr, &ptr, sizeof(void *));
    716 }
    717 
    718 void rsdScriptSetGlobalObj(const Context *dc, const Script *script, uint32_t slot, ObjectBase *data) {
    719     DrvScript *drv = (DrvScript *)script->mHal.drv;
    720     //rsAssert(script->mFieldIsObject[slot]);
    721     //ALOGE("setGlobalObj %p %p %i %p", dc, script, slot, data);
    722 
    723     int32_t *destPtr = reinterpret_cast<int32_t *>(drv->mFieldAddress[slot]);
    724     if (!destPtr) {
    725         //ALOGV("Calling setVar on slot = %i which is null", slot);
    726         return;
    727     }
    728 
    729     rsrSetObject(dc, script, (ObjectBase **)destPtr, data);
    730 }
    731 
    732 void rsdScriptDestroy(const Context *dc, Script *script) {
    733     DrvScript *drv = (DrvScript *)script->mHal.drv;
    734 
    735     if (drv == NULL) {
    736         return;
    737     }
    738 
    739     for (size_t i = 0; i < script->mHal.info.exportedVariableCount; ++i) {
    740         if (drv->mFieldIsObject[i]) {
    741             if (drv->mFieldAddress[i] != NULL) {
    742                 ObjectBase **obj_addr =
    743                     reinterpret_cast<ObjectBase **>(drv->mFieldAddress[i]);
    744                 rsrClearObject(dc, script, obj_addr);
    745             }
    746         }
    747     }
    748 
    749     delete[] drv->mInvokeFunctions;
    750     delete[] drv->mForEachFunctions;
    751     delete[] drv->mFieldAddress;
    752     delete[] drv->mFieldIsObject;
    753     delete[] drv->mForEachSignatures;
    754     delete[] drv->mBoundAllocs;
    755     if (drv->mScriptSO) {
    756         dlclose(drv->mScriptSO);
    757     }
    758     free(drv);
    759     script->mHal.drv = NULL;
    760 }
    761 
    762 Allocation * rsdScriptGetAllocationForPointer(const android::renderscript::Context *dc,
    763                                               const android::renderscript::Script *sc,
    764                                               const void *ptr) {
    765     DrvScript *drv = (DrvScript *)sc->mHal.drv;
    766     if (!ptr) {
    767         return NULL;
    768     }
    769 
    770     for (uint32_t ct=0; ct < sc->mHal.info.exportedVariableCount; ct++) {
    771         Allocation *a = drv->mBoundAllocs[ct];
    772         if (!a) continue;
    773         DrvAllocation *adrv = (DrvAllocation *)a->mHal.drv;
    774         if (adrv->lod[0].mallocPtr == ptr) {
    775             return a;
    776         }
    777     }
    778     ALOGE("rsGetAllocation, failed to find %p", ptr);
    779     return NULL;
    780 }
    781 
    782