Home | History | Annotate | Download | only in lib
      1 /*
      2  * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *     http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 /**
     17  * @file picopr.c
     18  *
     19  * text preprocessor
     20  *
     21  * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
     22  * All rights reserved.
     23  *
     24  * History:
     25  * - 2009-04-20 -- initial version
     26  *
     27  */
     28 
     29 #include "picodefs.h"
     30 #include "picoos.h"
     31 #include "picobase.h"
     32 #include "picodbg.h"
     33 #include "picodata.h"
     34 #include "picokpr.h"
     35 #include "picopr.h"
     36 #include "picoktab.h"
     37 
     38 #ifdef __cplusplus
     39 extern "C" {
     40 #endif
     41 #if 0
     42 }
     43 #endif
     44 
     45 
     46 /* *****************************************************************************/
     47 /* constants */
     48 /* *****************************************************************************/
     49 
     50 #define PR_TRACE_MEM      FALSE
     51 #define PR_TRACE_MAX_MEM  FALSE
     52 #define PR_TRACE_PATHCOST TRUE
     53 
     54 #define PR_WORK_MEM_SIZE  10000
     55 #define PR_DYN_MEM_SIZE   7000
     56 
     57 #define PR_ENABLED TRUE
     58 
     59 #define PR_MAX_NR_ITERATIONS 1000;
     60 
     61 #define SPEC_CHAR           "\\/"
     62 
     63 #define PICO_ERR_CONTEXT_NOT_FOUND            PICO_ERR_OTHER
     64 #define PICO_ERR_MAX_PREPROC_PATH_LEN_REACHED PICO_ERR_OTHER
     65 
     66 #define IN_BUF_SIZE   255
     67 #define OUT_BUF_SIZE  IN_BUF_SIZE + 3 * PICODATA_ITEM_HEADSIZE + 3
     68 
     69 #define PR_MAX_NR_PREPROC   (1 + PICOKNOW_MAX_NUM_UTPP)
     70 
     71 #define PR_MAX_PATH_LEN     130
     72 #define PR_MAX_DATA_LEN     IN_BUF_SIZE
     73 #define PR_MAX_DATA_LEN_Z   PR_MAX_DATA_LEN + 1      /* all strings in picopr should use this constant
     74                                                         to ensure zero termination */
     75 #define PR_COST_INIT        100000
     76 #define PR_COST             10
     77 #define PR_EOL              '\n'
     78 
     79 /* Bit mask constants for token sets with parameters */
     80 #define PR_TSE_MASK_OUT      (1<<PR_TSEOut)
     81 #define PR_TSE_MASK_MIN      (1<<PR_TSEMin)
     82 #define PR_TSE_MASK_MAX      (1<<PR_TSEMax)
     83 #define PR_TSE_MASK_LEN      (1<<PR_TSELen)
     84 #define PR_TSE_MASK_VAL      (1<<PR_TSEVal)
     85 #define PR_TSE_MASK_STR      (1<<PR_TSEStr)
     86 #define PR_TSE_MASK_HEAD     (1<<PR_TSEHead)
     87 #define PR_TSE_MASK_MID      (1<<PR_TSEMid)
     88 #define PR_TSE_MASK_TAIL     (1<<PR_TSETail)
     89 #define PR_TSE_MASK_PROD     (1<<PR_TSEProd)
     90 #define PR_TSE_MASK_PRODEXT  (1<<PR_TSEProdExt)
     91 #define PR_TSE_MASK_VAR      (1<<PR_TSEVar)
     92 #define PR_TSE_MASK_LEX      (1<<PR_TSELex)
     93 #define PR_TSE_MASK_COST     (1<<PR_TSECost)
     94 #define PR_TSE_MASK_ID       (1<<PR_TSEID)
     95 #define PR_TSE_MASK_DUMMY1   (1<<PR_TSEDummy1)
     96 #define PR_TSE_MASK_DUMMY2   (1<<PR_TSEDummy2)
     97 #define PR_TSE_MASK_DUMMY3   (1<<PR_TSEDummy3)
     98 
     99 /* Bit mask constants for token sets without parameters */
    100 #define PR_TSE_MASK_BEGIN      (1<<PR_TSEBegin)
    101 #define PR_TSE_MASK_END        (1<<PR_TSEEnd)
    102 #define PR_TSE_MASK_SPACE      (1<<PR_TSESpace)
    103 #define PR_TSE_MASK_DIGIT      (1<<PR_TSEDigit)
    104 #define PR_TSE_MASK_LETTER     (1<<PR_TSELetter)
    105 #define PR_TSE_MASK_CHAR       (1<<PR_TSEChar)
    106 #define PR_TSE_MASK_SEQ        (1<<PR_TSESeq)
    107 #define PR_TSE_MASK_CMPR       (1<<PR_TSECmpr)
    108 #define PR_TSE_MASK_NLZ        (1<<PR_TSENLZ)
    109 #define PR_TSE_MASK_ROMAN      (1<<PR_TSERoman)
    110 #define PR_TSE_MASK_CI         (1<<PR_TSECI)
    111 #define PR_TSE_MASK_CIS        (1<<PR_TSECIS)
    112 #define PR_TSE_MASK_AUC        (1<<PR_TSEAUC)
    113 #define PR_TSE_MASK_ALC        (1<<PR_TSEALC)
    114 #define PR_TSE_MASK_SUC        (1<<PR_TSESUC)
    115 #define PR_TSE_MASK_ACCEPT     (1<<PR_TSEAccept)
    116 #define PR_TSE_MASK_NEXT       (1<<PR_TSENext)
    117 #define PR_TSE_MASK_ALTL       (1<<PR_TSEAltL)
    118 #define PR_TSE_MASK_ALTR       (1<<PR_TSEAltR)
    119 
    120 #define PR_FIRST_TSE_WP PR_TSEOut
    121 
    122 #define PR_SMALLER 1
    123 #define PR_EQUAL   0
    124 #define PR_LARGER  2
    125 
    126 #define PR_SPELL_WITH_SENTENCE_BREAK  -2
    127 #define PR_SPELL_WITH_PHRASE_BREAK    -1
    128 #define PR_SPELL                       0
    129 
    130 #define PICO_SPEED_MIN           20
    131 #define PICO_SPEED_MAX          500
    132 #define PICO_SPEED_DEFAULT      100
    133 #define PICO_SPEED_FACTOR_MIN   500
    134 #define PICO_SPEED_FACTOR_MAX  2000
    135 
    136 #define PICO_PITCH_MIN           50
    137 #define PICO_PITCH_MAX          200
    138 #define PICO_PITCH_DEFAULT      100
    139 #define PICO_PITCH_FACTOR_MIN   500
    140 #define PICO_PITCH_FACTOR_MAX  2000
    141 #define PICO_PITCH_ADD_MIN     -100
    142 #define PICO_PITCH_ADD_MAX      100
    143 #define PICO_PITCH_ADD_DEFAULT    0
    144 
    145 #define PICO_VOLUME_MIN           0
    146 #define PICO_VOLUME_MAX         500
    147 #define PICO_VOLUME_DEFAULT     100
    148 #define PICO_VOLUME_FACTOR_MIN  500
    149 #define PICO_VOLUME_FACTOR_MAX 2000
    150 
    151 #define PICO_CONTEXT_DEFAULT   "DEFAULT"
    152 
    153 #define PICO_PARAGRAPH_PAUSE_DUR 500
    154 
    155 
    156 /* *****************************************************************************/
    157 /* types */
    158 /* *****************************************************************************/
    159 
    160 typedef enum {PR_OStr, PR_OVar, PR_OItem, PR_OSpell, PR_ORomanToCard, PR_OVal,
    161               PR_OLeft, PR_ORight, PR_ORLZ, PR_OIgnore, PR_OPitch, PR_OSpeed,
    162               PR_OVolume, PR_OVoice, PR_OContext, PR_OPhonSVOXPA, PR_OPhonSAMPA,
    163               PR_OPlay, PR_OUseSig, PR_OGenFile, PR_OAudioEdit, PR_OPara,
    164               PR_OSent, PR_OBreak, PR_OMark, PR_OConcat, PR_OLast}  pr_OutType;
    165 
    166 typedef enum {PR_TSEBegin, PR_TSEEnd, PR_TSESpace, PR_TSEDigit, PR_TSELetter, PR_TSEChar, PR_TSESeq,
    167               PR_TSECmpr, PR_TSENLZ, PR_TSERoman, PR_TSECI, PR_TSECIS, PR_TSEAUC, PR_TSEALC, PR_TSESUC,
    168               PR_TSEAccept, PR_TSENext, PR_TSEAltL, PR_TSEAltR}  pr_TokSetEleNP;
    169 
    170 typedef enum {PR_TSEOut, PR_TSEMin, PR_TSEMax, PR_TSELen, PR_TSEVal, PR_TSEStr, PR_TSEHead, PR_TSEMid,
    171               PR_TSETail, PR_TSEProd, PR_TSEProdExt, PR_TSEVar, PR_TSELex, PR_TSECost, PR_TSEID,
    172               PR_TSEDummy1, PR_TSEDummy2, PR_TSEDummy3}  pr_TokSetEleWP;
    173 
    174 typedef enum {PR_GSNoPreproc, PR_GS_START, PR_GSContinue, PR_GSNeedToken, PR_GSNotFound, PR_GSFound}  pr_GlobalState;
    175 
    176 typedef enum {PR_LSError, PR_LSInit, PR_LSGetToken, PR_LSGetToken2, PR_LSMatch, PR_LSGoBack,
    177               PR_LSGetProdToken, PR_LSInProd, PR_LSGetProdContToken, PR_LSInProdCont, PR_LSGetNextToken,
    178               PR_LSGetAltToken}  pr_LocalState;
    179 
    180 typedef enum {PR_MSNotMatched, PR_MSMatched, PR_MSMatchedContinue, PR_MSMatchedMulti}  pr_MatchState;
    181 
    182 typedef struct pr_Prod * pr_ProdList;
    183 typedef struct pr_Prod {
    184     picokpr_Preproc rNetwork;
    185     picokpr_ProdArrOffset rProdOfs;
    186     pr_ProdList rNext;
    187 } pr_Prod;
    188 
    189 typedef struct pr_Context * pr_ContextList;
    190 typedef struct pr_Context {
    191     picoos_uchar * rContextName;
    192     pr_ProdList rProdList;
    193     pr_ContextList rNext;
    194 } pr_Context;
    195 
    196 /* *****************************************************************************/
    197 /* used, but to be checked */
    198 
    199 #define MaxNrShortStrParams 2
    200 #define MaxPhoneLen 14
    201 #define ShortStrParamLen (2 * MaxPhoneLen)
    202 typedef picoos_uchar ShortStrParam[ShortStrParamLen];
    203 
    204 
    205 typedef struct pr_ioItem * pr_ioItemPtr;
    206 typedef struct pr_ioItem {
    207     pr_ioItemPtr next;
    208     picoos_int32 val;
    209     struct picodata_itemhead head;
    210     picoos_uchar * strci;
    211     picoos_uchar * strcis;
    212     picoos_bool alc;
    213     picoos_bool auc;
    214     picoos_bool suc;
    215     picoos_uchar data[PR_MAX_DATA_LEN_Z];
    216 } pr_ioItem;
    217 
    218 typedef struct pr_ioItem2 {
    219     pr_ioItemPtr next;
    220     picoos_int32 val;
    221     struct picodata_itemhead head;
    222     picoos_uchar * strci;
    223     picoos_uchar * strcis;
    224     picoos_bool alc;
    225     picoos_bool auc;
    226     picoos_bool suc;
    227 } pr_ioItem2;
    228 
    229 #define PR_IOITEM_MIN_SIZE sizeof(pr_ioItem2)
    230 
    231 typedef picoos_uint32 pr_MemState;
    232 typedef enum {pr_DynMem, pr_WorkMem} pr_MemTypes;
    233 
    234 /* *****************************************************************************/
    235 
    236 typedef struct pr_OutItemVar * pr_OutItemVarPtr;
    237 struct pr_OutItemVar {
    238     pr_ioItemPtr first;
    239     pr_ioItemPtr last;
    240     picoos_int32 id;
    241     pr_OutItemVarPtr next;
    242 };
    243 
    244 
    245 struct pr_WorkItem {
    246     pr_ioItemPtr rit;
    247 };
    248 typedef pr_ioItemPtr pr_WorkItems[PR_MAX_PATH_LEN+1];
    249 
    250 struct pr_PathEle {
    251     picokpr_Preproc rnetwork;
    252     picoos_int16 ritemid;
    253     picoos_int16 rcompare;
    254     picoos_int16 rdepth;
    255     picokpr_TokArrOffset rtok;
    256     picokpr_StrArrOffset rprodname;
    257     picoos_int32 rprodprefcost;
    258     pr_LocalState rlState;
    259 };
    260 
    261 typedef struct pr_Path {
    262     picoos_int32 rcost;
    263     picoos_int32 rlen;
    264     struct pr_PathEle rele[PR_MAX_PATH_LEN];
    265 } pr_Path;
    266 
    267 /* *****************************************************************************/
    268 
    269 /** subobject : PreprocUnit
    270  *  shortcut  : pr
    271  */
    272 typedef struct pr_subobj
    273 {
    274     pr_ioItemPtr rinItemList;
    275     pr_ioItemPtr rlastInItem;
    276     pr_ioItemPtr routItemList;
    277     pr_ioItemPtr rlastOutItem;
    278     pr_GlobalState rgState;
    279     pr_Path ractpath;
    280     pr_Path rbestpath;
    281     picoos_int32 rnritems;
    282     pr_WorkItems ritems;
    283     picoos_int32 rignore;
    284     picoos_int32 spellMode;
    285     picoos_int32 maxPathLen;
    286     picoos_bool insidePhoneme;
    287 
    288     picoos_uint8 inBuf[IN_BUF_SIZE+PICODATA_ITEM_HEADSIZE]; /* internal input buffer */
    289     picoos_uint16 inBufLen;
    290 
    291     picoos_uint8 outBuf[OUT_BUF_SIZE]; /* internal output buffer */
    292     picoos_uint16 outReadPos; /* next pos to read from outBuf */
    293     picoos_uint16 outWritePos; /* next pos to write to outBuf */
    294 
    295     picokpr_Preproc preproc[PR_MAX_NR_PREPROC];
    296     pr_ContextList ctxList;
    297     pr_ProdList prodList;
    298 
    299     pr_ContextList actCtx;
    300     picoos_bool actCtxChanged;
    301 
    302     picoos_uchar tmpStr1[PR_MAX_DATA_LEN_Z];
    303     picoos_uchar tmpStr2[PR_MAX_DATA_LEN_Z];
    304 
    305     picoos_uint8 pr_WorkMem[PR_WORK_MEM_SIZE];
    306     picoos_uint32 workMemTop;
    307     picoos_uint32 maxWorkMemTop;
    308     picoos_uint8 pr_DynMem[PR_DYN_MEM_SIZE];
    309     picoos_MemoryManager dynMemMM;
    310     picoos_int32 dynMemSize;
    311     picoos_int32 maxDynMemSize;
    312 
    313     picoos_bool outOfMemory;
    314 
    315     picoos_bool forceOutput;
    316     picoos_int16 nrIterations;
    317 
    318     picoos_uchar lspaces[128];
    319     picoos_uchar saveFile[IN_BUF_SIZE];
    320 
    321     pr_ioItem tmpItem;
    322 
    323     picotrns_SimpleTransducer transducer;
    324 
    325     /* kbs */
    326 
    327     picoktab_Graphs graphs;
    328     picokfst_FST xsampa_parser;
    329     picokfst_FST svoxpa_parser;
    330     picokfst_FST xsampa2svoxpa_mapper;
    331 
    332 } pr_subobj_t;
    333 
    334 /* *****************************************************************************/
    335 /* prototypes */
    336 
    337 static void pr_getOutputItemList (picodata_ProcessingUnit this, pr_subobj_t * pr,
    338                                   picokpr_Preproc network,
    339                                   picokpr_OutItemArrOffset outitem,
    340                                   pr_OutItemVarPtr vars,
    341                                   pr_ioItemPtr * first, pr_ioItemPtr * last);
    342 
    343 /* *****************************************************************************/
    344 
    345 #define pr_iABS(X) (((X) < 0) ? (-(X)) : (X))
    346 
    347 /* *****************************************************************************/
    348 /* module internal memory managment for dynamic and working memory using memory
    349    partitions allocated with pr_subobj_t.
    350    Dynamic memory is allocated in pr_subobj_t->pr_DynMem. Dynamic memory has
    351    to be deallocated again with pr_DEALLOCATE.
    352    Working memory is allocated in pr_subobj_t->pr_WorkMem. Working memory is stack
    353    based and may not to be deallocated with pr_DEALLOCATE, but with pr_resetMemState
    354    to a state previously saved with pr_getMemState.
    355 */
    356 
    357 static void pr_ALLOCATE (picodata_ProcessingUnit this, pr_MemTypes mType, void * * adr, unsigned int byteSize)
    358   /* allocates 'byteSize' bytes in the memery partition given by 'mType' */
    359 {
    360     pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
    361     picoos_int32 incrUsedBytes, prevmaxDynMemSize;
    362 
    363     if (mType == pr_WorkMem) {
    364         if ((pr->workMemTop + byteSize) < PR_WORK_MEM_SIZE) {
    365             (*adr) = (void *)(&(pr->pr_WorkMem[pr->workMemTop]));
    366             byteSize = ((byteSize + PICOOS_ALIGN_SIZE - 1) / PICOOS_ALIGN_SIZE) * PICOOS_ALIGN_SIZE;
    367             pr->workMemTop += byteSize;
    368 #if PR_TRACE_MEM
    369             PICODBG_INFO(("pr_WorkMem: +%u, tot:%i of %i", byteSize, pr->workMemTop, PR_WORK_MEM_SIZE));
    370 #endif
    371 
    372             if (pr->workMemTop > pr->maxWorkMemTop) {
    373                 pr->maxWorkMemTop = pr->workMemTop;
    374 #if PR_TRACE_MAX_MEM
    375                 PICODBG_INFO(("new max pr_WorkMem: %i of %i", pr->workMemTop, PR_WORK_MEM_SIZE));
    376 #endif
    377             }
    378         }
    379         else {
    380             (*adr) = NULL;
    381             PICODBG_ERROR(("pr out of working memory"));
    382             picoos_emRaiseException(this->common->em, PICO_EXC_OUT_OF_MEM, (picoos_char *)"pr out of dynamic memory", (picoos_char *)"");
    383             pr->outOfMemory = TRUE;
    384         }
    385     }
    386     else if (mType == pr_DynMem) {
    387         (*adr) = picoos_allocate(pr->dynMemMM, byteSize);
    388         if ((*adr) != NULL) {
    389             prevmaxDynMemSize = pr->maxDynMemSize;
    390             picoos_getMemUsage(pr->dynMemMM, 1, &pr->dynMemSize, &incrUsedBytes, &pr->maxDynMemSize);
    391 #if PR_TRACE_MEM
    392             PICODBG_INFO(("pr_DynMem : +%i, tot:%i of %i", incrUsedBytes, pr->dynMemSize, PR_DYN_MEM_SIZE));
    393 #endif
    394 
    395 #if PR_TRACE_MAX_MEM
    396             if (pr->maxDynMemSize > prevmaxDynMemSize) {
    397                 PICODBG_INFO(("new max pr_DynMem : %i of %i", pr->maxDynMemSize, PR_DYN_MEM_SIZE));
    398             }
    399 #endif
    400         }
    401         else {
    402             PICODBG_ERROR(("pr out of dynamic memory"));
    403             picoos_emRaiseException(this->common->em, PICO_EXC_OUT_OF_MEM, (picoos_char *)"pr out of dynamic memory", (picoos_char *)"");
    404             pr->outOfMemory = TRUE;
    405         }
    406     }
    407     else {
    408         (*adr) = NULL;
    409     }
    410 }
    411 
    412 
    413 static void pr_DEALLOCATE (picodata_ProcessingUnit this, pr_MemTypes mType, void * * adr)
    414 {
    415     pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
    416     picoos_int32 incrUsedBytes;
    417     if (mType == pr_WorkMem) {
    418         PICODBG_INFO(("not possible; use pr_resetMemState instead"));
    419     }
    420     else if (mType == pr_DynMem) {
    421         picoos_deallocate(pr->dynMemMM, &(*adr));
    422         picoos_getMemUsage(pr->dynMemMM, 1, &pr->dynMemSize, &incrUsedBytes, &pr->maxDynMemSize);
    423 #if PR_TRACE_MEM
    424         PICODBG_INFO(("pr_DynMem : %i, tot:%i of %i: adr: %u", incrUsedBytes, pr->dynMemSize, PR_DYN_MEM_SIZE, *adr));
    425 #endif
    426     }
    427     else {
    428         (*adr) = NULL;
    429     }
    430 }
    431 
    432 
    433 static void pr_getMemState(picodata_ProcessingUnit this, pr_MemTypes mType, picoos_uint32 *lmemState)
    434 {
    435     pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
    436     mType = mType;        /* avoid warning "var not used in this function"*/
    437     *lmemState = pr->workMemTop;
    438 }
    439 
    440 
    441 static void pr_resetMemState(picodata_ProcessingUnit this, pr_MemTypes mType, picoos_uint32 lmemState)
    442 {
    443     pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
    444 
    445 #if PR_TRACE_MEM
    446     PICODBG_INFO(("pr_WorkMem: -%i, tot:%i of %i", pr->workMemTop-lmemState, lmemState, PR_WORK_MEM_SIZE));
    447 #endif
    448     mType = mType;        /* avoid warning "var not used in this function"*/
    449     pr->workMemTop = lmemState;
    450 }
    451 
    452 
    453 /* *****************************************************************************/
    454 /* string operations */
    455 
    456 static picoos_int32 pr_strlen(const picoos_uchar * str)
    457 {
    458     picoos_int32 i;
    459 
    460     i=0;
    461     while ((i<PR_MAX_DATA_LEN) && (str[i] != 0)) {
    462         i++;
    463     }
    464     return i;
    465 }
    466 
    467 
    468 static picoos_uint32 pr_strcpy(picoos_uint8 * dest, const picoos_uint8 * src)
    469 {
    470     picoos_int32 i;
    471 
    472     i = 0;
    473     while ((i<PR_MAX_DATA_LEN) && (src[i] != 0)) {
    474         dest[i] = src[i];
    475         i++;
    476     }
    477     dest[i] = 0;
    478     return i;
    479 }
    480 
    481 
    482 static picoos_uint32 pr_strcat(picoos_uint8 * dest, const picoos_uint8 * src)
    483 {
    484     picoos_int32 i, j;
    485 
    486     i = 0;
    487     while ((i<PR_MAX_DATA_LEN) && (dest[i] != 0)) {
    488         i++;
    489     }
    490     j = 0;
    491     while ((i<PR_MAX_DATA_LEN) && (j<PR_MAX_DATA_LEN) && (src[j] != 0)) {
    492         dest[i] = src[j];
    493         i++;
    494         j++;
    495     }
    496     dest[i] = 0;
    497     return i;
    498 }
    499 
    500 
    501 static void pr_getTermPartStr (picoos_uchar string[], picoos_int32 * ind, picoos_uchar termCh, picoos_uchar str[], picoos_bool * done)
    502 {
    503     int j;
    504     picoos_bool done1;
    505 
    506     done1 = TRUE;
    507     j = 0;
    508     while ((*ind < PR_MAX_DATA_LEN) && (string[*ind] != termCh) && (string[*ind] != 0)) {
    509         if (j < PR_MAX_DATA_LEN) {
    510             str[j] = string[*ind];
    511             j++;
    512         } else {
    513             done1 = FALSE;
    514         }
    515         (*ind)++;
    516     }
    517     if (j < PR_MAX_DATA_LEN) {
    518         str[j] = 0;
    519     }
    520     *done = ((*ind < PR_MAX_DATA_LEN) && (string[*ind] == termCh));
    521     if (*done) {
    522         (*ind)++;
    523     }
    524     *done = *done && done1;
    525 }
    526 
    527 
    528 static picoos_int32 pr_removeSubstring (int pos, int len, unsigned char str[])
    529 {
    530     int i;
    531     int length;
    532 
    533     length = pr_strlen(str);
    534     if (pos >= length) {
    535         return length;
    536     } else {
    537         i = pos + len;
    538         while (i < length) {
    539             str[pos] = str[i];
    540             i++;
    541             pos++;
    542         }
    543         str[pos] = 0;
    544         return pos;
    545     }
    546 }
    547 
    548 
    549 static picoos_bool pr_strEqual(picoos_uchar * str1, picoos_uchar * str2)
    550 {
    551    return (picoos_strcmp((picoos_char *)str1, (picoos_char *)str2) == 0);
    552 }
    553 
    554 
    555 static void pr_int_to_string(picoos_int32 n, picoos_uchar * str, picoos_int32 maxstrlen)
    556 {
    557     picoos_int32 i, len;
    558     picoos_bool negative=FALSE;
    559 
    560     len = 0;
    561     str[0] = 0;
    562     if (n<0) {
    563         negative = TRUE;
    564         n = -n;
    565         len++;
    566     }
    567     i = n;
    568 
    569     while (i>0) {
    570         i = i / 10;
    571         len++;
    572     }
    573 
    574     if (len<maxstrlen) {
    575         str[len] = 0;
    576         i = n;
    577         while ((i>0) && (len>0)) {
    578             len--;
    579             str[len] = i % 10 + '0';
    580             i = i / 10;
    581         }
    582         if (negative) {
    583             len--;
    584             str[len] = '-';
    585         }
    586     }
    587 }
    588 /* *****************************************************************************/
    589 
    590 static void pr_firstLetterToLowerCase (const picoos_uchar src[], picoos_uchar dest[])
    591 {
    592 
    593     picoos_int32 i;
    594     picoos_int32 j;
    595     picoos_int32 l;
    596     picoos_bool done;
    597 
    598     i = 0;
    599     j = 0;
    600     l = picobase_det_utf8_length(src[0]);
    601     while ((i < l) && (j < PR_MAX_DATA_LEN)) {
    602         dest[j] = src[i];
    603         i++;
    604         j++;
    605     }
    606     if (j < PR_MAX_DATA_LEN) {
    607         dest[j] = 0;
    608     }
    609     picobase_lowercase_utf8_str(dest, (picoos_char*)dest, PR_MAX_DATA_LEN, &done);
    610     j = picobase_det_utf8_length(dest[0]);
    611     l = pr_strlen(src);
    612     while ((i < l) && (j < PR_MAX_DATA_LEN)) {
    613         dest[j] = src[i];
    614         i++;
    615         j++;
    616     }
    617     dest[j] = 0;
    618 }
    619 
    620 
    621 static picoos_int32 tok_tokenDigitStrToInt (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_uchar stokenStr[])
    622 {
    623     picoos_uint32 i;
    624     picoos_uint32 l;
    625     picoos_int32 id;
    626     picoos_int32 val;
    627     picoos_uint32 n;
    628     picobase_utf8char utf8char;
    629 
    630     val = 0;
    631     i = 0;
    632     l = pr_strlen(stokenStr);
    633     while (i < l) {
    634         picobase_get_next_utf8char(stokenStr, PR_MAX_DATA_LEN, & i, utf8char);
    635         id = picoktab_graphOffset(pr->graphs, utf8char);
    636         if (id > 0) {
    637           if (picoktab_getIntPropValue(pr->graphs, id, &n)) {
    638                 val = (10 * val) + n;
    639             } else {
    640                 val = ((10 * val) + (int)((int)utf8char[0] - (int)'0'));
    641             }
    642         } else if ((utf8char[0] >= '0') && (utf8char[0] <= '9')) {
    643             val = 10 * val + ((int)utf8char[0] - (int)'0');
    644         }
    645     }
    646     return val;
    647 }
    648 
    649 
    650 static picoos_bool pr_isLatinNumber (picoos_uchar str[], picoos_int32 * val)
    651 {
    652 
    653     picoos_uint32 li;
    654     picoos_uint32 llen;
    655     picoos_uchar lact;
    656     picoos_uchar lnext;
    657     picoos_uchar lprev;
    658     picoos_uchar llatinI;
    659     picoos_uchar llatinV;
    660     picoos_uchar llatinX;
    661     picoos_uchar llatinL;
    662     picoos_uchar llatinC;
    663     picoos_uchar llatinD;
    664     picoos_uchar llatinM;
    665     picoos_int32 lseq;
    666     picobase_utf8char utf8;
    667 
    668     *val = 0;
    669     llen = picobase_utf8_length(str, PR_MAX_DATA_LEN);
    670     if (llen > 0) {
    671         li = 0;
    672         picobase_get_next_utf8char(str, PR_MAX_DATA_LEN, & li,utf8);
    673         if (picobase_is_utf8_uppercase(utf8, PICOBASE_UTF8_MAXLEN)) {
    674             llatinI = 'I';
    675             llatinV = 'V';
    676             llatinX = 'X';
    677             llatinL = 'L';
    678             llatinC = 'C';
    679             llatinD = 'D';
    680             llatinM = 'M';
    681         } else {
    682             llatinI = 'i';
    683             llatinV = 'v';
    684             llatinX = 'x';
    685             llatinL = 'l';
    686             llatinC = 'c';
    687             llatinD = 'd';
    688             llatinM = 'm';
    689         }
    690         lseq = 1000;
    691         li = 0;
    692         while (li < llen) {
    693             if (li > 0) {
    694                 lprev = str[li - 1];
    695             } else {
    696                 lprev = 0;
    697             }
    698             lact = str[li];
    699             if (li < (llen - 1)) {
    700                 lnext = str[li + 1];
    701             } else {
    702                 lnext = 0;
    703             }
    704             if ((lseq > 1) && (lact == llatinI)) {
    705                 if ((lprev != lact) && (lseq >= 4)) {
    706                     if (lnext == llatinV) {
    707                         *val = *val + 4;
    708                         li++;
    709                         lseq = 1;
    710                     } else if (lnext == llatinX) {
    711                         *val = *val + 9;
    712                         li++;
    713                         lseq = 1;
    714                     } else {
    715                         *val = *val + 1;
    716                         lseq = 3;
    717                     }
    718                 } else {
    719                     *val = *val + 1;
    720                     lseq = lseq - 1;
    721                 }
    722             } else if ((lseq > 5) && (lact == llatinV)) {
    723                 *val = *val + 5;
    724                 lseq = 5;
    725             } else if ((lseq > 10) && (lact == llatinX)) {
    726                 if ((lprev != lact) && (lseq >= 40)) {
    727                     if (lnext == llatinL) {
    728                         *val = *val + 40;
    729                         li++;
    730                         lseq = 10;
    731                     } else if (lnext == llatinC) {
    732                         *val = *val + 90;
    733                         li++;
    734                         lseq = 10;
    735                     } else {
    736                         *val = *val + 10;
    737                         lseq = 30;
    738                     }
    739                 } else {
    740                     *val = *val + 10;
    741                     lseq = lseq - 10;
    742                 }
    743             } else if ((lseq > 50) && (lact == llatinL)) {
    744                 *val = *val + 50;
    745                 lseq = 50;
    746             } else if ((lseq > 100) && (lact == llatinC)) {
    747                 if ((lprev != lact) && (lseq >= 400)) {
    748                     if (lnext == llatinD) {
    749                         *val = *val + 400;
    750                         li++;
    751                         lseq = 100;
    752                     } else if (lnext == llatinM) {
    753                         *val = *val + 900;
    754                         li++;
    755                         lseq = 100;
    756                     } else {
    757                         *val = *val + 100;
    758                         lseq = 300;
    759                     }
    760                 } else {
    761                     *val = *val + 100;
    762                     lseq = lseq - 100;
    763                 }
    764             } else if ((lseq > 500) && (lact == llatinD)) {
    765                 *val = *val + 500;
    766                 lseq = 500;
    767             } else if ((lseq >= 1000) && (lact == llatinM)) {
    768                 *val = *val + 1000;
    769             } else {
    770                 return FALSE;
    771             }
    772             li++;
    773         }
    774     }
    775     return TRUE;
    776 }
    777 
    778 
    779 static picoos_bool pr_isSUC (picoos_uchar str[])
    780 {
    781 
    782     picoos_int32 li;
    783     picoos_bool lis;
    784     picobase_utf8char lutf;
    785     picoos_int32 lj;
    786     picoos_int32 ll;
    787     picoos_bool luc;
    788 
    789     li = 0;
    790     lis = TRUE;
    791     luc = TRUE;
    792     while (lis && (li < PR_MAX_DATA_LEN) && (str[li] != 0)) {
    793         lj = 0;
    794         ll = picobase_det_utf8_length(str[li]);
    795         while (lj < ll) {
    796             lutf[lj] = str[li];
    797             lj++;
    798             li++;
    799         }
    800         lutf[lj] = 0;
    801         if (luc) {
    802             lis = lis && picobase_is_utf8_uppercase(lutf,PICOBASE_UTF8_MAXLEN+1);
    803         } else {
    804             lis = lis && picobase_is_utf8_lowercase(lutf,PICOBASE_UTF8_MAXLEN+1);
    805         }
    806         luc = FALSE;
    807     }
    808     return lis;
    809 }
    810 
    811 /* *****************************************************************************/
    812 
    813 static picoos_bool pr_isCmdType (pr_ioItemPtr it, picoos_uint8 type)
    814 {
    815     if ((it != NULL) && (it->head.type == PICODATA_ITEM_CMD) && (it->head.info1 == type)) {
    816         return TRUE;
    817     } else {
    818         return FALSE;
    819     }
    820 }
    821 
    822 
    823 static picoos_bool pr_isCmdInfo2 (pr_ioItemPtr it, picoos_uint8 info2)
    824 {
    825     if ((it != NULL) && (it->head.type == PICODATA_ITEM_CMD) && (it->head.info2 == info2)) {
    826         return TRUE;
    827     } else {
    828         return FALSE;
    829     }
    830 }
    831 
    832 
    833 static void pr_initPathEle (struct pr_PathEle * ele)
    834 {
    835     ele->rnetwork = NULL;
    836     ele->rtok = 0;
    837     ele->ritemid =  -1;
    838     ele->rdepth = 1;
    839     ele->rlState = PR_LSInit;
    840     ele->rcompare =  -1;
    841     ele->rprodname = 0;
    842     ele->rprodprefcost = 0;
    843 }
    844 
    845 /* *****************************************************************************/
    846 
    847 static void pr_disposeProdList (register picodata_ProcessingUnit this,  pr_ProdList * prodList)
    848 {
    849     pr_ProdList p;
    850 
    851     while ((*prodList) != NULL) {
    852         p = (*prodList);
    853         (*prodList) = (*prodList)->rNext;
    854         picoos_deallocate(this->common->mm, (void *) &p);
    855     }
    856 }
    857 
    858 
    859 static pico_Status pr_addContext (register picodata_ProcessingUnit this,  pr_subobj_t * pr, pr_ContextList * ctxList, picokpr_VarStrPtr contextNamePtr, picokpr_VarStrPtr netNamePtr, picokpr_VarStrPtr prodNamePtr)
    860 {
    861     picokpr_Preproc net;
    862     pr_ContextList ctx;
    863     pr_ProdList prod;
    864     int i;
    865     picokpr_VarStrPtr strp;
    866     picoos_int32 lprodarrlen;
    867 
    868     ctx = (*ctxList);
    869     while ((ctx != NULL) &&  !(pr_strEqual(contextNamePtr, ctx->rContextName))) {
    870         ctx = ctx->rNext;
    871     }
    872     if (ctx == NULL) {
    873         ctx = picoos_allocate(this->common->mm, sizeof(pr_Context));
    874         if (ctx == NULL) {
    875             return PICO_EXC_OUT_OF_MEM;
    876         }
    877         ctx->rNext = (*ctxList);
    878         ctx->rProdList = NULL;
    879         ctx->rContextName = contextNamePtr;
    880         (*ctxList) = ctx;
    881     }
    882     i = 0;
    883     net = pr->preproc[i];
    884     while ((i<PR_MAX_NR_PREPROC) && (net != NULL) && !(pr_strEqual(netNamePtr, picokpr_getPreprocNetName(net)))) {
    885         i++;
    886         net = pr->preproc[i];
    887     }
    888     if (net != NULL) {
    889         i = 0;
    890         strp = picokpr_getVarStrPtr(net, picokpr_getProdNameOfs(net, i));
    891         lprodarrlen = picokpr_getProdArrLen(net);
    892         while ((i < lprodarrlen) &&  !(pr_strEqual(prodNamePtr, strp))) {
    893             i++;
    894             if (i < lprodarrlen) {
    895                 strp = picokpr_getVarStrPtr(net, picokpr_getProdNameOfs(net, i));
    896             }
    897         }
    898         if (i < lprodarrlen) {
    899             prod = picoos_allocate(this->common->mm, sizeof(pr_Prod));
    900             if (prod == NULL) {
    901               return PICO_EXC_OUT_OF_MEM;
    902             }
    903             prod->rNetwork = net;
    904             prod->rProdOfs = i;
    905             prod->rNext = ctx->rProdList;
    906             ctx->rProdList = prod;
    907         }
    908     }
    909     return PICO_OK;
    910 }
    911 
    912 
    913 static pico_Status pr_createContextList (register picodata_ProcessingUnit this)
    914 {
    915     pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
    916     picokpr_VarStrPtr ctxNamePtr;
    917     picokpr_VarStrPtr netNamePtr;
    918     picokpr_VarStrPtr prodNamePtr;
    919     picoos_int32 p, i, n;
    920     pico_Status status;
    921 
    922     pr->ctxList = NULL;
    923     for (p=0; p<PR_MAX_NR_PREPROC; p++) {
    924         if (pr->preproc[p] != NULL) {
    925             n = picokpr_getCtxArrLen(pr->preproc[p]);
    926             for (i = 1; i<n; i++) {
    927                 ctxNamePtr = picokpr_getVarStrPtr(pr->preproc[p], picokpr_getCtxCtxNameOfs(pr->preproc[p], i));
    928                 netNamePtr = picokpr_getVarStrPtr(pr->preproc[p], picokpr_getCtxNetNameOfs(pr->preproc[p], i));
    929                 prodNamePtr = picokpr_getVarStrPtr(pr->preproc[p], picokpr_getCtxProdNameOfs(pr->preproc[p], i));
    930                 status = pr_addContext(this, pr, &pr->ctxList, ctxNamePtr,netNamePtr, prodNamePtr);
    931                 if (status != PICO_OK) {
    932                     return status;
    933                 }
    934             }
    935         }
    936     }
    937     return PICO_OK;
    938 }
    939 
    940 
    941 static void pr_disposeContextList (register picodata_ProcessingUnit this)
    942 {
    943     pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
    944     pr_ContextList c;
    945 
    946     while (pr->ctxList != NULL) {
    947         c = pr->ctxList;
    948         pr->ctxList = pr->ctxList->rNext;
    949         pr_disposeProdList(this, & c->rProdList);
    950         picoos_deallocate(this->common->mm, (void *) &c);
    951     }
    952 }
    953 
    954 
    955 static pr_ContextList pr_findContext (pr_ContextList contextList, picoos_uchar contextName[])
    956 {
    957     pr_ContextList context;
    958 
    959     context = contextList;
    960     while ((context != NULL) &&  !(pr_strEqual(context->rContextName,contextName))) {
    961         context = context->rNext;
    962     }
    963     return context;
    964 }
    965 
    966 
    967 static void pr_setContext (register picodata_ProcessingUnit this,  pr_subobj_t * pr, picoos_uchar context[])
    968 {
    969 
    970     pr_ContextList ctx;
    971 
    972     ctx = pr_findContext(pr->ctxList,context);
    973     if (ctx != NULL) {
    974         pr->actCtx = ctx;
    975         pr->actCtxChanged = TRUE;
    976     } else {
    977         PICODBG_WARN(("context '%s' not found; no change",context));
    978         picoos_emRaiseWarning(this->common->em, PICO_ERR_CONTEXT_NOT_FOUND, (picoos_char*)"context '%s' not found; no change",(picoos_char*)context);
    979     }
    980 }
    981 
    982 /* *****************************************************************************/
    983 /* item handling routines */
    984 
    985 
    986 static picoos_uint32 pr_copyData(picoos_uint8 * dest, const picoos_uint8 * src, picoos_int32 nrBytes, picoos_bool zeroTerm)
    987 {
    988     picoos_int32 i=0;
    989 
    990     if ((src != NULL) && (dest != NULL)) {
    991         i = 0;
    992         while ((i<nrBytes) && (i<PR_MAX_DATA_LEN)) {
    993             dest[i] = src[i];
    994             i++;
    995         }
    996         if (zeroTerm) {
    997             dest[i] = 0;
    998         }
    999     }
   1000     return i;
   1001 }
   1002 
   1003 
   1004 static void pr_initItem(picodata_ProcessingUnit this, pr_ioItem * item)
   1005 {
   1006     item->next = NULL;
   1007     item->val = 0;
   1008     item->head.len = 0;
   1009     item->strci = NULL;
   1010     item->strcis = NULL;
   1011     item->suc = FALSE;
   1012     item->alc = FALSE;
   1013     item->auc = FALSE;
   1014 }
   1015 
   1016 
   1017 static void pr_newItem (picodata_ProcessingUnit this, pr_MemTypes mType, pr_ioItemPtr * item, picoos_uint8 itemType, picoos_int32 size, picoos_bool inItem)
   1018 {
   1019     pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
   1020 
   1021     if (mType == pr_WorkMem) {
   1022         pr_ALLOCATE(this, mType, (void * *) & (*item),PR_IOITEM_MIN_SIZE+size+1);
   1023         if (pr->outOfMemory) return;
   1024         pr_initItem(this, *item);
   1025     }
   1026     else if ((mType == pr_DynMem) && inItem) {
   1027         pr_ALLOCATE(this, mType, (void * *) & (*item), PR_IOITEM_MIN_SIZE+3*size+3);
   1028         if (pr->outOfMemory) return;
   1029         pr_initItem(this, *item);
   1030         if (itemType == PICODATA_ITEM_TOKEN) {
   1031             (*item)->strci = &((*item)->data[size+1]);
   1032             (*item)->strcis = &((*item)->data[2*size+2]);
   1033             (*item)->strci[0] = 0;
   1034             (*item)->strcis[0] = 0;
   1035         }
   1036     }
   1037     else if ((mType == pr_DynMem) && !inItem) {
   1038         pr_ALLOCATE(this, mType, (void * *) & (*item), PR_IOITEM_MIN_SIZE+size+1);
   1039         if (pr->outOfMemory) return;
   1040         pr_initItem(this, *item);
   1041     }
   1042 
   1043     (*item)->data[0] = 0;
   1044 }
   1045 
   1046 
   1047 static void pr_copyItemContent (picodata_ProcessingUnit this, pr_ioItem * inItem, pr_ioItem * outItem)
   1048 {
   1049     if (outItem != NULL) {
   1050         outItem->next = inItem->next;
   1051         outItem->val = inItem->val;
   1052         outItem->head = inItem->head;
   1053         outItem->suc = inItem->suc;
   1054         outItem->alc = inItem->alc;
   1055         outItem->auc = inItem->auc;
   1056         if (inItem->head.len > 0 ) {
   1057             pr_copyData(outItem->data, inItem->data, inItem->head.len, /*zeroTerm*/TRUE);
   1058             pr_copyData(outItem->strci, inItem->strci, inItem->head.len, /*zeroTerm*/TRUE);
   1059             pr_copyData(outItem->strcis, inItem->strcis, inItem->head.len, /*zeroTerm*/TRUE);
   1060         }
   1061     }
   1062 }
   1063 
   1064 
   1065 static void pr_copyItem (picodata_ProcessingUnit this, pr_MemTypes mType, pr_ioItemPtr inItem, pr_ioItemPtr * outItem)
   1066 {
   1067     pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
   1068 
   1069     if (inItem != NULL) {
   1070         pr_newItem(this, mType,& (*outItem), inItem->head.type, inItem->head.len, FALSE);
   1071         if (pr->outOfMemory) return;
   1072         pr_copyItemContent(this, inItem, *outItem);
   1073     }
   1074     else {
   1075         outItem = NULL;
   1076     }
   1077 }
   1078 
   1079 
   1080 static void pr_startItemList (pr_ioItemPtr * firstItem, pr_ioItemPtr * lastItem)
   1081 {
   1082   *firstItem = NULL;
   1083   *lastItem = NULL;
   1084 }
   1085 
   1086 
   1087 static void pr_appendItem (picodata_ProcessingUnit this, pr_ioItemPtr * firstItem, pr_ioItemPtr * lastItem, pr_ioItemPtr item)
   1088 {
   1089     if (item != NULL) {
   1090         item->next = NULL;
   1091         if ((*lastItem) == NULL) {
   1092             *firstItem = item;
   1093         } else {
   1094             (*lastItem)->next = item;
   1095         }
   1096         (*lastItem) = item;
   1097     }
   1098 }
   1099 
   1100 
   1101 static void pr_disposeItem (picodata_ProcessingUnit this, pr_ioItemPtr * item)
   1102 {
   1103     if ((*item) != NULL) {
   1104         pr_DEALLOCATE(this, pr_DynMem, (void * *) & (*item));
   1105     }
   1106 }
   1107 
   1108 
   1109 static void pr_putItem (picodata_ProcessingUnit this,  pr_subobj_t * pr,
   1110                         pr_ioItemPtr * first, pr_ioItemPtr * last,
   1111                         picoos_uint8 itemType, picoos_uint8 info1, picoos_uint8 info2,
   1112                         picoos_uint16 val,
   1113                         picoos_uchar str[])
   1114 {
   1115     picoos_int32 i;
   1116     pr_ioItemPtr item;
   1117 
   1118     pr->tmpItem.next = NULL;
   1119     pr->tmpItem.val = 0;
   1120     pr->tmpItem.head.type = itemType;
   1121     pr->tmpItem.head.info1 = info1;
   1122     pr->tmpItem.head.info2 = info2;
   1123 
   1124     pr_initItem(this, &pr->tmpItem);
   1125     switch (itemType) {
   1126     case PICODATA_ITEM_CMD:
   1127         switch (info1) {
   1128         case PICODATA_ITEMINFO1_CMD_CONTEXT:
   1129         case PICODATA_ITEMINFO1_CMD_VOICE:
   1130         case PICODATA_ITEMINFO1_CMD_MARKER:
   1131         case PICODATA_ITEMINFO1_CMD_PLAY:
   1132         case PICODATA_ITEMINFO1_CMD_SAVE:
   1133         case PICODATA_ITEMINFO1_CMD_UNSAVE:
   1134         case PICODATA_ITEMINFO1_CMD_PROSDOMAIN:
   1135             pr->tmpItem.head.len = picoos_strlen((picoos_char*)str);
   1136             for (i=0; i<pr->tmpItem.head.len; i++) {
   1137                 pr->tmpItem.data[i] = str[i];
   1138             }
   1139             pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & item);
   1140             if (pr->outOfMemory) return;
   1141             pr_appendItem(this, & (*first),& (*last),item);
   1142             break;
   1143         case PICODATA_ITEMINFO1_CMD_IGNSIG:
   1144         case PICODATA_ITEMINFO1_CMD_IGNORE:
   1145         case PICODATA_ITEMINFO1_CMD_FLUSH:
   1146             pr->tmpItem.head.len = 0;
   1147             pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & item);
   1148             if (pr->outOfMemory) return;
   1149             pr_appendItem(this, & (*first),& (*last),item);
   1150             break;
   1151         case PICODATA_ITEMINFO1_CMD_SPEED:
   1152         case PICODATA_ITEMINFO1_CMD_PITCH:
   1153         case PICODATA_ITEMINFO1_CMD_VOLUME:
   1154         case PICODATA_ITEMINFO1_CMD_SPELL:
   1155         case PICODATA_ITEMINFO1_CMD_SIL:
   1156             pr->tmpItem.head.len = 2;
   1157             pr->tmpItem.data[0] = val % 256;
   1158             pr->tmpItem.data[1] = val / 256;
   1159             pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & item);
   1160             if (pr->outOfMemory) return;
   1161             pr_appendItem(this, & (*first),& (*last),item);
   1162            break;
   1163         case PICODATA_ITEMINFO1_CMD_PHONEME:
   1164             PICODBG_WARN(("phoneme command not yet implemented"));
   1165             break;
   1166         default:
   1167             PICODBG_WARN(("pr_putItem: unknown command type"));
   1168         }
   1169         break;
   1170     case PICODATA_ITEM_TOKEN:
   1171         pr->tmpItem.head.len = picoos_strlen((picoos_char*)str);
   1172         for (i=0; i<pr->tmpItem.head.len; i++) {
   1173             pr->tmpItem.data[i] = str[i];
   1174         }
   1175         pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & item);
   1176         if (pr->outOfMemory) return;
   1177         pr_appendItem(this, & (*first),& (*last),item);
   1178         break;
   1179      default:
   1180          PICODBG_WARN(("pr_putItem: unknown item type"));
   1181     }
   1182 }
   1183 
   1184 
   1185 static void pr_appendItemToOutItemList (picodata_ProcessingUnit this, pr_subobj_t * pr,
   1186                                         pr_ioItemPtr * firstItem, pr_ioItemPtr * lastItem, pr_ioItemPtr item)
   1187 {
   1188     pr_ioItemPtr litem;
   1189     picoos_int32 li;
   1190     picoos_int32 li2;
   1191     picoos_int32 lid;
   1192     picoos_int32 ln;
   1193     picoos_int32 ln2;
   1194     picoos_uint8 ltype;
   1195     picoos_int8 lsubtype;
   1196     picoos_uchar lstr[10];
   1197     picoos_bool ldone;
   1198 
   1199     item->next = NULL;
   1200     if (((pr->spellMode != 0) && (item->head.type == PICODATA_ITEM_TOKEN) && (item->head.info1 != PICODATA_ITEMINFO1_TOKTYPE_SPACE))) {
   1201         li = 0;
   1202         ln = pr_strlen(item->data);
   1203         while (li < ln) {
   1204             ln2 = picobase_det_utf8_length(item->data[li]);
   1205             for (li2 = 0; li2<ln2; li2++) {
   1206                 lstr[li2] = item->data[li];
   1207                 li++;
   1208             }
   1209             lstr[ln2] = 0;
   1210             lid = picoktab_graphOffset(pr->graphs, lstr);
   1211             if ((lid > 0) && picoktab_getIntPropTokenType(pr->graphs, lid, &ltype) &&
   1212                 ((ltype == PICODATA_ITEMINFO1_TOKTYPE_LETTERV) /*|| (ltype == PICODATA_ITEMINFO1_TOKTYPE_DIGIT)*/)) {
   1213                 ln2 = pr_strcat(lstr,(picoos_uchar*)SPEC_CHAR);
   1214                 picoktab_getIntPropTokenSubType(pr->graphs,lid, &lsubtype);
   1215             }
   1216             else {
   1217                 ltype = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
   1218                 lsubtype =  -(1);
   1219             }
   1220             pr_newItem(this, pr_DynMem,& litem, PICODATA_ITEM_TOKEN, ln2, /*inItem*/FALSE);
   1221             if (pr->outOfMemory) return;
   1222             litem->head.type = PICODATA_ITEM_TOKEN;
   1223             litem->head.info1 = item->head.info1;
   1224             litem->head.info2 = item->head.info2;
   1225             pr_strcpy(litem->data, lstr);
   1226             litem->data[ln2] = 0;
   1227             litem->head.len = ln2;
   1228             if (litem->head.info1 == PICODATA_ITEMINFO1_TOKTYPE_DIGIT) {
   1229                 litem->val = tok_tokenDigitStrToInt(this, pr, litem->data);
   1230             } else {
   1231                 litem->val = 0;
   1232             }
   1233             picobase_lowercase_utf8_str(litem->data,litem->strci,PR_MAX_DATA_LEN, &ldone);
   1234             pr_firstLetterToLowerCase(litem->data,litem->strcis);
   1235             litem->alc = picobase_is_utf8_lowercase(litem->data,PR_MAX_DATA_LEN);
   1236             litem->auc = picobase_is_utf8_uppercase(litem->data,PR_MAX_DATA_LEN);
   1237             litem->suc = pr_isSUC(litem->data);
   1238 
   1239             pr_appendItem(this, firstItem, lastItem, litem);
   1240             if (pr->spellMode == PR_SPELL_WITH_SENTENCE_BREAK) {
   1241                 pr_newItem(this, pr_DynMem,& litem, PICODATA_ITEM_TOKEN, 2, /*inItem*/FALSE);
   1242                 if (pr->outOfMemory) return;
   1243                 litem->head.type = PICODATA_ITEM_TOKEN;
   1244                 litem->head.info1 = PICODATA_ITEMINFO1_TOKTYPE_CHAR;
   1245                 litem->head.info2 =  -1;
   1246                 litem->head.len =  1;
   1247                 litem->data[0] = ',';
   1248                 litem->data[1] = 0;
   1249                 litem->strci[0] = ',';
   1250                 litem->strci[1] = 0;
   1251                 litem->strcis[0] = ',';
   1252                 litem->strcis[1] = 0;
   1253                 litem->val = 0;
   1254                 pr_appendItem(this, firstItem, lastItem, litem);
   1255             } else if (pr->spellMode == PR_SPELL_WITH_SENTENCE_BREAK) {
   1256                 pr_newItem(this, pr_DynMem,& litem, PICODATA_ITEM_CMD, 0, /*inItem*/FALSE);
   1257                 if (pr->outOfMemory) return;
   1258                 litem->head.type = PICODATA_ITEM_CMD;
   1259                 litem->head.info1 = PICODATA_ITEMINFO1_CMD_FLUSH;
   1260                 litem->head.info2 = PICODATA_ITEMINFO2_NA;
   1261                 litem->head.len = 0;
   1262                 pr_appendItem(this, firstItem, lastItem,litem);
   1263             } else if (pr->spellMode > 0) {
   1264                 pr_newItem(this, pr_DynMem,& litem, PICODATA_ITEM_CMD, 2, /*inItem*/FALSE);
   1265                 if (pr->outOfMemory) return;
   1266                 litem->head.type = PICODATA_ITEM_CMD;
   1267                 litem->head.info1 = PICODATA_ITEMINFO1_CMD_SIL;
   1268                 litem->head.info2 = PICODATA_ITEMINFO2_NA;
   1269                 litem->head.len = 2;
   1270                 litem->data[0] = pr->spellMode % 256;
   1271                 litem->data[1] = pr->spellMode / 256;
   1272                 pr_appendItem(this, firstItem, lastItem, litem);
   1273             }
   1274         }
   1275         pr_disposeItem(this, & item);
   1276     } else if (pr_isCmdType(item, PICODATA_ITEMINFO1_CMD_SPELL) && pr_isCmdInfo2(item, PICODATA_ITEMINFO2_CMD_START)) {
   1277         pr->spellMode = item->data[0]+256*item->data[1];
   1278         pr_disposeItem(this, & item);
   1279     } else if (pr_isCmdType(item, PICODATA_ITEMINFO1_CMD_SPELL) && pr_isCmdInfo2(item, PICODATA_ITEMINFO2_CMD_END)) {
   1280         pr->spellMode = 0;
   1281         pr_disposeItem(this, & item);
   1282     } else {
   1283         pr_appendItem(this, firstItem,lastItem,item);
   1284     }
   1285 }
   1286 
   1287 
   1288 /* *****************************************************************************/
   1289 
   1290 static pr_OutItemVarPtr pr_findVariable (pr_OutItemVarPtr vars, picoos_int32 id)
   1291 {
   1292     while ((vars != NULL) && (vars->id != id)) {
   1293         vars = vars->next;
   1294     }
   1295     if ((vars != NULL)) {
   1296         return vars;
   1297     } else {
   1298         return NULL;
   1299     }
   1300 }
   1301 
   1302 
   1303 static void pr_genCommands (picodata_ProcessingUnit this, pr_subobj_t * pr,
   1304                             picokpr_Preproc network, picokpr_OutItemArrOffset outitem, pr_OutItemVarPtr vars, pr_ioItemPtr * first, pr_ioItemPtr * last)
   1305 {
   1306 
   1307     pr_ioItemPtr litem;
   1308     pr_OutItemVarPtr lvar;
   1309     picoos_uint8 lcmd;
   1310     picoos_uint8 linfo2;
   1311     picoos_bool ldone;
   1312 #if 0
   1313     picoos_int32 lphontype;
   1314 #endif
   1315     picokpr_VarStrPtr lstrp;
   1316     picoos_int32 lnum;
   1317     pr_ioItemPtr lf;
   1318     pr_ioItemPtr ll;
   1319     picoos_int32 lf0beg;
   1320     picoos_int32 lf0end;
   1321     ShortStrParam lxfadebeg;
   1322     ShortStrParam lxfadeend;
   1323     picoos_bool lout;
   1324     picoos_int32 ltype;
   1325     picoos_int32 argOfs;
   1326     picoos_int32 nextOfs;
   1327     picoos_int32 nextOfs2;
   1328     picoos_int32 nextOfs3;
   1329     picoos_int32 nextOfs4;
   1330     picoos_uchar alphabet[32];
   1331 
   1332     lcmd = 0;
   1333     lnum = 0;
   1334     litem = NULL;
   1335     ltype = picokpr_getOutItemType(network, outitem);
   1336     switch (ltype) {
   1337         case PR_OIgnore:
   1338             if (picokpr_getOutItemVal(network, outitem) == 0) {
   1339                 pr_putItem(this, pr, & (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_IGNORE,PICODATA_ITEMINFO2_CMD_START,0,(picoos_uchar*)"");
   1340             } else {
   1341                 pr_putItem(this, pr, & (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_IGNORE,PICODATA_ITEMINFO2_CMD_END,0,(picoos_uchar*)"");
   1342             }
   1343             break;
   1344         case PR_OPitch:   case PR_OSpeed:   case PR_OVolume:
   1345             switch (ltype) {
   1346                 case PR_OPitch:
   1347                     lcmd = PICODATA_ITEMINFO1_CMD_PITCH;
   1348                     lnum = PICO_PITCH_DEFAULT;
   1349                     break;
   1350                 case PR_OSpeed:
   1351                     lcmd = PICODATA_ITEMINFO1_CMD_SPEED;
   1352                     lnum = PICO_SPEED_DEFAULT;
   1353                     break;
   1354                 case PR_OVolume:
   1355                     lcmd = PICODATA_ITEMINFO1_CMD_VOLUME;
   1356                     lnum = PICO_VOLUME_DEFAULT;
   1357                     break;
   1358             default:
   1359                 break;
   1360             }
   1361             if ((picokpr_getOutItemArgOfs(network, outitem) != 0)) {
   1362                 switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
   1363                     case PR_OVal:
   1364                         pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,PICODATA_ITEMINFO2_CMD_ABSOLUTE, picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)),(picoos_uchar*)"");
   1365                         break;
   1366                     case PR_OVar:
   1367                         lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
   1368                         if ((lvar != NULL) && (lvar->first != NULL) && (lvar->first->head.type == PICODATA_ITEM_TOKEN)) {
   1369                             pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,PICODATA_ITEMINFO2_CMD_ABSOLUTE,picoos_atoi((picoos_char*)lvar->first->data),(picoos_uchar*)"");
   1370                         }
   1371                         break;
   1372                 default:
   1373                     pr_startItemList(& lf,& ll);
   1374                     pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
   1375                     if (pr->outOfMemory) return;
   1376                     if (((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN))) {
   1377                         pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,PICODATA_ITEMINFO2_CMD_ABSOLUTE,picoos_atoi((picoos_char*)lf->data),(picoos_uchar*)"");
   1378                     }
   1379                     break;
   1380                 }
   1381             } else {
   1382                 pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,PICODATA_ITEMINFO2_CMD_ABSOLUTE,lnum,(picoos_uchar*)"");
   1383             }
   1384             break;
   1385 
   1386         case PR_OPhonSVOXPA:   case PR_OPhonSAMPA:
   1387             if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
   1388                 if (ltype == PR_OPhonSVOXPA) {
   1389                     picoos_strlcpy(alphabet, PICODATA_SVOXPA, sizeof(alphabet));
   1390                 }
   1391                 else {
   1392                     picoos_strlcpy(alphabet, PICODATA_SAMPA, sizeof(alphabet));
   1393                 }
   1394                 pr_startItemList(& lf,& ll);
   1395                 pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
   1396                 if (pr->outOfMemory) return;
   1397                 if (lf != NULL) {
   1398                     ldone = FALSE;
   1399                     if (lf->head.type == PICODATA_ITEM_TOKEN) {
   1400                         if (picodata_mapPAStrToPAIds(pr->transducer, this->common, pr->xsampa_parser, pr->svoxpa_parser, pr->xsampa2svoxpa_mapper, lf->data, alphabet, pr->tmpStr1, sizeof(pr->tmpStr1)-1) == PICO_OK) {
   1401                             pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME,
   1402                                 PICODATA_ITEMINFO2_CMD_START, 0, pr->tmpStr1);
   1403                             ldone = TRUE;
   1404                         }
   1405                         else {
   1406                             PICODBG_WARN(("cannot map phonetic string '%s'; synthesizeing text instead", lf->data));
   1407                             picoos_emRaiseWarning(this->common->em, PICO_ERR_OTHER,(picoos_char*)"", (picoos_char*)"cannot map phonetic string '%s'; synthesizing text instead", lf->data);
   1408                         }
   1409                     }
   1410                     if (ldone) {
   1411                         lf = lf->next;
   1412                         while (lf != NULL) {
   1413                             if (lf->head.type == PICODATA_ITEM_TOKEN) {
   1414                                 pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_TOKEN, PICODATA_ITEMINFO1_CMD_PHONEME,
   1415                                                PICODATA_ITEMINFO2_CMD_END, 0, (picoos_char *)"");
   1416                             }
   1417                             lf = lf->next;
   1418                         }
   1419                         pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME,
   1420                             PICODATA_ITEMINFO2_CMD_END, 0, (picoos_char *)"");
   1421                     }
   1422                 }
   1423             }
   1424             break;
   1425 
   1426         case PR_OSent:
   1427             pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
   1428             break;
   1429         case PR_OPara:
   1430             pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
   1431             if (picokpr_getOutItemVal(network, outitem) == 1) {
   1432                 pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, PICO_PARAGRAPH_PAUSE_DUR, (picoos_uchar*)"");
   1433             }
   1434             break;
   1435         case PR_OBreak:
   1436             if ((picokpr_getOutItemArgOfs(network, outitem) != 0)) {
   1437                 switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
   1438                     case PR_OVal:
   1439                         pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)), (picoos_uchar*)"");
   1440                         break;
   1441                     case PR_OVar:
   1442                         lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
   1443                         if ((lvar != NULL) && (lvar->first != NULL) && (lvar->first->head.type == PICODATA_ITEM_TOKEN)) {
   1444                             pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, picoos_atoi((picoos_char*)lvar->first->data), (picoos_uchar*)"");
   1445                         }
   1446                         break;
   1447                 default:
   1448                     pr_startItemList(& lf,& ll);
   1449                     pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
   1450                     if (pr->outOfMemory) return;
   1451                     if (((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN))) {
   1452                         pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, picoos_atoi((picoos_char*)lf->data), (picoos_uchar*)"");
   1453                     }
   1454                     break;
   1455                 }
   1456             }
   1457             break;
   1458         case PR_OVoice:   case PR_OContext:   case PR_OMark:
   1459             if (picokpr_getOutItemType(network, outitem) == PR_OVoice) {
   1460                 lcmd = PICODATA_ITEMINFO1_CMD_VOICE;
   1461                 pr->tmpStr1[0] = 0;
   1462                 lnum = 1;
   1463             } else if (picokpr_getOutItemType(network, outitem) == PR_OContext) {
   1464                 lcmd = PICODATA_ITEMINFO1_CMD_CONTEXT;
   1465                 pr_strcpy(pr->tmpStr1, (picoos_uchar*)PICO_CONTEXT_DEFAULT);
   1466                 lnum = 1;
   1467             } else if ((picokpr_getOutItemType(network, outitem) == PR_OMark)) {
   1468                 lcmd = PICODATA_ITEMINFO1_CMD_MARKER;
   1469                 pr->tmpStr1[0] = 0;
   1470                 lnum = 0;
   1471             }
   1472             if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
   1473                 switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
   1474                     case PR_OVar:
   1475                         lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
   1476                         if (lvar != NULL) {
   1477                             litem = lvar->first;
   1478                         }
   1479                         pr->tmpStr1[0] = 0;
   1480                         while (litem != NULL) {
   1481                             if (litem->head.type == PICODATA_ITEM_TOKEN) {
   1482                                 pr_strcat(pr->tmpStr1,litem->data);
   1483                             }
   1484                             litem = litem->next;
   1485                         }
   1486                         pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,lnum,0,pr->tmpStr1);
   1487                         break;
   1488                     case PR_OStr:
   1489                         if (picokpr_getOutItemStrOfs(network, picokpr_getOutItemArgOfs(network, outitem)) != 0) {
   1490                             lstrp = picokpr_getOutItemStr(network, picokpr_getOutItemArgOfs(network, outitem));
   1491                             pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,lstrp);
   1492                         }
   1493                         break;
   1494                 default:
   1495                     pr_startItemList(& lf,& ll);
   1496                     pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
   1497                     if (pr->outOfMemory) return;
   1498                     if (((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN))) {
   1499                         pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,lf->data);
   1500                     }
   1501                     break;
   1502                 }
   1503             } else {
   1504                 pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,pr->tmpStr1);
   1505             }
   1506             break;
   1507         case PR_OGenFile:
   1508             if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
   1509                 lcmd = PICODATA_ITEMINFO1_CMD_SAVE;
   1510             } else {
   1511                 lcmd = PICODATA_ITEMINFO1_CMD_UNSAVE;
   1512             }
   1513             pr->tmpStr1[0] = 0;
   1514             lnum = 0;
   1515             if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
   1516                 switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
   1517                     case PR_OVar:
   1518                         lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
   1519                         if (lvar != NULL) {
   1520                             litem = lvar->first;
   1521                         }
   1522                         pr->tmpStr1[0] = 0;
   1523                         while (litem != NULL) {
   1524                             if (litem->head.type == PICODATA_ITEM_TOKEN) {
   1525                                 pr_strcat(pr->tmpStr1,litem->data);
   1526                             }
   1527                             litem = litem->next;
   1528                         }
   1529                         if ((lnum = picodata_getPuTypeFromExtension(pr->tmpStr1, /*input*/FALSE)) != PICODATA_ITEMINFO2_CMD_TO_UNKNOWN) {
   1530                             if (pr->saveFile[0] != 0) {
   1531                               pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE,
   1532                                   picodata_getPuTypeFromExtension(pr->saveFile, /*input*/FALSE),0,pr->saveFile);
   1533                             }
   1534                             pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,lnum,0,pr->tmpStr1);
   1535                             pr_strcpy(pr->saveFile, pr->tmpStr1);
   1536                         }
   1537                         break;
   1538                     case PR_OStr:
   1539                         if (picokpr_getOutItemStrOfs(network, picokpr_getOutItemArgOfs(network, outitem)) != 0) {
   1540                             lstrp = picokpr_getOutItemStr(network, picokpr_getOutItemArgOfs(network, outitem));
   1541                             if ((lnum = picodata_getPuTypeFromExtension(lstrp, /*input*/FALSE)) != PICODATA_ITEMINFO2_CMD_TO_UNKNOWN) {
   1542                                 if (pr->saveFile[0] != 0) {
   1543                                     pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE,
   1544                                         picodata_getPuTypeFromExtension(pr->saveFile, /*input*/FALSE),0,pr->saveFile);
   1545                                 }
   1546                                 pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,lnum,0,lstrp);
   1547                                 pr_strcpy(pr->saveFile, lstrp);
   1548                             }
   1549                             pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,lstrp);
   1550                         }
   1551                         break;
   1552                 default:
   1553                     pr_startItemList(& lf,& ll);
   1554                     pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
   1555                     if (pr->outOfMemory) return;
   1556                     if (((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN))) {
   1557                         if ((lnum = picodata_getPuTypeFromExtension(lf->data, /*input*/FALSE)) != PICODATA_ITEMINFO2_CMD_TO_UNKNOWN) {
   1558                             if (pr->saveFile[0] != 0) {
   1559                                 pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE,
   1560                                     picodata_getPuTypeFromExtension(pr->saveFile, /*input*/FALSE),0,pr->saveFile);
   1561                             }
   1562                             pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,lnum,0,lf->data);
   1563                             pr_strcpy(pr->saveFile, lf->data);
   1564                         }
   1565                     }
   1566                     break;
   1567                 }
   1568 /*
   1569             } else {
   1570                 pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,pr->tmpStr1);
   1571 */
   1572             }
   1573             break;
   1574         case PR_OUseSig:   case PR_OPlay:
   1575             lout = FALSE;
   1576             lf0beg =  -(1);
   1577             lf0end =  -(1);
   1578             lxfadebeg[0] = 0;
   1579             lxfadeend[0] = 0;
   1580             pr->tmpStr1[0] = 0;
   1581             if ((picokpr_getOutItemType(network, outitem) == PR_OUseSig)) {
   1582                 lcmd = PICODATA_ITEMINFO1_CMD_IGNSIG;
   1583             } else {
   1584                 lcmd = PICODATA_ITEMINFO1_CMD_IGNORE;
   1585             }
   1586             if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
   1587                 linfo2 = PICODATA_ITEMINFO2_CMD_START;
   1588             } else {
   1589                 linfo2 = PICODATA_ITEMINFO2_CMD_END;
   1590             }
   1591             if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
   1592                 switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
   1593                     case PR_OVar:
   1594                         lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
   1595                         if (lvar != NULL) {
   1596                             litem = lvar->first;
   1597                         }
   1598                         pr->tmpStr1[0] = 0;
   1599                         while (litem != NULL) {
   1600                             if (litem->head.type == PICODATA_ITEM_TOKEN) {
   1601                                 pr_strcat(pr->tmpStr1,litem->data);
   1602                             }
   1603                             litem = litem->next;
   1604                         }
   1605                         pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PLAY,
   1606                             picodata_getPuTypeFromExtension(pr->tmpStr1, /*input*/TRUE),0, pr->tmpStr1);
   1607                         lout = TRUE;
   1608                         break;
   1609                     case PR_OStr:
   1610                         if (picokpr_getOutItemStrOfs(network, picokpr_getOutItemArgOfs(network, outitem)) != 0) {
   1611                             lstrp = picokpr_getOutItemStr(network, picokpr_getOutItemArgOfs(network, outitem));
   1612                             pr_strcpy(pr->tmpStr1, lstrp);
   1613                             lout = TRUE;
   1614                         }
   1615                         break;
   1616                 default:
   1617                     pr_startItemList(& lf,& ll);
   1618                     pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
   1619                     if (pr->outOfMemory) return;
   1620                     if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
   1621                         pr_strcpy(pr->tmpStr1, lf->data);
   1622                         lout = TRUE;
   1623                     }
   1624                     break;
   1625                 }
   1626             }
   1627             argOfs = picokpr_getOutItemArgOfs(network, outitem);
   1628             if (argOfs != 0) {
   1629                 nextOfs = picokpr_getOutItemNextOfs(network, outitem);
   1630                 if (nextOfs != 0) {
   1631                     if (picokpr_getOutItemType(network, nextOfs) == PR_OVal) {
   1632                         lf0beg = picokpr_getOutItemVal(network, nextOfs);
   1633                     }
   1634                     nextOfs2 = picokpr_getOutItemNextOfs(network, nextOfs);
   1635                     if (nextOfs2 != 0) {
   1636                         if (picokpr_getOutItemType(network, nextOfs2) == PR_OVal) {
   1637                             lf0end = picokpr_getOutItemVal(network, nextOfs2);
   1638                         }
   1639                         nextOfs3 = picokpr_getOutItemNextOfs(network, nextOfs2);
   1640                         if (nextOfs3 != 0) {
   1641                             if ((picokpr_getOutItemType(network, nextOfs3) == PR_OStr) && (picokpr_getOutItemStrOfs(network, nextOfs3) != 0)) {
   1642                                 lstrp = picokpr_getOutItemStr(network, nextOfs3);
   1643                                 pr_strcpy(lxfadebeg, lstrp);
   1644                             }
   1645                             nextOfs4 = picokpr_getOutItemNextOfs(network, nextOfs3);
   1646                             if (nextOfs4 != 0) {
   1647                                 if ((picokpr_getOutItemType(network, nextOfs4) == PR_OStr) && (picokpr_getOutItemStrOfs(network, nextOfs4) != 0)) {
   1648                                     lstrp = picokpr_getOutItemStr(network, nextOfs4);
   1649                                     pr_strcpy(lxfadeend, lstrp);
   1650                                 }
   1651                             }
   1652                         }
   1653                     }
   1654                 }
   1655             }
   1656             if (lout) {
   1657                 pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,PICODATA_ITEMINFO1_CMD_PLAY,
   1658                     picodata_getPuTypeFromExtension(pr->tmpStr1, /*input*/TRUE),0,pr->tmpStr1);
   1659             }
   1660             pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,linfo2,0,(picoos_uchar*)"");
   1661             break;
   1662     default:
   1663         PICODBG_INFO(("unknown command"));
   1664         break;
   1665     }
   1666 }
   1667 
   1668 
   1669 static void pr_getOutputItemList (picodata_ProcessingUnit this,
   1670                                   pr_subobj_t * pr,
   1671                                   picokpr_Preproc network,
   1672                                   picokpr_OutItemArrOffset outitem,
   1673                                   pr_OutItemVarPtr vars,
   1674                                   pr_ioItemPtr * first,
   1675                                   pr_ioItemPtr * last)
   1676 {
   1677 
   1678     picokpr_OutItemArrOffset lo;
   1679     picoos_int32 llen;
   1680     picoos_int32 llen2;
   1681     picokpr_VarStrPtr lstrp;
   1682     picoos_int32 lval32;
   1683     picoos_int32 li;
   1684     picoos_int32 li2;
   1685     picoos_int32 ln;
   1686     picoos_int32 ln2;
   1687     pr_ioItemPtr litem2;
   1688     pr_ioItemPtr lf;
   1689     pr_ioItemPtr ll;
   1690     picoos_int32 lid;
   1691     picoos_uint8 ltype;
   1692     picoos_int8 lsubtype;
   1693     pr_OutItemVarPtr lvar;
   1694     picoos_int32 lspellmode;
   1695     picoos_int32 largOfs;
   1696     picoos_int32 lnextOfs;
   1697 
   1698 
   1699     lo = outitem;
   1700     while (lo != 0) {
   1701         switch (picokpr_getOutItemType(network, lo)) {
   1702             case PR_OStr:
   1703                 lstrp = picokpr_getOutItemStr(network, lo);
   1704                 if (pr->outOfMemory) return;
   1705                 pr_initItem(this, &pr->tmpItem);
   1706                 pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
   1707                 pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
   1708                 pr->tmpItem.head.info2 =  -1;
   1709                 pr->tmpItem.head.len = pr_strcpy(pr->tmpItem.data, lstrp);
   1710                 pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
   1711                 if (pr->outOfMemory) return;
   1712                 pr_appendItem(this, & (*first),& (*last),litem2);
   1713                 break;
   1714             case PR_OVar:
   1715                 lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, lo));
   1716                 if (lvar != NULL) {
   1717                     lf = lvar->first;
   1718                 } else {
   1719                     lf = NULL;
   1720                 }
   1721                 while (lf != NULL) {
   1722                     pr_copyItem(this, pr_WorkMem,& (*lf),& litem2);
   1723                     if (pr->outOfMemory) return;
   1724                     pr_appendItem(this, & (*first),& (*last),litem2);
   1725                     lf = lf->next;
   1726                 }
   1727                 break;
   1728             case PR_OSpell:
   1729                 lspellmode = PR_SPELL;
   1730                 largOfs = picokpr_getOutItemArgOfs(network, lo);
   1731                 if ((largOfs!= 0) && ((lnextOfs = picokpr_getOutItemNextOfs(network, largOfs)) != 0)) {
   1732                     lspellmode = picokpr_getOutItemVal(network, lnextOfs);
   1733                 }
   1734                 pr_startItemList(& lf,& ll);
   1735                 pr_getOutputItemList(this, pr, network,largOfs,vars,& lf,& ll);
   1736                 if (pr->outOfMemory) return;
   1737                 while (lf != NULL) {
   1738                     switch (lf->head.type) {
   1739                         case PICODATA_ITEM_TOKEN:
   1740                             li = 0;
   1741                             ln = pr_strlen(lf->data);
   1742                             while (li < ln) {
   1743                                 pr_initItem(this, &pr->tmpItem);
   1744                                 if (pr->outOfMemory) return;
   1745                                 pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
   1746                                 pr->tmpItem.head.info1 = lf->head.info1;
   1747                                 pr->tmpItem.head.info2 = lf->head.info2;
   1748                                 pr->tmpItem.head.len = picobase_det_utf8_length(lf->data[li]);
   1749                                 for (li2 = 0; li2 < pr->tmpItem.head.len; li2++) {
   1750                                     pr->tmpItem.data[li2] = lf->data[li];
   1751                                     li++;
   1752                                 }
   1753                                 pr->tmpItem.data[pr->tmpItem.head.len] = 0;
   1754                                 pr->tmpItem.val = 0;
   1755                                 lid = picoktab_graphOffset(pr->graphs,pr->tmpItem.data);
   1756                                 if (lid > 0) {
   1757                                     if (picoktab_getIntPropTokenType(pr->graphs, lid, &ltype)) {
   1758                                         if ((ltype == PICODATA_ITEMINFO1_TOKTYPE_LETTERV)/* || (ltype == PICODATA_ITEMINFO1_TOKTYPE_DIGIT)*/) {
   1759                                             pr->tmpItem.head.len = pr_strcat(pr->tmpItem.data,(picoos_uchar*)SPEC_CHAR);
   1760                                         }
   1761                                     }
   1762                                     picoktab_getIntPropTokenSubType(pr->graphs, lid, &lsubtype);
   1763                                 } else {
   1764                                     ltype = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
   1765                                     lsubtype =  -(1);
   1766                                 }
   1767                                 pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
   1768                                 if (pr->outOfMemory) return;
   1769                                 pr_appendItem(this, & (*first),& (*last), litem2);
   1770                                 if (lspellmode == PR_SPELL_WITH_PHRASE_BREAK) {
   1771                                     pr_initItem(this, &pr->tmpItem);
   1772                                     pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
   1773                                     pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_CHAR;
   1774                                     pr->tmpItem.head.info2 = lsubtype;
   1775                                     pr->tmpItem.head.len = 1;
   1776                                     pr->tmpItem.data[0] = ',';
   1777                                     pr->tmpItem.data[1] = 0;
   1778                                     pr->tmpItem.val = 0;
   1779                                     pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
   1780                                     if (pr->outOfMemory) return;
   1781                                     pr_appendItem(this, & (*first),& (*last),litem2);
   1782                                 } else if (lspellmode == PR_SPELL_WITH_SENTENCE_BREAK) {
   1783                                     pr_initItem(this, &pr->tmpItem);
   1784                                     pr->tmpItem.head.type = PICODATA_ITEM_CMD;
   1785                                     pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_CMD_FLUSH;
   1786                                     pr->tmpItem.head.info2 = PICODATA_ITEMINFO2_NA;
   1787                                     pr->tmpItem.head.len = 0;
   1788                                     pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
   1789                                     if (pr->outOfMemory) return;
   1790                                     pr_appendItem(this, & (*first),& (*last),litem2);
   1791                                 } else if (lspellmode > 0) {
   1792                                     pr_initItem(this, &pr->tmpItem);
   1793                                     pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
   1794                                     pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_CHAR;
   1795                                     pr->tmpItem.head.info2 = lsubtype;
   1796                                     pr->tmpItem.head.len = 1;
   1797                                     pr->tmpItem.data[0] = ',';
   1798                                     pr->tmpItem.data[1] = 0;
   1799                                     pr->tmpItem.val = 0;
   1800                                     pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
   1801                                     if (pr->outOfMemory) return;
   1802                                     pr_appendItem(this, & (*first),& (*last),litem2);
   1803                                 }
   1804                             }
   1805                             break;
   1806                     default:
   1807                         pr_copyItem(this, pr_WorkMem,& (*lf),& litem2);
   1808                         if (pr->outOfMemory) return;
   1809                         pr_appendItem(this, & (*first),& (*last),litem2);
   1810                         break;
   1811                     }
   1812                     ll = lf;
   1813                     lf = lf->next;
   1814                     ll->next = NULL;
   1815                 }
   1816                 break;
   1817             case PR_OConcat:
   1818                 pr_startItemList(& lf,& ll);
   1819                 pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, lo),vars,& lf,& ll);
   1820                 if (pr->outOfMemory) return;
   1821                 pr_initItem(this, &pr->tmpItem);
   1822                 pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
   1823                 pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
   1824                 pr->tmpItem.head.info2 =  -(1);
   1825                 pr->tmpItem.head.len = 0;
   1826                 pr->tmpItem.data[0] = 0;
   1827                 pr->tmpItem.val = 0;
   1828                 while (lf != NULL) {
   1829                     switch (lf->head.type) {
   1830                         case PICODATA_ITEM_TOKEN:
   1831                             pr->tmpItem.head.len = pr_strcat(pr->tmpItem.data,lf->data);
   1832                             break;
   1833                         case PICODATA_ITEM_CMD:
   1834                             pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
   1835                             if (pr->outOfMemory) return;
   1836                             pr_appendItem(this, & (*first),& (*last),litem2);
   1837 
   1838                             pr_copyItem(this, pr_WorkMem, lf, &litem2);
   1839                             if (pr->outOfMemory) return;
   1840                             pr_appendItem(this, & (*first),& (*last),litem2);
   1841 
   1842                             pr_initItem(this, &pr->tmpItem);
   1843                             pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
   1844                             pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
   1845                             pr->tmpItem.head.info2 =  -(1);
   1846                             pr->tmpItem.head.len = 0;
   1847                             pr->tmpItem.data[0] = 0;
   1848                             pr->tmpItem.val = 0;
   1849                             break;
   1850                     default:
   1851                         break;
   1852                     }
   1853                     lf = lf->next;
   1854                 }
   1855                 pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
   1856                 if (pr->outOfMemory) return;
   1857                 pr_appendItem(this, & (*first),& (*last),litem2);
   1858                 break;
   1859             case PR_ORomanToCard:
   1860                 pr_startItemList(& lf,& ll);
   1861                 pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, lo),vars,& lf,& ll);
   1862                 if (pr->outOfMemory) return;
   1863                 if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
   1864                     pr_initItem(this, &pr->tmpItem);
   1865                     pr_copyItemContent(this, lf, &pr->tmpItem);
   1866                     if (pr_isLatinNumber(lf->data, & lval32)) {
   1867                         pr_int_to_string(lval32, (picoos_char *)pr->tmpItem.data, PR_MAX_DATA_LEN_Z);
   1868                         pr->tmpItem.head.len = pr_strlen(pr->tmpItem.data);
   1869                         pr->tmpItem.val = lval32;
   1870                     }
   1871                     pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
   1872                     pr_appendItem(this, & (*first),& (*last),litem2);
   1873                 }
   1874                 break;
   1875             case PR_OVal:
   1876                 break;
   1877             case PR_OLeft:
   1878                 pr_startItemList(& lf,& ll);
   1879                 pr_getOutputItemList(this, pr, network,picokpr_getOutItemNextOfs(network, picokpr_getOutItemArgOfs(network, lo)),vars,& lf,& ll);
   1880                 if (pr->outOfMemory) return;
   1881                 if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
   1882                     pr_initItem(this, &pr->tmpItem);
   1883                     pr_copyItemContent(this, lf, &pr->tmpItem);
   1884                     llen = lf->head.len;
   1885                     llen2 = picobase_utf8_length(pr->tmpItem.data,PR_MAX_DATA_LEN);
   1886                     ln = 0;
   1887                     ln2 = 0;
   1888                     largOfs = picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, lo));
   1889                     while ((ln < llen) && (ln2 < llen2) && (ln2 < largOfs)) {
   1890                         ln = ln + picobase_det_utf8_length(pr->tmpItem.data[ln]);
   1891                         ln2++;
   1892                     }
   1893                     pr->tmpItem.data[ln] = 0;
   1894                     pr->tmpItem.head.len = ln;
   1895                     pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
   1896                     if (pr->outOfMemory) return;
   1897                     pr_appendItem(this, & (*first),& (*last),litem2);
   1898                 }
   1899                 break;
   1900             case PR_ORight:
   1901                 pr_startItemList(& lf,& ll);
   1902                 pr_getOutputItemList(this, pr, network,picokpr_getOutItemNextOfs(network, picokpr_getOutItemArgOfs(network, lo)),vars,& lf,& ll);
   1903                 if (pr->outOfMemory) return;
   1904                 if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
   1905                     pr_initItem(this, &pr->tmpItem);
   1906                     pr_copyItemContent(this, lf, & pr->tmpItem);
   1907                     llen = lf->head.len;
   1908                     llen2 = picobase_utf8_length(pr->tmpItem.data,PR_MAX_DATA_LEN);
   1909                     ln = 0;
   1910                     ln2 = 0;
   1911                     while ((ln < llen) && (ln2 < llen2) && (ln2 < (llen2 - picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, lo))))) {
   1912                         ln = ln + picobase_det_utf8_length(pr->tmpItem.data[ln]);
   1913                         ln2++;
   1914                     }
   1915                     pr->tmpItem.head.len = pr_removeSubstring(0,ln,pr->tmpItem.data);
   1916                     pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & litem2);
   1917                     if (pr->outOfMemory) return;
   1918                     pr_appendItem(this, & (*first),& (*last),litem2);
   1919                 }
   1920                 break;
   1921             case PR_OItem:
   1922                 pr_startItemList(& lf,& ll);
   1923                 pr_getOutputItemList(this, pr, network,picokpr_getOutItemNextOfs(network, picokpr_getOutItemArgOfs(network, lo)),vars,& lf,& ll);
   1924                 if (pr->outOfMemory) return;
   1925                 ln = picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, lo));
   1926                 li = 1;
   1927                 while ((li < ln) && (lf != NULL)) {
   1928                     lf = lf->next;
   1929                     li++;
   1930                 }
   1931                 if ((lf != NULL) && (li == ln) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
   1932                     pr_copyItem(this, pr_WorkMem, lf, & litem2);
   1933                     if (pr->outOfMemory) return;
   1934                     pr_appendItem(this, & (*first),& (*last),litem2);
   1935                 }
   1936                 break;
   1937             case PR_ORLZ:
   1938                 pr_startItemList(& lf,& ll);
   1939                 pr_getOutputItemList(this, pr, network, picokpr_getOutItemArgOfs(network, lo),vars,& lf,& ll);
   1940                 if (pr->outOfMemory) return;
   1941                 if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
   1942                     pr_initItem(this, &pr->tmpItem);
   1943                     pr_copyItemContent(this, lf, & pr->tmpItem);
   1944                     li = 0;
   1945                     while ((li < lf->head.len) && (pr->tmpItem.data[li] == '0')) {
   1946                         li++;
   1947                     }
   1948                     pr->tmpItem.head.len = pr_removeSubstring(0,li,pr->tmpItem.data);
   1949                     pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & litem2);
   1950                     if (pr->outOfMemory) return;
   1951                     pr_appendItem(this, & (*first),& (*last),litem2);
   1952                 }
   1953                 break;
   1954             case PR_OIgnore:   case PR_OPitch:   case PR_OSpeed:   case PR_OVolume:   case PR_OPhonSVOXPA:   case PR_OPhonSAMPA:   case PR_OBreak:   case PR_OMark:   case PR_OPara:   case PR_OSent:   case PR_OPlay:
   1955             case PR_OUseSig:   case PR_OGenFile:   case PR_OAudioEdit:   case PR_OContext:   case PR_OVoice:
   1956                 pr_genCommands(this, pr, network,lo,vars,& (*first),& (*last));
   1957                 if (pr->outOfMemory) return;
   1958                 break;
   1959         default:
   1960             PICODBG_INFO(("unkown command"));
   1961             break;
   1962         }
   1963         lo = picokpr_getOutItemNextOfs(network, lo);
   1964     }
   1965 }
   1966 
   1967 
   1968 static picoos_int32 pr_attrVal (picokpr_Preproc network, picokpr_TokArrOffset tok, pr_TokSetEleWP type)
   1969 {
   1970 
   1971     pr_TokSetEleWP tse;
   1972     picoos_int32 n;
   1973     picokpr_TokSetWP set;
   1974 
   1975     n = 0;
   1976     tse = PR_FIRST_TSE_WP;
   1977     set = picokpr_getTokSetWP(network, tok);
   1978     while (tse < type) {
   1979         if (((1<<tse) & set) != 0) {
   1980             n++;
   1981         }
   1982         tse = (pr_TokSetEleWP)((picoos_int32)tse+1);
   1983     }
   1984     return picokpr_getAttrValArrInt32(network, picokpr_getTokAttribOfs(network, tok) + n);
   1985 }
   1986 
   1987 
   1988 static void pr_getOutput (picodata_ProcessingUnit this, pr_subobj_t * pr,
   1989                           picoos_int32 * i, picoos_int32 d, pr_ioItemPtr * o, pr_ioItemPtr * ol)
   1990 {
   1991 
   1992     register struct pr_PathEle * with__0;
   1993     pr_OutItemVarPtr lvars;
   1994     pr_OutItemVarPtr lvar;
   1995     pr_ioItemPtr lit;
   1996     pr_ioItemPtr ldit;
   1997     pr_ioItemPtr ldlit;
   1998     picoos_bool lfirst;
   1999     pr_ioItemPtr lcopy;
   2000     picokpr_TokSetWP wpset;
   2001     picokpr_TokSetNP npset;
   2002     picoos_int32 li;
   2003 
   2004     lfirst = TRUE;
   2005     (*i)++;
   2006     lit = NULL;
   2007     lvars = NULL;
   2008     ldit = NULL;
   2009     ldlit = NULL;
   2010     while ((*i) < pr->rbestpath.rlen) {
   2011         with__0 = & pr->rbestpath.rele[*i];
   2012         li = 0;
   2013         if (*i > 0) {
   2014             while ((li < 127) && (li < pr->rbestpath.rele[*i].rdepth)) {
   2015                 pr->lspaces[li++] = ' ';
   2016             }
   2017         }
   2018         pr->lspaces[li] = 0;
   2019         if (with__0->rprodname != 0) {
   2020             PICODBG_INFO(("pp path  :%s%s(", pr->lspaces, picokpr_getVarStrPtr(with__0->rnetwork,with__0->rprodname)));
   2021         }
   2022         if ((pr->ritems[with__0->ritemid+1] != NULL) && (pr->ritems[with__0->ritemid+1]->head.type == PICODATA_ITEM_TOKEN)) {
   2023             PICODBG_INFO(("pp in (1): %s'%s'", pr->lspaces, pr->ritems[with__0->ritemid+1]->data));
   2024         }
   2025         if ((pr->ritems[with__0->ritemid+1] != NULL)) {
   2026             while ((pr->rinItemList != NULL) && (pr->rinItemList != pr->ritems[with__0->ritemid+1]) && (pr->rinItemList->head.type != PICODATA_ITEM_TOKEN)) {
   2027                 lit = pr->rinItemList;
   2028                 pr->rinItemList = pr->rinItemList->next;
   2029                 lit->next = NULL;
   2030                 pr_copyItem(this, pr_WorkMem,& (*lit),& lcopy);
   2031                 if (pr->outOfMemory) return;
   2032                 pr_disposeItem(this, & lit);
   2033                 pr_appendItem(this, & (*o),& (*ol),lcopy);
   2034             }
   2035             if (pr->rinItemList != NULL) {
   2036                 lit = pr->rinItemList;
   2037                 pr->rinItemList = pr->rinItemList->next;
   2038                 lit->next = NULL;
   2039             } else {
   2040                 lit = NULL;
   2041             }
   2042         }
   2043         wpset = picokpr_getTokSetWP(with__0->rnetwork, with__0->rtok);
   2044         npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
   2045 
   2046         if ((PR_TSE_MASK_PROD & wpset) != 0) {
   2047             if ((PR_TSE_MASK_VAR & wpset) != 0) {
   2048                 lvar = pr_findVariable(lvars,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVar));
   2049                 if (lvar == NULL) {
   2050                     pr_ALLOCATE(this, pr_WorkMem, (void *) & lvar,(sizeof (*lvar)));
   2051                     lvar->next = lvars;
   2052                     lvar->id = pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVar);
   2053                     lvars = lvar;
   2054                 }
   2055                 pr_startItemList(& lvar->first,& lvar->last);
   2056                 pr_getOutput(this, pr, & (*i),(d + 1),& lvar->first,& lvar->last);
   2057                 if (pr->outOfMemory) return;
   2058             } else {
   2059                 pr_startItemList(& ldit,& ldlit);
   2060                 pr_getOutput(this, pr, & (*i),(d + 1),& ldit,& ldlit);
   2061                 if (pr->outOfMemory) return;
   2062             }
   2063             (*i)++;
   2064         } else if ((PR_TSE_MASK_VAR & wpset) != 0) {
   2065             lvar = pr_findVariable(lvars,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVar));
   2066             if (lvar == NULL) {
   2067                 pr_ALLOCATE(this, pr_WorkMem, (void *) & lvar,(sizeof (*lvar)));
   2068                 lvar->next = lvars;
   2069                 lvar->id = pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVar);
   2070                 lvars = lvar;
   2071             }
   2072             if (((PR_TSE_MASK_LEX & wpset) != 0) && ((PR_TSE_MASK_LETTER & npset) == 0)) {
   2073                 if (lfirst) {
   2074                     pr_newItem(this, pr_WorkMem,& lit, PICODATA_ITEM_TOKEN, sizeof(struct pr_ioItem), /*inItem*/FALSE);
   2075                     if (pr->outOfMemory) return;
   2076                     lit->head.type = PICODATA_ITEM_TOKEN;
   2077                     lit->head.info1 = pr->ritems[with__0->ritemid+1]->head.info1;
   2078                     lit->head.info2 = pr->ritems[with__0->ritemid+1]->head.info2;
   2079                     if (pr->ritems[with__0->ritemid+1]->head.info1 == PICODATA_ITEMINFO1_TOKTYPE_SPACE) {
   2080                         lit->head.len = pr_strcpy(lit->data, (picoos_uchar*)"_");
   2081                     } else {
   2082                         lit->head.len = pr_strcpy(lit->data, pr->ritems[with__0->ritemid+1]->data);
   2083                     }
   2084                     lvar->first = lit;
   2085                     lvar->last = lit;
   2086                     lfirst = FALSE;
   2087                 } else {
   2088                     if ((pr->ritems[with__0->ritemid+1]->head.info1 == PICODATA_ITEMINFO1_TOKTYPE_SPACE)) {
   2089                         lvar->last->head.len = pr_strcat(lvar->last->data,(picoos_uchar*)"_");
   2090                     } else {
   2091                         lvar->last->head.len = pr_strcat(lvar->last->data,pr->ritems[with__0->ritemid+1]->data);
   2092                     }
   2093                     lvar->last->head.info1 = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
   2094                     lvar->last->head.info2 =  -(1);
   2095                 }
   2096             } else {
   2097                 lvar->first = pr->ritems[with__0->ritemid+1];
   2098                 lvar->last = pr->ritems[with__0->ritemid+1];
   2099             }
   2100             (*i)++;
   2101         } else if ((PR_TSE_MASK_OUT & wpset) != 0) {
   2102             pr_getOutputItemList(this, pr, with__0->rnetwork,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEOut),lvars,& (*o),& (*ol));
   2103             if (pr->outOfMemory) return;
   2104             (*i)++;
   2105         } else if (((*i) < (pr->rbestpath.rlen - 1)) && (d != pr->rbestpath.rele[(*i) + 1].rdepth)) {
   2106             if ((*i > 0) && (with__0->rdepth-1) == pr->rbestpath.rele[*i + 1].rdepth) {
   2107                 li = 0;
   2108                 while ((li < 127) && (li < with__0->rdepth-1)) {
   2109                     pr->lspaces[li++] = ' ';
   2110                 }
   2111                 pr->lspaces[li] = 0;
   2112                 PICODBG_INFO(("pp path  :%s)", pr->lspaces));
   2113             }
   2114             return;
   2115         } else {
   2116             (*i)++;
   2117         }
   2118         if ((PR_TSE_MASK_LEX & wpset) == 0) {
   2119             lfirst = TRUE;
   2120         }
   2121     }
   2122     li = 0;
   2123     while ((li < 127) && (li < pr->rbestpath.rele[*i].rdepth-1)) {
   2124         pr->lspaces[li++] = ' ';
   2125     }
   2126     pr->lspaces[li] = 0;
   2127     PICODBG_INFO(("pp path  :%s)", pr->lspaces));
   2128 }
   2129 
   2130 
   2131 static void pr_outputPath (picodata_ProcessingUnit this, pr_subobj_t * pr)
   2132 {
   2133     register struct pr_PathEle * with__0;
   2134     picoos_int32 li;
   2135     pr_ioItemPtr lf;
   2136     pr_ioItemPtr ll;
   2137     pr_ioItemPtr lit;
   2138     pr_ioItemPtr lit2;
   2139     pr_MemState lmemState;
   2140     picoos_bool lastPlayFileFound;
   2141 
   2142     pr_getMemState(this, pr_WorkMem, & lmemState);
   2143     lf = NULL;
   2144     ll = NULL;
   2145     li =  -(1);
   2146     pr_getOutput(this, pr, & li,1,& lf,& ll);
   2147     if (pr->outOfMemory) return;
   2148     lastPlayFileFound = TRUE;
   2149     while (lf != NULL) {
   2150         lit = lf;
   2151         lf = lf->next;
   2152         lit->next = NULL;
   2153         if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY)) {
   2154             lastPlayFileFound = picoos_FileExists(this->common, (picoos_char*)lit->data);
   2155             if (!lastPlayFileFound) {
   2156                 PICODBG_WARN(("file '%s' not found; synthesizing enclosed text instead", lit->data));
   2157                 picoos_emRaiseWarning(this->common->em, PICO_EXC_CANT_OPEN_FILE, (picoos_char*)"", (picoos_char*)"file '%s' not found; synthesizing enclosed text instead",lit->data);
   2158             }
   2159         }
   2160         if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START)) {
   2161             pr->insidePhoneme = TRUE;
   2162         } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME)&& pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END)) {
   2163             pr->insidePhoneme = FALSE;
   2164         }
   2165         if ((pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY) &&  !lastPlayFileFound)) {
   2166         } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNORE) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START)) {
   2167             if (lastPlayFileFound) {
   2168                 pr->rignore++;
   2169             }
   2170         } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNORE) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END)) {
   2171             if (lastPlayFileFound) {
   2172                 if (pr->rignore > 0) {
   2173                     pr->rignore--;
   2174                 }
   2175             }
   2176         } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START) &&  !lastPlayFileFound) {
   2177         } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END) &&  !lastPlayFileFound) {
   2178         } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_CONTEXT)) {
   2179             if (pr->rignore <= 0) {
   2180                 pr_setContext(this, pr, lit->data);
   2181             }
   2182         } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_VOICE)) {
   2183             if (pr->rignore <= 0) {
   2184                 pr_copyItem(this, pr_DynMem,lit,& lit2);
   2185                 if (pr->outOfMemory) return;
   2186                 pr_appendItem(this, & pr->routItemList,& pr->rlastOutItem, lit2);
   2187             }
   2188         } else {
   2189             if ((pr->rignore <= 0) &&  !(pr->insidePhoneme && (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY) ||
   2190                                                                pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG)))) {
   2191                 PICODBG_INFO(("pp out(1): '%s'", lit->data));
   2192                 pr_copyItem(this, pr_DynMem,lit,& lit2);
   2193                 if (pr->outOfMemory) return;
   2194                 pr_appendItemToOutItemList(this, pr, & pr->routItemList,& pr->rlastOutItem,lit2);
   2195                 if (pr->outOfMemory) return;
   2196             }
   2197         }
   2198     }
   2199     for (li = 0; li<pr->rbestpath.rlen; li++) {
   2200         with__0 = & pr->rbestpath.rele[li];
   2201         pr_disposeItem(this, & pr->ritems[with__0->ritemid+1]);
   2202     }
   2203     pr_resetMemState(this, pr_WorkMem, lmemState);
   2204 }
   2205 
   2206 /* *****************************************************************************/
   2207 
   2208 static void pr_compare (picoos_uchar str1lc[], picoos_uchar str2[], picoos_int16 * result)
   2209 {
   2210 
   2211     picoos_int32 i;
   2212     picoos_int32 j;
   2213     picoos_int32 l;
   2214     picoos_uint32 pos;
   2215     picoos_bool finished1;
   2216     picoos_bool finished2;
   2217     picoos_bool done;
   2218     picobase_utf8char utf8char;
   2219 
   2220     pos = 0;
   2221     l = picobase_get_next_utf8char(str2, PR_MAX_DATA_LEN, & pos,utf8char);
   2222     picobase_lowercase_utf8_str(utf8char,(picoos_char*)utf8char,PICOBASE_UTF8_MAXLEN+1, &done);
   2223     l = picobase_det_utf8_length(utf8char[0]);
   2224     j = 0;
   2225     i = 0;
   2226     while ((i < PR_MAX_DATA_LEN) && (str1lc[i] != 0) && (l > 0) && (j <= 3) && (str1lc[i] == utf8char[j])) {
   2227         i++;
   2228         j++;
   2229         if (j >= l) {
   2230             l = picobase_get_next_utf8char(str2, PR_MAX_DATA_LEN, & pos,utf8char);
   2231             picobase_lowercase_utf8_str(utf8char,(picoos_char*)utf8char,PICOBASE_UTF8_MAXLEN+1, &done);
   2232             l = picobase_det_utf8_length(utf8char[0]);
   2233             j = 0;
   2234         }
   2235     }
   2236     finished1 = (i >= PR_MAX_DATA_LEN) || (str1lc[i] == 0);
   2237     finished2 = (j > 3) || (utf8char[j] == 0);
   2238     if (finished1 && finished2) {
   2239         *result = PR_EQUAL;
   2240     } else if (finished1) {
   2241         *result = PR_SMALLER;
   2242     } else if (finished2) {
   2243         *result = PR_LARGER;
   2244     } else {
   2245         if (str1lc[i] < utf8char[j]) {
   2246             *result = PR_SMALLER;
   2247         } else {
   2248             *result = PR_LARGER;
   2249         }
   2250     }
   2251 }
   2252 
   2253 
   2254 static picoos_bool pr_hasToken (picokpr_TokSetWP * tswp, picokpr_TokSetNP * tsnp)
   2255 {
   2256     return ((((  PR_TSE_MASK_SPACE | PR_TSE_MASK_DIGIT | PR_TSE_MASK_LETTER | PR_TSE_MASK_SEQ
   2257                | PR_TSE_MASK_CHAR | PR_TSE_MASK_BEGIN | PR_TSE_MASK_END) & (*tsnp)) != 0) ||
   2258             ((PR_TSE_MASK_LEX & (*tswp)) != 0));
   2259 }
   2260 
   2261 
   2262 static picoos_bool pr_getNextToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
   2263 {
   2264     register struct pr_PathEle * with__0;
   2265     picoos_int32 len;
   2266     picokpr_TokSetNP npset;
   2267 
   2268     len = pr->ractpath.rlen;
   2269     with__0 = & pr->ractpath.rele[len - 1];
   2270     npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
   2271     if ((len > 0) && (pr->ractpath.rlen < PR_MAX_PATH_LEN) && ((PR_TSE_MASK_NEXT & npset) != 0)) {
   2272         pr_initPathEle(& pr->ractpath.rele[len]);
   2273         pr->ractpath.rele[len].rnetwork = with__0->rnetwork;
   2274         pr->ractpath.rele[len].rtok = picokpr_getTokNextOfs(with__0->rnetwork, with__0->rtok);
   2275         pr->ractpath.rele[len].rdepth = with__0->rdepth;
   2276         pr->ractpath.rlen++;
   2277         return TRUE;
   2278     } else {
   2279         if (len >= PR_MAX_PATH_LEN) {
   2280           PICODBG_INFO(("max path len reached (pr_getNextToken)"));
   2281         }
   2282         return FALSE;
   2283     }
   2284 }
   2285 
   2286 
   2287 static picoos_bool pr_getAltToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
   2288 {
   2289     register struct pr_PathEle * with__0;
   2290     picokpr_TokArrOffset lTok;
   2291     picokpr_TokSetNP npset;
   2292 
   2293 
   2294     with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
   2295     if ((pr->ractpath.rlen > 0) && (pr->ractpath.rlen < PR_MAX_PATH_LEN)) {
   2296         npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
   2297         if (with__0->rcompare == PR_SMALLER) {
   2298             if ((PR_TSE_MASK_ALTL & npset) != 0) {
   2299                 lTok = picokpr_getTokAltLOfs(with__0->rnetwork, with__0->rtok);
   2300             } else {
   2301                 return FALSE;
   2302             }
   2303         } else {
   2304             if ((PR_TSE_MASK_ALTR & npset) != 0) {
   2305                 lTok = picokpr_getTokAltROfs(with__0->rnetwork, with__0->rtok);
   2306             } else {
   2307                 return FALSE;
   2308             }
   2309         }
   2310         with__0->rlState = PR_LSInit;
   2311         with__0->rtok = lTok;
   2312         with__0->ritemid =  -1;
   2313         with__0->rcompare =  -1;
   2314         with__0->rprodname = 0;
   2315         with__0->rprodprefcost = 0;
   2316         return TRUE;
   2317     } else {
   2318         if (pr->ractpath.rlen >= PR_MAX_PATH_LEN) {
   2319           PICODBG_INFO(("max path len reached (pr_getAltToken)"));
   2320         }
   2321         return FALSE;
   2322     }
   2323 }
   2324 
   2325 
   2326 static picoos_bool pr_findProduction (picodata_ProcessingUnit this, pr_subobj_t * pr,
   2327                                       picoos_uchar str[], picokpr_Preproc * network, picokpr_TokArrOffset * tokOfs)
   2328 {
   2329     picoos_bool found;
   2330     picoos_int32 p;
   2331     picoos_int32 ind;
   2332     picoos_int32 i;
   2333     picoos_bool done;
   2334     picokpr_VarStrPtr lstrp;
   2335     picoos_int32 lprodarrlen;
   2336 
   2337     ind = 0;
   2338     pr_getTermPartStr(str,& ind,'.',pr->tmpStr1,& done);
   2339     pr_getTermPartStr(str,& ind,'.',pr->tmpStr2,& done);
   2340     found = FALSE;
   2341 
   2342     for (p=0; p<PR_MAX_NR_PREPROC; p++) {
   2343         if (!found && (pr->preproc[p] != NULL)) {
   2344             if (pr_strEqual(pr->tmpStr1, picokpr_getPreprocNetName(pr->preproc[p]))) {
   2345                 i = 0;
   2346                 lprodarrlen = picokpr_getProdArrLen(pr->preproc[p]);
   2347                 while (!found && (i <= (lprodarrlen - 1))) {
   2348                     lstrp = picokpr_getVarStrPtr(pr->preproc[p],picokpr_getProdNameOfs(pr->preproc[p], i));
   2349                     if (pr_strEqual(pr->tmpStr2, lstrp)) {
   2350                         *network = pr->preproc[p];
   2351                         *tokOfs = picokpr_getProdATokOfs(pr->preproc[p], i);
   2352                         return TRUE;
   2353                     }
   2354                     i++;
   2355                 }
   2356            }
   2357         }
   2358     }
   2359     return FALSE;
   2360 }
   2361 
   2362 
   2363 static picoos_bool pr_getProdToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
   2364 {
   2365     register struct pr_PathEle * with__0;
   2366     picokpr_VarStrPtr lstrp;
   2367     picokpr_TokSetWP wpset;
   2368 
   2369     if ((pr->ractpath.rlen > 0) && (pr->ractpath.rlen < PR_MAX_PATH_LEN)) {
   2370         with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
   2371         wpset = picokpr_getTokSetWP(with__0->rnetwork, with__0->rtok);
   2372         if ((PR_TSE_MASK_PROD & wpset) != 0) {
   2373             if ((PR_TSE_MASK_PRODEXT & wpset) != 0) {
   2374                 pr_initPathEle(& pr->ractpath.rele[pr->ractpath.rlen]);
   2375                 lstrp = picokpr_getVarStrPtr(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEProdExt));
   2376                 if (pr_findProduction(this, pr, lstrp,& pr->ractpath.rele[pr->ractpath.rlen].rnetwork,& pr->ractpath.rele[pr->ractpath.rlen].rtok)) {
   2377                     with__0->rprodname = picokpr_getProdNameOfs(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok,PR_TSEProd));
   2378                     with__0->rprodprefcost = picokpr_getProdPrefCost(with__0->rnetwork, pr_attrVal(with__0->rnetwork,with__0->rtok,PR_TSEProd));
   2379                     pr->ractpath.rele[pr->ractpath.rlen].rdepth = with__0->rdepth + 1;
   2380                     pr->ractpath.rlen++;
   2381                     return TRUE;
   2382                 } else {
   2383                     return FALSE;
   2384                 }
   2385             } else {
   2386                 pr_initPathEle(& pr->ractpath.rele[pr->ractpath.rlen]);
   2387                 pr->ractpath.rele[pr->ractpath.rlen].rnetwork = with__0->rnetwork;
   2388                 pr->ractpath.rele[pr->ractpath.rlen].rtok = picokpr_getProdATokOfs(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok,PR_TSEProd));
   2389                 with__0->rprodname = picokpr_getProdNameOfs(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEProd));
   2390                 with__0->rprodprefcost = picokpr_getProdPrefCost(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEProd));
   2391                 pr->ractpath.rele[pr->ractpath.rlen].rdepth = with__0->rdepth + 1;
   2392                 pr->ractpath.rlen++;
   2393                 return TRUE;
   2394             }
   2395         }
   2396     }
   2397     if (pr->ractpath.rlen >= PR_MAX_PATH_LEN) {
   2398         PICODBG_INFO(("max path len reached (pr_getProdToken)"));
   2399     }
   2400     return FALSE;
   2401 }
   2402 
   2403 
   2404 static picoos_bool pr_getProdContToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
   2405 {
   2406     picoos_int32 li;
   2407 
   2408     li = pr->ractpath.rlen - 1;
   2409     while ((li > 0) &&  !((pr->ractpath.rele[li].rdepth == (pr->ractpath.rele[pr->ractpath.rlen - 1].rdepth - 1)) && ((PR_TSE_MASK_PROD &picokpr_getTokSetWP(pr->ractpath.rele[li].rnetwork, pr->ractpath.rele[li].rtok)) != 0))) {
   2410         li--;
   2411     }
   2412     if (((li >= 0) && (pr->ractpath.rlen < PR_MAX_PATH_LEN) && (PR_TSE_MASK_NEXT &picokpr_getTokSetNP(pr->ractpath.rele[li].rnetwork, pr->ractpath.rele[li].rtok)) != 0)) {
   2413         pr_initPathEle(& pr->ractpath.rele[pr->ractpath.rlen]);
   2414         pr->ractpath.rele[pr->ractpath.rlen].rnetwork = pr->ractpath.rele[li].rnetwork;
   2415         pr->ractpath.rele[pr->ractpath.rlen].rtok = picokpr_getTokNextOfs(pr->ractpath.rele[li].rnetwork, pr->ractpath.rele[li].rtok);
   2416         pr->ractpath.rele[pr->ractpath.rlen].rdepth = pr->ractpath.rele[li].rdepth;
   2417         pr->ractpath.rlen++;
   2418         return TRUE;
   2419     } else {
   2420         if (pr->ractpath.rlen >= PR_MAX_PATH_LEN) {
   2421             PICODBG_INFO(("max path len reached (pr_getProdContToken)"));
   2422         }
   2423         return FALSE;
   2424     }
   2425 }
   2426 
   2427 /* *****************************************************************************/
   2428 
   2429 static picoos_bool pr_getTopLevelToken (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_bool firstprod)
   2430 {
   2431     if (firstprod) {
   2432         if (pr->actCtx != NULL) {
   2433             pr->prodList = pr->actCtx->rProdList;
   2434         } else {
   2435             pr->prodList = NULL;
   2436         }
   2437     } else if (pr->prodList != NULL) {
   2438         pr->prodList = pr->prodList->rNext;
   2439     }
   2440     if ((pr->prodList != NULL) && (pr->prodList->rProdOfs != 0) && (picokpr_getProdATokOfs(pr->prodList->rNetwork, pr->prodList->rProdOfs) != 0)) {
   2441         pr_initPathEle(& pr->ractpath.rele[pr->ractpath.rlen]);
   2442         pr->ractpath.rele[pr->ractpath.rlen].rdepth = 1;
   2443         pr->ractpath.rele[pr->ractpath.rlen].rnetwork = pr->prodList->rNetwork;
   2444         pr->ractpath.rele[pr->ractpath.rlen].rtok = picokpr_getProdATokOfs(pr->prodList->rNetwork, pr->prodList->rProdOfs);
   2445         pr->ractpath.rele[pr->ractpath.rlen].rlState = PR_LSInit;
   2446         pr->ractpath.rele[pr->ractpath.rlen].rcompare =  -1;
   2447         pr->ractpath.rele[pr->ractpath.rlen].rprodname = picokpr_getProdNameOfs(pr->prodList->rNetwork, pr->prodList->rProdOfs);
   2448         pr->ractpath.rele[pr->ractpath.rlen].rprodprefcost = picokpr_getProdPrefCost(pr->prodList->rNetwork, pr->prodList->rProdOfs);
   2449         pr->ractpath.rlen++;
   2450         return TRUE;
   2451     } else {
   2452         return FALSE;
   2453     }
   2454 }
   2455 
   2456 
   2457 static picoos_bool pr_getToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
   2458 {
   2459     picoos_int32 ln;
   2460     picoos_int32 lid;
   2461 
   2462     ln = (pr->ractpath.rlen - 2);
   2463     while ((ln >= 0) && (pr->ractpath.rele[ln].ritemid ==  -1)) {
   2464         ln = ln - 1;
   2465     }
   2466     if (ln >= 0) {
   2467         lid = pr->ractpath.rele[ln].ritemid + 1;
   2468     } else {
   2469         lid = 0;
   2470     }
   2471     if (lid < pr->rnritems) {
   2472         pr->ractpath.rele[pr->ractpath.rlen - 1].ritemid = lid;
   2473     } else {
   2474         pr->ractpath.rele[pr->ractpath.rlen - 1].ritemid =  -1;
   2475     }
   2476     return (lid < pr->rnritems);
   2477 }
   2478 
   2479 
   2480 static picoos_bool pr_getNextMultiToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
   2481 {
   2482     picoos_int32 len;
   2483 
   2484     len = pr->ractpath.rlen;
   2485     if ((len > 0) && (len < PR_MAX_PATH_LEN)) {
   2486         pr->ractpath.rele[len].rtok = pr->ractpath.rele[len - 1].rtok;
   2487         pr->ractpath.rele[len].ritemid =  -(1);
   2488         pr->ractpath.rele[len].rcompare = pr->ractpath.rele[len - 1].rcompare;
   2489         pr->ractpath.rele[len].rdepth = pr->ractpath.rele[len - 1].rdepth;
   2490         pr->ractpath.rele[len].rlState = PR_LSInit;
   2491         pr->ractpath.rlen++;
   2492         return TRUE;
   2493     } else {
   2494         if (len >= PR_MAX_PATH_LEN) {
   2495             PICODBG_INFO(("max path len reached (pr_getNextMultiToken)"));
   2496         }
   2497         return FALSE;
   2498     }
   2499     return FALSE;
   2500 }
   2501 
   2502 
   2503 static pr_MatchState pr_matchMultiToken (picodata_ProcessingUnit this, pr_subobj_t * pr,
   2504                                          picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
   2505 {
   2506     picoos_bool lcontinue=FALSE;
   2507     picoos_bool lmatch=FALSE;
   2508 
   2509     if (lmatch) {
   2510         return PR_MSMatchedMulti;
   2511     } else if (lcontinue) {
   2512         return PR_MSMatchedContinue;
   2513     } else {
   2514         return PR_MSNotMatched;
   2515     }
   2516     pr = pr;        /* avoid warning "var not used in this function"*/
   2517     npset = npset;    /* avoid warning "var not used in this function"*/
   2518     wpset = wpset;    /* avoid warning "var not used in this function"*/
   2519 
   2520 }
   2521 
   2522 
   2523 static pr_MatchState pr_matchTokensSpace (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
   2524                                           picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
   2525 {
   2526     register struct pr_PathEle * with__0;
   2527     picoos_int32 llen;
   2528     picoos_int32 lulen;
   2529     picoos_int32 li;
   2530     picokpr_VarStrPtr lstrp;
   2531     picoos_int32 leol;
   2532 
   2533     with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
   2534     if ((PR_TSE_MASK_SPACE & npset) == 0) {
   2535         return PR_MSNotMatched;
   2536     }
   2537     lstrp = (picokpr_VarStrPtr)&pr->ritems[with__0->ritemid+1]->data;
   2538     lulen = picobase_utf8_length(lstrp,PR_MAX_DATA_LEN);
   2539     if (((PR_TSE_MASK_LEN & wpset) != 0) && (lulen != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSELen))) {
   2540         return PR_MSNotMatched;
   2541     }
   2542     if (((PR_TSE_MASK_MIN & wpset) != 0) && (lulen < pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMin))) {
   2543         return PR_MSNotMatched;
   2544     }
   2545     if (((PR_TSE_MASK_MAX & wpset) != 0) && (lulen > pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMax))) {
   2546         return PR_MSNotMatched;
   2547     }
   2548     if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
   2549         return PR_MSNotMatched;
   2550     }
   2551     if ((PR_TSE_MASK_VAL & wpset) != 0) {
   2552         leol = 0;
   2553         llen = pr_strlen(lstrp);
   2554         for (li = 0; li < llen; li++) {
   2555             if (lstrp[li] == PR_EOL) {
   2556                 leol++;
   2557             }
   2558         }
   2559         if (leol != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVal)) {
   2560             return PR_MSNotMatched;
   2561         }
   2562     }
   2563     if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
   2564         return PR_MSNotMatched;
   2565     }
   2566     return PR_MSMatched;
   2567 }
   2568 
   2569 
   2570 static pr_MatchState pr_matchTokensDigit (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
   2571                                           picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
   2572 {
   2573     register struct pr_PathEle * with__0;
   2574     picoos_int32 lulen;
   2575     picoos_int32 lval;
   2576     picokpr_VarStrPtr lstrp;
   2577 
   2578     with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
   2579     if ((PR_TSE_MASK_DIGIT & npset) == 0) {
   2580         return PR_MSNotMatched;
   2581     }
   2582     lstrp = (picokpr_VarStrPtr)&pr->ritems[with__0->ritemid+1]->data;
   2583     lulen = picobase_utf8_length(lstrp,PR_MAX_DATA_LEN);
   2584     if ((((PR_TSE_MASK_LEN & wpset) != 0) && (lulen != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSELen)))) {
   2585         return PR_MSNotMatched;
   2586     }
   2587     lval = pr->ritems[with__0->ritemid+1]->val;
   2588     if (((PR_TSE_MASK_MIN & wpset) != 0) && (lval < pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMin))) {
   2589         return PR_MSNotMatched;
   2590     }
   2591     if (((PR_TSE_MASK_MAX & wpset) != 0) && (lval > pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMax))) {
   2592         return PR_MSNotMatched;
   2593     }
   2594     if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
   2595         return PR_MSNotMatched;
   2596     }
   2597     if (((PR_TSE_MASK_VAL & wpset) != 0) && (lval != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVal))) {
   2598         return PR_MSNotMatched;
   2599     }
   2600     if ((((PR_TSE_MASK_NLZ & npset) != 0) && lstrp[0] == '0')) {
   2601         return PR_MSNotMatched;
   2602     }
   2603     if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
   2604         return PR_MSNotMatched;
   2605     }
   2606     if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
   2607         return PR_MSNotMatched;
   2608     }
   2609     if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
   2610         return PR_MSNotMatched;
   2611     }
   2612     if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
   2613         return PR_MSNotMatched;
   2614     }
   2615     return PR_MSMatched;
   2616 }
   2617 
   2618 
   2619 static pr_MatchState pr_matchTokensSeq (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
   2620                                         picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
   2621 {
   2622 
   2623     register struct pr_PathEle * with__0;
   2624     picoos_int32 lulen;
   2625     picokpr_VarStrPtr lstrp;
   2626 
   2627     with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
   2628 
   2629     if (!((PR_TSE_MASK_SEQ & npset) != 0)) {
   2630         return PR_MSNotMatched;
   2631     }
   2632     lstrp = (picokpr_VarStrPtr)(void *) &pr->ritems[with__0->ritemid+1]->data;
   2633     lulen = picobase_utf8_length(lstrp,PR_MAX_DATA_LEN);
   2634     if (((PR_TSE_MASK_LEN & wpset) != 0) && (lulen != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSELen))) {
   2635         return PR_MSNotMatched;
   2636     }
   2637     if (((PR_TSE_MASK_MIN & wpset) != 0) && (lulen < pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMin))) {
   2638         return PR_MSNotMatched;
   2639     }
   2640     if (((PR_TSE_MASK_MAX & wpset) != 0) && (lulen > pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMax))) {
   2641         return PR_MSNotMatched;
   2642     }
   2643     if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
   2644         return PR_MSNotMatched;
   2645     }
   2646     if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
   2647         return PR_MSNotMatched;
   2648     }
   2649     if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN ,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
   2650         return PR_MSNotMatched;
   2651     }
   2652     if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
   2653         return PR_MSNotMatched;
   2654     }
   2655     if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
   2656         return PR_MSNotMatched;
   2657     }
   2658     return PR_MSMatched;
   2659 }
   2660 
   2661 
   2662 static pr_MatchState pr_matchTokensChar (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
   2663                                          picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
   2664 {
   2665     register struct pr_PathEle * with__0;
   2666 
   2667     with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
   2668 
   2669     if (!((PR_TSE_MASK_CHAR & npset) != 0)) {
   2670         return PR_MSNotMatched;
   2671     }
   2672     if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
   2673         return PR_MSNotMatched;
   2674     }
   2675     if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
   2676         return PR_MSNotMatched;
   2677     }
   2678     return PR_MSMatched;
   2679 }
   2680 
   2681 
   2682 static pr_MatchState pr_matchTokensLetter (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
   2683                                            picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
   2684 {
   2685 
   2686     register struct pr_PathEle * with__0;
   2687     picoos_int32 lulen;
   2688     picoos_int32 lromanval;
   2689 
   2690     with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
   2691 
   2692     if ( !((PR_TSE_MASK_LETTER & npset) != 0)) {
   2693         return PR_MSNotMatched;
   2694     }
   2695     lulen = picobase_utf8_length(pr->ritems[with__0->ritemid+1]->data, PR_MAX_DATA_LEN);
   2696     if (((PR_TSE_MASK_LEN & wpset) != 0) && (lulen != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSELen))) {
   2697         return PR_MSNotMatched;
   2698     }
   2699     if (((PR_TSE_MASK_MIN & wpset) != 0) && (lulen < pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMin))) {
   2700         return PR_MSNotMatched;
   2701     }
   2702     if (((PR_TSE_MASK_MAX & wpset) != 0) && (lulen > pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMax))) {
   2703         return PR_MSNotMatched;
   2704     }
   2705     if ((PR_TSE_MASK_CI & npset) != 0) {
   2706         if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
   2707             return PR_MSNotMatched;
   2708         }
   2709         if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strci,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
   2710             return PR_MSNotMatched;
   2711         }
   2712         if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strci,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
   2713             return PR_MSNotMatched;
   2714         }
   2715         if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strci,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
   2716             return PR_MSNotMatched;
   2717         }
   2718     } else if ((PR_TSE_MASK_CIS & npset) != 0) {
   2719         if (((PR_TSE_MASK_STR & wpset) != 0) &&  !(picokpr_isEqual(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strcis,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEStr)))) {
   2720             return PR_MSNotMatched;
   2721         }
   2722         if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strcis,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
   2723             return PR_MSNotMatched;
   2724         }
   2725         if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strcis,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
   2726             return PR_MSNotMatched;
   2727         }
   2728         if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strcis,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
   2729             return PR_MSNotMatched;
   2730         }
   2731     } else {
   2732         if (((PR_TSE_MASK_STR & wpset) != 0) &&  !(picokpr_isEqual(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->data,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEStr)))) {
   2733             return PR_MSNotMatched;
   2734         }
   2735         if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->data,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
   2736             return PR_MSNotMatched;
   2737         }
   2738         if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->data,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
   2739             return PR_MSNotMatched;
   2740         }
   2741         if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->data,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
   2742             return PR_MSNotMatched;
   2743         }
   2744     }
   2745     if (((PR_TSE_MASK_AUC & npset) != 0) &&  !(pr->ritems[with__0->ritemid+1]->auc)) {
   2746         return PR_MSNotMatched;
   2747     }
   2748     if (((PR_TSE_MASK_ALC & npset) != 0) &&  !(pr->ritems[with__0->ritemid+1]->alc)) {
   2749         return PR_MSNotMatched;
   2750     }
   2751     if (((PR_TSE_MASK_SUC & npset) != 0) &&  !(pr->ritems[with__0->ritemid+1]->suc)) {
   2752         return PR_MSNotMatched;
   2753     }
   2754     if (((PR_TSE_MASK_ROMAN & npset) != 0) &&  !(pr_isLatinNumber(pr->ritems[with__0->ritemid+1]->data,& lromanval))) {
   2755         return PR_MSNotMatched;
   2756     }
   2757     if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
   2758         return PR_MSNotMatched;
   2759     }
   2760     return PR_MSMatched;
   2761 }
   2762 
   2763 
   2764 static pr_MatchState pr_matchTokensBegin (picodata_ProcessingUnit this, pr_subobj_t * pr,
   2765                                           picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
   2766 {
   2767     npset = npset;        /* avoid warning "var not used in this function"*/
   2768     wpset = wpset;        /* avoid warning "var not used in this function"*/
   2769     if ((PR_TSE_MASK_BEGIN &picokpr_getTokSetNP(pr->ractpath.rele[pr->ractpath.rlen - 1].rnetwork, pr->ractpath.rele[pr->ractpath.rlen - 1].rtok)) != 0) {
   2770         return PR_MSMatched;
   2771     } else {
   2772         return PR_MSNotMatched;
   2773     }
   2774 }
   2775 
   2776 
   2777 
   2778 static pr_MatchState pr_matchTokensEnd (picodata_ProcessingUnit this, pr_subobj_t * pr,
   2779                                         picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
   2780 {
   2781     npset = npset;        /* avoid warning "var not used in this function"*/
   2782     wpset = wpset;        /* avoid warning "var not used in this function"*/
   2783     if ((PR_TSE_MASK_END &picokpr_getTokSetNP(pr->ractpath.rele[pr->ractpath.rlen - 1].rnetwork, pr->ractpath.rele[pr->ractpath.rlen - 1].rtok)) != 0) {
   2784         return PR_MSMatched;
   2785     } else {
   2786         return PR_MSNotMatched;
   2787     }
   2788 }
   2789 
   2790 
   2791 static pr_MatchState pr_matchTokens (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int16 * cmpres)
   2792 {
   2793 
   2794     register struct pr_PathEle * with__0;
   2795     picokpr_VarStrPtr lstrp;
   2796     picokpr_TokSetNP npset;
   2797     picokpr_TokSetWP wpset;
   2798 
   2799     with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
   2800     npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
   2801     wpset = picokpr_getTokSetWP(with__0->rnetwork, with__0->rtok);
   2802 
   2803     *cmpres = PR_EQUAL;
   2804     if ((PR_TSE_MASK_STR & wpset) != 0) {
   2805         lstrp = picokpr_getVarStrPtr(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEStr));
   2806         pr_compare(pr->ritems[with__0->ritemid+1]->strci,lstrp,cmpres);
   2807     }
   2808     if (((PR_TSE_MASK_LEX & wpset) == PR_TSE_MASK_LEX) && ((PR_TSE_MASK_LETTER & npset) == 0)) {
   2809         return pr_matchMultiToken(this, pr, npset, wpset);
   2810     } else {
   2811         switch (pr->ritems[with__0->ritemid+1]->head.info1) {
   2812             case PICODATA_ITEMINFO1_TOKTYPE_BEGIN:
   2813                 return pr_matchTokensBegin(this, pr, npset, wpset);
   2814                 break;
   2815             case PICODATA_ITEMINFO1_TOKTYPE_END:
   2816                 return pr_matchTokensEnd(this, pr, npset, wpset);
   2817                 break;
   2818             case PICODATA_ITEMINFO1_TOKTYPE_SPACE:
   2819                 return pr_matchTokensSpace(this, pr, *cmpres, npset, wpset);
   2820                 break;
   2821             case PICODATA_ITEMINFO1_TOKTYPE_DIGIT:
   2822                 return pr_matchTokensDigit(this, pr, *cmpres, npset, wpset);
   2823                 break;
   2824             case PICODATA_ITEMINFO1_TOKTYPE_LETTER:
   2825                 return pr_matchTokensLetter(this, pr, *cmpres, npset, wpset);
   2826                 break;
   2827             case PICODATA_ITEMINFO1_TOKTYPE_SEQ:
   2828                 return pr_matchTokensSeq(this, pr, *cmpres, npset, wpset);
   2829                 break;
   2830             case PICODATA_ITEMINFO1_TOKTYPE_CHAR:
   2831                 return pr_matchTokensChar(this, pr, *cmpres, npset, wpset);
   2832                 break;
   2833         default:
   2834             PICODBG_INFO(("pr_matchTokens: unknown token type"));
   2835             return PR_MSNotMatched;
   2836             break;
   2837         }
   2838     }
   2839 }
   2840 
   2841 
   2842 static void pr_calcPathCost (struct pr_Path * path)
   2843 {
   2844     picoos_int32 li;
   2845     picoos_bool lfirst;
   2846     picokpr_TokSetWP wpset;
   2847     picokpr_TokSetNP npset;
   2848 #if PR_TRACE_PATHCOST
   2849     picoos_uchar str[1000];
   2850     picoos_uchar * strp;
   2851 #endif
   2852 
   2853 #if PR_TRACE_PATHCOST
   2854     str[0] = 0;
   2855 #endif
   2856 
   2857     lfirst = TRUE;
   2858     path->rcost = PR_COST_INIT;
   2859     for (li = 0; li < path->rlen; li++) {
   2860         if (li == 0) {
   2861             path->rcost = path->rcost + path->rele[li].rprodprefcost;
   2862         }
   2863         wpset = picokpr_getTokSetWP(path->rele[li].rnetwork, path->rele[li].rtok);
   2864         npset = picokpr_getTokSetNP(path->rele[li].rnetwork, path->rele[li].rtok);
   2865         if ((PR_TSE_MASK_COST & wpset) != 0) {
   2866             if (((PR_TSE_MASK_LEX & wpset) == PR_TSE_MASK_LEX) && ((PR_TSE_MASK_LETTER & npset) == 0)) {
   2867                 if (lfirst) {
   2868                     path->rcost = path->rcost - PR_COST + pr_attrVal(path->rele[li].rnetwork, path->rele[li].rtok, PR_TSECost);
   2869                 } else {
   2870                     path->rcost = path->rcost - PR_COST;
   2871                 }
   2872                 lfirst = FALSE;
   2873             } else {
   2874                 path->rcost = path->rcost - PR_COST + pr_attrVal(path->rele[li].rnetwork, path->rele[li].rtok, PR_TSECost);
   2875                 lfirst = TRUE;
   2876             }
   2877         } else if (pr_hasToken(& wpset,& npset)) {
   2878             path->rcost = path->rcost - PR_COST;
   2879         }
   2880 #if PR_TRACE_PATHCOST
   2881         if ((path->rele[li].rprodname != 0)) {
   2882             strp = picokpr_getVarStrPtr(path->rele[li].rnetwork, path->rele[li].rprodname);
   2883             picoos_strcat(str, (picoos_char *)" ");
   2884             picoos_strcat(str, strp);
   2885         }
   2886 #endif
   2887     }
   2888 #if PR_TRACE_PATHCOST
   2889     PICODBG_INFO(("pp cost: %i %s", path->rcost, str));
   2890 #endif
   2891 }
   2892 
   2893 
   2894 void pr_processToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
   2895 {
   2896     register struct pr_PathEle * with__0;
   2897     picoos_bool ldummy;
   2898     picoos_int32 li;
   2899     picokpr_TokSetNP npset;
   2900     picokpr_TokSetWP wpset;
   2901 
   2902     do {
   2903         pr->rgState = PR_GSContinue;
   2904         if ((pr->ractpath.rlen == 0)) {
   2905             if (pr_getTopLevelToken(this, pr, FALSE)) {
   2906                 pr->rgState = PR_GSContinue;
   2907             } else if (pr->rbestpath.rlen == 0) {
   2908                 pr->rgState = PR_GSNotFound;
   2909             } else {
   2910                 pr->rgState = PR_GSFound;
   2911             }
   2912         } else {
   2913             if (pr->maxPathLen < pr->ractpath.rlen) {
   2914                 pr->maxPathLen = pr->ractpath.rlen;
   2915             }
   2916             with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
   2917             switch (with__0->rlState) {
   2918                 case PR_LSInit:
   2919                     npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
   2920                     wpset = picokpr_getTokSetWP(with__0->rnetwork, with__0->rtok);
   2921                     if ((PR_TSE_MASK_ACCEPT & npset) != 0){
   2922                         if (with__0->rdepth == 1) {
   2923                             pr_calcPathCost(&pr->ractpath);
   2924                             if ((pr->rbestpath.rlen == 0) || (pr->ractpath.rcost < pr->rbestpath.rcost)) {
   2925                                 pr->rbestpath.rlen = pr->ractpath.rlen;
   2926                                 pr->rbestpath.rcost = pr->ractpath.rcost;
   2927                                 for (li = 0; li < pr->ractpath.rlen; li++) {
   2928                                     pr->rbestpath.rele[li] = pr->ractpath.rele[li];
   2929                                 }
   2930                             }
   2931                             with__0->rlState = PR_LSGetNextToken;
   2932                         } else {
   2933                             with__0->rlState = PR_LSGetProdContToken;
   2934                         }
   2935                     } else if ((PR_TSE_MASK_PROD & wpset) != 0) {
   2936                         with__0->rlState = PR_LSGetProdToken;
   2937                     } else if ((PR_TSE_MASK_OUT & wpset) != 0) {
   2938                         with__0->rlState = PR_LSGetNextToken;
   2939                     } else if (pr_hasToken(& wpset,& npset)) {
   2940                         with__0->rlState = PR_LSGetToken;
   2941                     } else {
   2942                         with__0->rlState = PR_LSGetNextToken;
   2943                     }
   2944                     break;
   2945                 case PR_LSGetProdToken:
   2946                     with__0->rlState = PR_LSGetAltToken;
   2947                     ldummy = pr_getProdToken(this, pr);
   2948                     break;
   2949                 case PR_LSGetProdContToken:
   2950                     with__0->rlState = PR_LSGetAltToken;
   2951                     ldummy = pr_getProdContToken(this, pr);
   2952                     break;
   2953                 case PR_LSGoBack:
   2954                     pr->ractpath.rlen--;
   2955                     break;
   2956                 case PR_LSGetToken:
   2957                     if (pr_getToken(this, pr)) {
   2958                         with__0->rlState = PR_LSMatch;
   2959                     } else if (pr->forceOutput) {
   2960                         with__0->rlState = PR_LSGetAltToken;
   2961                     } else {
   2962                         with__0->rlState = PR_LSGetToken2;
   2963                         pr->rgState = PR_GSNeedToken;
   2964                     }
   2965                     break;
   2966                 case PR_LSGetToken2:
   2967                     if (pr_getToken(this, pr)) {
   2968                         with__0->rlState = PR_LSMatch;
   2969                     } else {
   2970                         with__0->rlState = PR_LSGoBack;
   2971                     }
   2972                     break;
   2973                 case PR_LSMatch:
   2974                     switch (pr_matchTokens(this, pr, & with__0->rcompare)) {
   2975                         case PR_MSMatched:
   2976                             with__0->rlState = PR_LSGetNextToken;
   2977                             break;
   2978                         case PR_MSMatchedContinue:
   2979                             with__0->rlState = PR_LSGetAltToken;
   2980                             ldummy = pr_getNextMultiToken(this, pr);
   2981                             break;
   2982                         case PR_MSMatchedMulti:
   2983                             with__0->rlState = PR_LSGetNextToken;
   2984                             ldummy = pr_getNextMultiToken(this, pr);
   2985                             break;
   2986                     default:
   2987                         with__0->rlState = PR_LSGetAltToken;
   2988                         break;
   2989                     }
   2990                     break;
   2991                 case PR_LSGetNextToken:
   2992                     with__0->rlState = PR_LSGetAltToken;
   2993                     ldummy = pr_getNextToken(this, pr);
   2994                     break;
   2995                 case PR_LSGetAltToken:
   2996                     with__0->rlState = PR_LSGoBack;
   2997                     ldummy = pr_getAltToken(this, pr);
   2998                     break;
   2999             default:
   3000                 PICODBG_INFO(("unhandled local state"));
   3001                 break;
   3002             }
   3003         }
   3004         pr->nrIterations--;
   3005     } while ((pr->rgState == PR_GSContinue) && (pr->nrIterations > 0));
   3006 }
   3007 
   3008 
   3009 void pr_process (picodata_ProcessingUnit this, pr_subobj_t * pr)
   3010 {
   3011     switch (pr->rgState) {
   3012         case PR_GS_START:
   3013         case PR_GSFound:
   3014         case PR_GSNotFound:
   3015             pr->ractpath.rlen = 0;
   3016             pr->ractpath.rcost = PR_COST_INIT;
   3017             pr->rbestpath.rlen = 0;
   3018             pr->rbestpath.rcost = PR_COST_INIT;
   3019             if (pr_getTopLevelToken(this, pr, TRUE)) {
   3020                 pr->rgState = PR_GSContinue;
   3021             } else {
   3022                 pr->rgState = PR_GSNotFound;
   3023             }
   3024             break;
   3025         case PR_GSContinue:
   3026             pr_processToken(this, pr);
   3027             break;
   3028         case PR_GSNeedToken:
   3029             pr->rgState = PR_GSContinue;
   3030             break;
   3031     default:
   3032         pr->rgState = PR_GS_START;
   3033         break;
   3034     }
   3035 }
   3036 
   3037 
   3038 static void pr_prepareItem (picodata_ProcessingUnit this, pr_subobj_t * pr, pr_ioItemPtr item)
   3039 {
   3040     pr->ritems[pr->rnritems + 1] = item;
   3041     pr->rnritems++;
   3042 }
   3043 
   3044 
   3045 static void pr_processItems (picodata_ProcessingUnit this, pr_subobj_t * pr)
   3046 {
   3047     pr_ioItemPtr lit;
   3048     pr_MemState lmemState;
   3049 
   3050     pr_getMemState(this, pr_WorkMem,& lmemState);
   3051 
   3052     while ((pr->rinItemList != NULL) && (pr->rinItemList->head.type != PICODATA_ITEM_TOKEN)) {
   3053         lit = pr->rinItemList;
   3054         PICODBG_INFO(("pp in (0)"));
   3055         PICODBG_INFO(("pp out(0)"));
   3056         pr->rinItemList = pr->rinItemList->next;
   3057         lit->next = NULL;
   3058         if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START)) {
   3059             pr->insidePhoneme = TRUE;
   3060         } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END)) {
   3061             pr->insidePhoneme = FALSE;
   3062         }
   3063         if (pr->insidePhoneme && (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY) || pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG))) {
   3064             pr_disposeItem(this, & lit);
   3065         } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_CONTEXT)) {
   3066             pr_setContext(this, pr, lit->data);
   3067             pr_disposeItem(this, & lit);
   3068         } else if (pr->rignore <= 0) {
   3069             pr_appendItemToOutItemList(this, pr, & pr->routItemList,& pr->rlastOutItem,lit);
   3070             if (pr->outOfMemory) return;
   3071         } else {
   3072             pr_disposeItem(this, & lit);
   3073         }
   3074         pr->rgState = PR_GS_START;
   3075     }
   3076     if (pr->rinItemList != NULL) {
   3077         pr_process(this, pr);
   3078         if (pr->rgState == PR_GSNotFound) {
   3079             lit = pr->rinItemList;
   3080             pr->rinItemList = pr->rinItemList->next;
   3081             lit->next = NULL;
   3082             PICODBG_INFO(("pp in (2): '%s'", lit->data));
   3083             if (pr->rignore <= 0) {
   3084                 PICODBG_INFO(("pp out(2): '%s'", lit->data));
   3085             }
   3086 
   3087             if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START)) {
   3088                 pr->insidePhoneme = TRUE;
   3089             } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END)) {
   3090                 pr->insidePhoneme = FALSE;
   3091             }
   3092             if (((pr->rignore <= 0) &&  !((pr->insidePhoneme && (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY) || pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG)))))) {
   3093                 pr_appendItemToOutItemList(this, pr, & pr->routItemList,& pr->rlastOutItem,lit);
   3094                 if (pr->outOfMemory) return;
   3095             } else {
   3096                 pr_disposeItem(this, & lit);
   3097             }
   3098             pr->rgState = PR_GS_START;
   3099             pr->rnritems = 0;
   3100         } else if (pr->rgState == PR_GSFound) {
   3101             pr_outputPath(this, pr);
   3102             if (pr->outOfMemory) return;
   3103             pr->rgState = PR_GS_START;
   3104             pr->rnritems = 0;
   3105         }
   3106     }
   3107     if (pr->rinItemList == NULL) {
   3108         pr->rlastInItem = NULL;
   3109     } else if (pr->rnritems == 0) {
   3110         lit = pr->rinItemList;
   3111         while (lit != NULL) {
   3112             if (lit->head.type == PICODATA_ITEM_TOKEN) {
   3113                 pr_prepareItem(this, pr, lit);
   3114             }
   3115             lit = lit->next;
   3116         }
   3117     }
   3118     pr_resetMemState(this, pr_WorkMem,lmemState);
   3119 }
   3120 
   3121 
   3122 
   3123 extern void pr_treatItem (picodata_ProcessingUnit this, pr_subobj_t * pr, pr_ioItemPtr item)
   3124 {
   3125     pr_ioItemPtr lit;
   3126 
   3127     pr_startItemList(& pr->routItemList,& pr->rlastOutItem);
   3128 
   3129     if (!PR_ENABLED || (pr->rgState == PR_GSNoPreproc)) {
   3130         /* preprocessing disabled or no preproc networks available:
   3131            append items directly to output item list */
   3132         PICODBG_INFO(("pp in (3): '%s'", item->data));
   3133         PICODBG_INFO(("pp out(3): '%s'", item->data));
   3134         pr_appendItemToOutItemList(this, pr, & pr->routItemList,& pr->rlastOutItem,item);
   3135     } else {
   3136 
   3137         if (pr->actCtxChanged) {
   3138             pr->rgState = PR_GS_START;
   3139             pr->ractpath.rcost = PR_COST_INIT;
   3140             pr->ractpath.rlen = 0;
   3141             pr->rbestpath.rcost = PR_COST_INIT;
   3142             pr->rbestpath.rlen = 0;
   3143             pr->prodList = NULL;
   3144             pr->rnritems = 0;
   3145             pr->actCtxChanged = FALSE;
   3146         }
   3147         if (pr_isCmdType(item , PICODATA_ITEMINFO1_CMD_CONTEXT) || pr_isCmdType(item, PICODATA_ITEMINFO1_CMD_FLUSH)) {
   3148             /* context switch or flush: force processing and empty input item list */
   3149             pr->forceOutput = TRUE;
   3150         }
   3151         pr_appendItem(this, & pr->rinItemList,& pr->rlastInItem, item);
   3152         if (pr->rnritems == 0) {
   3153             lit = pr->rinItemList;
   3154             while (lit != NULL) {
   3155                 if (lit->head.type == PICODATA_ITEM_TOKEN) {
   3156                     pr_prepareItem(this, pr, lit);
   3157                 }
   3158                 lit = lit->next;
   3159             }
   3160         } else if (item->head.type == PICODATA_ITEM_TOKEN) {
   3161             pr_prepareItem(this, pr, item);
   3162         }
   3163     }
   3164 }
   3165 
   3166 /* *****************************************************************************/
   3167 /* *****************************************************************************/
   3168 /* *****************************************************************************/
   3169 
   3170 
   3171 pico_status_t prReset(register picodata_ProcessingUnit this, picoos_int32 resetMode)
   3172 {
   3173 
   3174     picoos_int32 i;
   3175     pr_subobj_t * pr;
   3176 
   3177     if (NULL == this || NULL == this->subObj) {
   3178         return PICO_ERR_OTHER;
   3179     }
   3180     pr = (pr_subobj_t *) this->subObj;
   3181 
   3182     pr->rinItemList = NULL;
   3183     pr->rlastInItem = NULL;
   3184     pr->routItemList = NULL;
   3185     pr->rlastOutItem = NULL;
   3186     pr->ractpath.rcost = PR_COST_INIT;
   3187     pr->ractpath.rlen = 0;
   3188     pr->rbestpath.rcost = PR_COST_INIT;
   3189     pr->rbestpath.rlen = 0;
   3190     pr->rnritems = 0;
   3191     pr->ritems[0] = NULL;
   3192     pr->rignore = 0;
   3193     pr->spellMode = 0;
   3194     pr->maxPathLen = 0;
   3195     pr->insidePhoneme = FALSE;
   3196     pr->saveFile[0] = 0;
   3197 
   3198     pr->outReadPos = 0;
   3199     pr->outWritePos = 0;
   3200     pr->inBufLen = 0;
   3201 
   3202     pr->rgState = PR_GSNoPreproc;
   3203     for (i=0; i<PR_MAX_NR_PREPROC; i++) {
   3204         if (pr->preproc[i] != NULL) {
   3205             pr->rgState = PR_GS_START;
   3206         }
   3207     }
   3208     pr->actCtx = pr_findContext(pr->ctxList, (picoos_uchar*)PICO_CONTEXT_DEFAULT);
   3209     pr->actCtxChanged = FALSE;
   3210     pr->prodList = NULL;
   3211 
   3212     if (((uintptr_t)pr->pr_WorkMem % PICOOS_ALIGN_SIZE) == 0) {
   3213         pr->workMemTop = 0;
   3214     }
   3215     else {
   3216         pr->workMemTop = PICOOS_ALIGN_SIZE - ((uintptr_t)pr->pr_WorkMem % PICOOS_ALIGN_SIZE);
   3217     }
   3218     pr->maxWorkMemTop=0;
   3219     pr->dynMemSize=0;
   3220     pr->maxDynMemSize=0;
   3221     /* this is ok to be in 'initialize' because it is a private memory within pr. Creating a new mm
   3222      * here amounts to resetting this internal memory
   3223      */
   3224     pr->dynMemMM = picoos_newMemoryManager((void *)pr->pr_DynMem, PR_DYN_MEM_SIZE,
   3225             /*enableMemProt*/ FALSE);
   3226     pr->outOfMemory = FALSE;
   3227 
   3228     pr->forceOutput = FALSE;
   3229 
   3230     if (resetMode == PICO_RESET_SOFT) {
   3231         /*following initializations needed only at startup or after a full reset*/
   3232         return PICO_OK;
   3233     }
   3234 
   3235     pr->xsampa_parser = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_XSAMPA_PARSE]);
   3236 
   3237     pr->svoxpa_parser = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_SVOXPA_PARSE]);
   3238 
   3239     pr->xsampa2svoxpa_mapper = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_XSAMPA2SVOXPA]);
   3240 
   3241 
   3242 
   3243     return PICO_OK;
   3244 }
   3245 
   3246 
   3247 pico_status_t prInitialize(register picodata_ProcessingUnit this, picoos_int32 resetMode)
   3248 {
   3249 /*
   3250     if (NULL == this || NULL == this->subObj) {
   3251         return PICO_ERR_OTHER;
   3252     }
   3253 */
   3254     return prReset(this, resetMode);
   3255 }
   3256 
   3257 
   3258 pico_status_t prTerminate(register picodata_ProcessingUnit this)
   3259 {
   3260     return PICO_OK;
   3261 }
   3262 
   3263 picodata_step_result_t prStep(register picodata_ProcessingUnit this, picoos_int16 mode, picoos_uint16 * numBytesOutput);
   3264 
   3265 pico_status_t prSubObjDeallocate(register picodata_ProcessingUnit this,
   3266         picoos_MemoryManager mm)
   3267 {
   3268     pr_subobj_t * pr;
   3269 
   3270     if (NULL != this) {
   3271         pr = (pr_subobj_t *) this->subObj;
   3272         mm = mm;        /* avoid warning "var not used in this function"*/
   3273         PICODBG_INFO(("max pr_WorkMem: %i of %i", pr->maxWorkMemTop, PR_WORK_MEM_SIZE));
   3274         PICODBG_INFO(("max pr_DynMem: %i of %i", pr->maxDynMemSize, PR_DYN_MEM_SIZE));
   3275 
   3276         pr_disposeContextList(this);
   3277         picoos_deallocate(this->common->mm, (void *) &this->subObj);
   3278     }
   3279     return PICO_OK;
   3280 }
   3281 
   3282 picodata_ProcessingUnit picopr_newPreprocUnit(picoos_MemoryManager mm, picoos_Common common,
   3283         picodata_CharBuffer cbIn, picodata_CharBuffer cbOut,
   3284         picorsrc_Voice voice)
   3285 {
   3286     picoos_int32 i;
   3287     pr_subobj_t * pr;
   3288 
   3289 
   3290     picodata_ProcessingUnit this = picodata_newProcessingUnit(mm, common, cbIn, cbOut, voice);
   3291     if (this == NULL) {
   3292         return NULL;
   3293     }
   3294 
   3295     this->initialize = prInitialize;
   3296     PICODBG_DEBUG(("set this->step to prStep"));
   3297     this->step = prStep;
   3298     this->terminate = prTerminate;
   3299     this->subDeallocate = prSubObjDeallocate;
   3300     this->subObj = picoos_allocate(mm, sizeof(pr_subobj_t));
   3301 #if PR_TRACE_MEM || PR_TRACE_MAX_MEM
   3302     PICODBG_INFO(("preproc alloc: %i", sizeof(pr_subobj_t)));
   3303     PICODBG_INFO(("max dyn size: %i", PR_MAX_PATH_LEN*((((PR_IOITEM_MIN_SIZE+2) + PICOOS_ALIGN_SIZE - 1) / PICOOS_ALIGN_SIZE) * PICOOS_ALIGN_SIZE + 16)));
   3304 #endif
   3305     if (this->subObj == NULL) {
   3306         picoos_deallocate(mm, (void *)&this);
   3307         return NULL;
   3308     }
   3309     pr = (pr_subobj_t *) this->subObj;
   3310 
   3311     pr->graphs = picoktab_getGraphs(this->voice->kbArray[PICOKNOW_KBID_TAB_GRAPHS]);
   3312     pr->preproc[0] = picokpr_getPreproc(this->voice->kbArray[PICOKNOW_KBID_TPP_MAIN]);
   3313     for (i=0; i<PICOKNOW_MAX_NUM_UTPP; i++) {
   3314       pr->preproc[1+i] = picokpr_getPreproc(this->voice->kbArray[PICOKNOW_KBID_TPP_USER_1+i]);
   3315     }
   3316 
   3317    if (pr_createContextList(this) != PICO_OK) {
   3318         pr_disposeContextList(this);
   3319         picoos_deallocate(mm, (void *)&this);
   3320         return NULL;
   3321     }
   3322     prInitialize(this, PICO_RESET_FULL);
   3323     return this;
   3324 }
   3325 
   3326 /**
   3327  * fill up internal buffer
   3328  */
   3329 picodata_step_result_t prStep(register picodata_ProcessingUnit this,
   3330         picoos_int16 mode, picoos_uint16 * numBytesOutput)
   3331 {
   3332     register pr_subobj_t * pr;
   3333     pr_ioItemPtr it;
   3334     picoos_int32 len, i;
   3335     pico_status_t rv;
   3336     picoos_int32 id;
   3337     picoos_uint8 info1;
   3338     picoos_uint8 info2;
   3339     picoos_int32 nrUtfChars;
   3340     picoos_uint32 pos;
   3341     picobase_utf8char inUtf8char, outUtf8char;
   3342     picoos_int32 inUtf8charlen, outUtf8charlen;
   3343     picoos_int32 lenpos;
   3344     picoos_bool ldone;
   3345     picoos_bool split;
   3346 
   3347     if (NULL == this || NULL == this->subObj) {
   3348         return PICODATA_PU_ERROR;
   3349     }
   3350     pr = (pr_subobj_t *) this->subObj;
   3351 
   3352     if (pr->outOfMemory) return PICODATA_PU_ERROR;
   3353 
   3354     mode = mode;        /* avoid warning "var not used in this function"*/
   3355     pr->nrIterations = PR_MAX_NR_ITERATIONS;
   3356 
   3357     *numBytesOutput = 0;
   3358     while (1) { /* exit via return */
   3359         if ((pr->outWritePos - pr->outReadPos) > 0) {
   3360             /* deliver the data in the output buffer */
   3361             if (picodata_cbPutItem(this->cbOut, &pr->outBuf[pr->outReadPos], pr->outWritePos - pr->outReadPos, numBytesOutput) == PICO_OK) {
   3362                 pr->outReadPos += *numBytesOutput;
   3363                 if (pr->outWritePos == pr->outReadPos) {
   3364                     pr->outWritePos = 0;
   3365                     pr->outReadPos = 0;
   3366                 }
   3367             }
   3368             else {
   3369                 return PICODATA_PU_OUT_FULL;
   3370             }
   3371         }
   3372         else if (pr->routItemList != NULL) {
   3373             /* there are item(s) in the output item list, move them to the output buffer */
   3374             it = pr->routItemList;
   3375             pr->routItemList = pr->routItemList->next;
   3376             if (pr->routItemList == NULL) {
   3377                 pr->rlastOutItem = NULL;
   3378             }
   3379             if (it->head.type == PICODATA_ITEM_TOKEN) {
   3380                 if ((it->head.info1 != PICODATA_ITEMINFO1_TOKTYPE_SPACE) && (it->head.len > 0)) {
   3381                     nrUtfChars = picobase_utf8_length(it->data, PR_MAX_DATA_LEN);
   3382                     if ((nrUtfChars == 1)
   3383                         && (((id = picoktab_graphOffset(pr->graphs, it->data)) > 0))
   3384                         && picoktab_getIntPropPunct(pr->graphs, id, &info1, &info2)) {
   3385                         /* single punctuation chars have to be delivered as PICODATA_ITEM_PUNC items
   3386                            instead as PICODATA_ITEM_WORDGRAPH items */
   3387                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEM_PUNC;
   3388                         pr->outBuf[pr->outWritePos++] = info1;
   3389                         pr->outBuf[pr->outWritePos++] = info2;
   3390                         pr->outBuf[pr->outWritePos++] = 0;
   3391                         PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
   3392                             (picoos_uint8 *)"pr: ", pr->outBuf, pr->outWritePos);
   3393                     }
   3394                     else {
   3395                         /* do subgraphs substitutions and deliver token items as PICODATA_ITEM_WORDGRAPH
   3396                            items to the output buffer */
   3397                         split = FALSE;
   3398                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEM_WORDGRAPH;
   3399                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO1_NA;
   3400                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO2_NA;
   3401                         lenpos=pr->outWritePos;
   3402                         pr->outBuf[pr->outWritePos++] = 0;
   3403                         pos = 0;
   3404                         len = pr_strlen(it->data);
   3405                         while (pos < (picoos_uint32)len) {
   3406                             if (picobase_get_next_utf8char(it->data, it->head.len, &pos, inUtf8char)) {
   3407                                 if (inUtf8char[0] <= 32) {
   3408                                     /* do not add whitespace characters to the output buffer,
   3409                                        but initiate token splitting instead
   3410 
   3411                                     */
   3412                                     split = TRUE;
   3413                                 }
   3414                                 else if (((id = picoktab_graphOffset(pr->graphs, inUtf8char)) > 0) && picoktab_getStrPropGraphsubs1(pr->graphs, id, outUtf8char)) {
   3415                                     if (split) {
   3416                                         /* split the token, eg. start a new item */
   3417                                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEM_WORDGRAPH;
   3418                                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO1_NA;
   3419                                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO2_NA;
   3420                                         lenpos=pr->outWritePos;
   3421                                         pr->outBuf[pr->outWritePos++] = 0;
   3422                                     }
   3423                                     outUtf8charlen = picobase_det_utf8_length(outUtf8char[0]);
   3424                                     for (i=0; i<outUtf8charlen; i++) {
   3425                                         pr->outBuf[pr->outWritePos++] = outUtf8char[i];
   3426                                         pr->outBuf[lenpos]++;
   3427                                     }
   3428                                     if (picoktab_getStrPropGraphsubs2(pr->graphs, id, outUtf8char)) {
   3429                                         outUtf8charlen = picobase_det_utf8_length(outUtf8char[0]);
   3430                                         for (i=0; i<outUtf8charlen; i++) {
   3431                                             pr->outBuf[pr->outWritePos++] = outUtf8char[i];
   3432                                             pr->outBuf[lenpos]++;
   3433                                         }
   3434                                     }
   3435                                     split = FALSE;
   3436                                 }
   3437                                 else {
   3438                                     if (split) {
   3439                                         /* split the token, eg. start a new item */
   3440                                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEM_WORDGRAPH;
   3441                                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO1_NA;
   3442                                         pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO2_NA;
   3443                                         lenpos=pr->outWritePos;
   3444                                         pr->outBuf[pr->outWritePos++] = 0;
   3445                                     }
   3446                                     inUtf8charlen = picobase_det_utf8_length(inUtf8char[0]);
   3447                                     for (i=0; i<inUtf8charlen; i++) {
   3448                                         pr->outBuf[pr->outWritePos++] = inUtf8char[i];
   3449                                         pr->outBuf[lenpos]++;
   3450                                     }
   3451                                     split = FALSE;
   3452                                 }
   3453                             }
   3454                         }
   3455                         PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
   3456                             (picoos_uint8 *)"pr: ", pr->outBuf, pr->outWritePos);
   3457                     }
   3458                 }
   3459             }
   3460             else {
   3461                 /* handle all other item types and put them to the output buffer */
   3462                 pr->outBuf[pr->outWritePos++] = it->head.type;
   3463                 pr->outBuf[pr->outWritePos++] = it->head.info1;
   3464                 pr->outBuf[pr->outWritePos++] = it->head.info2;
   3465                 pr->outBuf[pr->outWritePos++] = it->head.len;
   3466                 for (i=0; i<it->head.len; i++) {
   3467                     pr->outBuf[pr->outWritePos++] = it->data[i];
   3468                 }
   3469                 PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
   3470                                    (picoos_uint8 *)"pr: ", pr->outBuf, pr->outWritePos);
   3471             }
   3472             pr_disposeItem(this, &it);
   3473         }
   3474         else if (pr->forceOutput) {
   3475             pr_processItems(this, pr);
   3476             if (pr->rinItemList == NULL) {
   3477                 pr->forceOutput = FALSE;
   3478             }
   3479         }
   3480         else if ((pr->rgState != PR_GSNeedToken) && (pr->rinItemList != NULL)) {
   3481             pr_processItems(this, pr);
   3482         }
   3483         else if (pr->inBufLen > 0) {
   3484             /* input data is available in the input buffer, copy it to an input item
   3485                and treat it */
   3486             if (pr->dynMemSize < (45*PR_DYN_MEM_SIZE / 100)) {
   3487                 pr_newItem(this, pr_DynMem, &it, pr->inBuf[0], pr->inBuf[3], /*inItem*/TRUE);
   3488                 if (pr->outOfMemory) return PICODATA_PU_ERROR;
   3489                 it->head.type = pr->inBuf[0];
   3490                 it->head.info1 = pr->inBuf[1];
   3491                 it->head.info2 = pr->inBuf[2];
   3492                 it->head.len = pr->inBuf[3];
   3493                 for (i=0; i<pr->inBuf[3]; i++) {
   3494                     it->data[i] = pr->inBuf[4+i];
   3495                 }
   3496                 it->data[pr->inBuf[3]] = 0;
   3497                 if ((pr->inBuf[0] == PICODATA_ITEM_TOKEN) && ((pr->inBuf[1] == PICODATA_ITEMINFO1_TOKTYPE_DIGIT))) {
   3498                     it->val = tok_tokenDigitStrToInt(this, pr, it->data);
   3499                 } else {
   3500                     it->val = 0;
   3501                 }
   3502                 if (pr->inBuf[0] == PICODATA_ITEM_TOKEN) {
   3503                     picobase_lowercase_utf8_str(it->data,it->strci,PR_MAX_DATA_LEN, &ldone);
   3504                     pr_firstLetterToLowerCase(it->data,it->strcis);
   3505                     it->alc = picobase_is_utf8_lowercase(it->data,PR_MAX_DATA_LEN);
   3506                     it->auc = picobase_is_utf8_uppercase(it->data,PR_MAX_DATA_LEN);
   3507                     it->suc = pr_isSUC(it->data);
   3508                 }
   3509 
   3510                 pr_treatItem(this, pr, it);
   3511                 if (pr->outOfMemory) return PICODATA_PU_ERROR;
   3512                 pr_processItems(this, pr);
   3513                 pr->inBufLen = 0;
   3514             }
   3515             else {
   3516                 pr->forceOutput = TRUE;
   3517             }
   3518         }
   3519         else {
   3520             /* there is not data in the output buffer and there is no data in the output item list, so
   3521                check whether input data is available */
   3522             rv = picodata_cbGetItem(this->cbIn, pr->inBuf, IN_BUF_SIZE+PICODATA_ITEM_HEADSIZE, &pr->inBufLen);
   3523             if (PICO_OK == rv) {
   3524             } else if (PICO_EOF == rv) {
   3525                 /* there was no item in the char buffer */
   3526                 return PICODATA_PU_IDLE;
   3527             } else if ((PICO_EXC_BUF_UNDERFLOW == rv) || (PICO_EXC_BUF_OVERFLOW == rv)) {
   3528                 pr->inBufLen = 0;
   3529                 PICODBG_ERROR(("problem getting item"));
   3530                 picoos_emRaiseException(this->common->em, rv, NULL, NULL);
   3531                 return PICODATA_PU_ERROR;
   3532             } else {
   3533                 pr->inBufLen = 0;
   3534                 PICODBG_ERROR(("problem getting item, unhandled"));
   3535                 picoos_emRaiseException(this->common->em, rv, NULL, NULL);
   3536                 return PICODATA_PU_ERROR;
   3537             }
   3538         }
   3539 #if PR_TRACE_MEM
   3540         PICODBG_INFO(("memory: dyn=%u, work=%u", pr->dynMemSize, pr->workMemTop));
   3541 #endif
   3542         if (pr->nrIterations <= 0) {
   3543             return PICODATA_PU_BUSY;
   3544         }
   3545     } /* while */
   3546     return PICODATA_PU_ERROR;
   3547 }
   3548 
   3549 #ifdef __cplusplus
   3550 }
   3551 #endif
   3552 
   3553 
   3554 
   3555 /* end */
   3556