1 /* 2 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 /** 17 * @file picospho.c 18 * 19 * sentence phonemic/phonetic FSTs PU 20 * 21 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland 22 * All rights reserved. 23 * 24 * History: 25 * - 2009-04-20 -- initial version 26 * 27 */ 28 29 #include "picoos.h" 30 #include "picodbg.h" 31 #include "picodata.h" 32 33 #include "picoknow.h" 34 #include "picokfst.h" 35 #include "picoktab.h" 36 #include "picotrns.h" 37 38 #include "picospho.h" 39 40 #ifdef __cplusplus 41 extern "C" { 42 #endif 43 #if 0 44 } 45 #endif 46 47 #define SPHO_BUFSIZE (3 * PICODATA_BUFSIZE_DEFAULT) 48 49 50 51 #define SPHO_MAX_ALTDESC_SIZE (60 * PICOTRNS_MAX_NUM_POSSYM) 52 53 54 #define SPHO_SMALLEST_SIL_DUR 1 55 56 57 /** @addtogroup picospho 58 * 59 * Algorithmic description 60 * ======================= 61 * The main function, sphoStep, is divided into the subprocesses (processing states) described further down. 62 * 63 * Flow control: 64 * ------------ 65 * The processing flow is controlled by setting 66 * - 'procState' : the next state to be processed 67 * - 'feedFollowState' : the state to be processed after the feed state (the feed state is treated like a primitive "subroutine") 68 * - some other flags 69 * 70 * Buffering: 71 * --------- 72 * - The input items are mainly stored and processed in two buffers, collectively called 'inBuf' 73 * - cbuf : unstructured buffer containing item contents 74 * - headx : structured buffer containing item heads, each expanded by a pointer to the item contents 75 * and space for a boundary potentially to be inserted (to the left of the original item) 76 * - For transduction, phonemes and their position are extracted from inBuf into 77 * - phonBuf, 78 * processed there, and the resulting phonemes realigned with inBuf. 79 * - Word items are split into syllables, stored in 80 * - sylBuf 81 * - Items to be output are stored in outBuf 82 * 83 * Windowing: 84 * --------- 85 * Optimal solutions are achieved if a whole sentence is processed at once. However, if any of the buffers are too small, 86 * only sentence parts are processed. To improve the quality of such sub-optimal solutions, a moving-window-with-overlap is applied: 87 * - [0,headxReadPos[ : the window considered for transduction 88 * - [activeStartPos,activeEndPos[ : the "active" subrange of the window actually used for output 89 * - penultima : the position (within the active range) that should be used as new window start when shifting the window 90 * 91 * After PROCESS_PARSE: 92 * 0 activeStartPos penultima activeEndPos headxReadPos headxWritePos 93 * | | | | | | 94 * |-------------=================================---------------| ['----': context '====' : active subrange) 95 * 96 * After PROCESS_SHIFT: 97 * 0 activeStartPos headWritePos 98 * | | | | 99 * |------------... (only left context is known; new active range, penultima, and right context to be established at next parse) 100 * 101 * Processing states: 102 * ----------------- 103 * - INIT : initialize state variables 104 * - COLLECT : collect items into internal buffers ("inBuf") 105 * - PROCESS_PARSE : go through inBuf items and extract position/phoneme pairs into phoneme buffer 'phonBuf' 106 * word boundary phonemes are inserted between words 107 * - PROCESS_TRANSDUCE : transduce phonBuf 108 * - PROCESS_BOUNDS : go through inBuf items again and match against transduced pos/phoneme 109 * this is the first round of alignment, only inserting/deleting/modifying bounds, according to 110 * - existing BOUND items 111 * - newly produced word bounds separating WORDPHON items 112 * - bound upgrades/downgrades from transduction 113 * - bound upgrades/downgrades/insertions from SIL command items (originating e.g. from <break> text commands) 114 * all relevant bounds are placed in the corresponding headx extention; original bound items become invalid. 115 * - PROCESS_RECOMB : go through inBuf items again and match against transduced pos/phoneme 116 * this is the second round of alignment, treating non-BOUND items 117 * - WORDPHONs are broken into syllables by "calling" PROCESS_SYL 118 * - "side-bounds" (in the headx extension) are output by "calling" FEED 119 * - BOUND items are consumed with no effect 120 * - other items are output unchanged "calling" FEED 121 * - PROCESS_SYL : the WORDPHON coming from RECOMB is matched against the phonBuf and (new) SYLLPHON items 122 * are created. (the original wordphon is consumed) 123 * - FEED : feeds one item and returns to spho->feedFollowState 124 * - SHIFT : items in inBuf are shifted left to make room for new items. If a sentence doesn't fit 125 * inBuf in its entirety, left and/or right contexts are kept so they can be considered in 126 * the next transduction. 127 */ 128 129 130 131 /* PU sphoStep states */ 132 #define SPHO_STEPSTATE_INIT 0 133 #define SPHO_STEPSTATE_COLLECT 1 134 #define SPHO_STEPSTATE_PROCESS_PARSE 2 135 #define SPHO_STEPSTATE_PROCESS_TRANSDUCE 3 136 #define SPHO_STEPSTATE_PROCESS_BOUNDS 4 137 #define SPHO_STEPSTATE_PROCESS_RECOMB 5 138 #define SPHO_STEPSTATE_PROCESS_SYL 6 139 #define SPHO_STEPSTATE_FEED 7 140 #define SPHO_STEPSTATE_SHIFT 8 141 142 #define SPHO_POS_INVALID (PICOTRNS_POS_INVALID) /* indicates that no position was set yet */ 143 144 /* nr item restriction: maximum number of extended item heads in headx */ 145 #define SPHO_MAXNR_HEADX 60 146 147 /* nr item restriction: maximum size of all item contents together in cont */ 148 #define SPHO_MAXSIZE_CBUF (30 * 255) 149 150 /* "expanded head": item head expanded by a content position and a by boundary information 151 * potentially inserted "to the left" of the item */ 152 typedef struct { 153 picodata_itemhead_t head; 154 picoos_uint16 cind; 155 picoos_uint8 boundstrength; /* bstrength to the left, 0 if not set */ 156 picoos_uint8 phrasetype; /* btype for following phrase, 0 if not set */ 157 picoos_int16 sildur; /* silence duration for boundary, -1 if not set */ 158 } picospho_headx_t; 159 160 161 162 #define SPHO_MSGSTR_SIZE 32 163 164 /** object : SentPhoUnit 165 * shortcut : spho 166 * derived from : picodata_ProcessingUnit 167 */ 168 typedef struct spho_subobj { 169 picoos_Common common; 170 171 /* we use int16 for buffer positions so we can indicate exceptional positions (invalid etc.) with negative 172 * integers */ 173 picoos_uint8 procState; /* for next processing step decision */ 174 175 /* buffer for item headers */ 176 picoos_uint8 tmpbuf[PICODATA_MAX_ITEMSIZE]; /* tmp. location for an item */ 177 178 picospho_headx_t headx[SPHO_MAXNR_HEADX]; /* "expanded head" buffer */ 179 picoos_uint16 headxBufSize; /* actually allocated size (if one day headxBuf is allocated dynamically) */ 180 picoos_uint16 headxReadPos, headxWritePos; 181 182 picoos_uint8 cbuf[SPHO_MAXSIZE_CBUF]; 183 picoos_uint16 cbufBufSize; /* actually allocated size */ 184 picoos_uint16 cbufWritePos; /* next position to write to, 0 if buffer empty */ 185 186 picoos_uint8 outBuf[PICODATA_BUFSIZE_DEFAULT]; /* internal output buffer to hold just one item */ 187 picoos_uint16 outBufSize; /* actually allocated size (if one day outBuf is allocated dynamically) */ 188 picoos_uint16 outReadPos; /* next pos to read from inBuf for output */ 189 190 /* picoos_int16 outWritePos; */ /* next pos to output from in buf */ 191 192 picoos_uint8 sylBuf[255]; /* internal buffer to hold contents of syl item to be output */ 193 picoos_uint8 sylReadPos, sylWritePos; /* next pos to read from sylBuf, next pos to write to sylBuf */ 194 195 /* buffer for internal calculation of transducer */ 196 picotrns_AltDesc altDescBuf; 197 /* the number of AltDesc in the buffer */ 198 picoos_uint16 maxAltDescLen; 199 200 /* the input to a transducer should not be larger than PICOTRNS_MAX_NUM_POSSYM 201 * so the output may expand (up to 4*PICOTRNS_MAX_NUM_POSSYM) */ 202 203 picotrns_possym_t phonBufA[4 * PICOTRNS_MAX_NUM_POSSYM + 1]; 204 picotrns_possym_t phonBufB[4 * PICOTRNS_MAX_NUM_POSSYM + 1]; 205 picotrns_possym_t * phonBuf; 206 picotrns_possym_t * phonBufOut; 207 picoos_uint16 phonReadPos, phonWritePos; /* next pos to read from phonBufIn, next pos to write to phonBufIn */ 208 209 picoos_int16 activeStartPos; /* start position of items to be treated (at end of left context) */ 210 picoos_int16 penultima, activeEndPos; /* positions of last two bounds/words; SPHO_POS_INVALID means uninitialized */ 211 picoos_int16 lastPhraseBoundPos; /* position of the last bound encountered (<0 if inexistent or not reachable */ 212 picoos_uint8 lastPhraseType; /* phrase type of the last phrase boundary, 0 if not set */ 213 214 picoos_uint8 needMoreInput, /* more data necessary to decide on token */ 215 suppressParseWordBound, /* dont produce word boundary */ 216 suppressRecombWordBound, /* dont produce word boundary */ 217 breakPending, /* received a break but didn't interpret it yet */ 218 /* sentEnd, */ /* sentence end detected */ 219 force, /* in forced state */ 220 wordStarted, /* is it the first syl in the word: expect POS */ 221 sentenceStarted; 222 223 picoos_uint16 breakTime; /* time argument of the pending break command */ 224 225 picoos_uint8 feedFollowState; /* where to return after feed */ 226 227 /* fst knowledge bases */ 228 picoos_uint8 numFsts; 229 picokfst_FST fst[PICOKNOW_MAX_NUM_SPHO_FSTS]; 230 picoos_uint8 curFst; /* the fst to be applied next */ 231 232 /* fixed ids knowledge base */ 233 picoktab_FixedIds fixedIds; 234 235 /* phones kb */ 236 picoktab_Phones phones; 237 238 /* some soecial ids from phones */ 239 picoos_uint8 primStressId, secondStressId, syllSepId; 240 241 } spho_subobj_t; 242 243 244 static pico_status_t sphoReset(register picodata_ProcessingUnit this) 245 { 246 247 spho_subobj_t * spho; 248 249 if (NULL == this || NULL == this->subObj) { 250 return picoos_emRaiseException(this->common->em, 251 PICO_ERR_NULLPTR_ACCESS, NULL, NULL); 252 } 253 spho = (spho_subobj_t *) this->subObj; 254 255 spho->curFst = 0; 256 257 /* processing state */ 258 spho->procState = SPHO_STEPSTATE_INIT; 259 spho->needMoreInput = TRUE; 260 spho->suppressParseWordBound = FALSE; 261 spho->suppressRecombWordBound = FALSE; 262 spho->breakPending = FALSE; 263 spho->force = 0; 264 spho->sentenceStarted = 0; 265 266 267 /* item buffer headx/cbuf */ 268 spho->headxBufSize = SPHO_MAXNR_HEADX; 269 spho->headxReadPos = 0; 270 spho->headxWritePos = 0; 271 272 spho->cbufWritePos = 0; 273 spho->cbufBufSize = SPHO_MAXSIZE_CBUF; 274 275 /* possym buffer */ 276 spho->phonBuf = spho->phonBufA; 277 spho->phonBufOut = spho->phonBufB; 278 spho->phonReadPos = 0; 279 280 /* overlapping */ 281 spho->activeStartPos = 0; 282 spho->penultima = SPHO_POS_INVALID; 283 spho->activeEndPos = SPHO_POS_INVALID; 284 285 return PICO_OK; 286 } 287 288 289 static pico_status_t sphoInitialize(register picodata_ProcessingUnit this, picoos_int32 resetMode) 290 { 291 picoos_uint8 i; 292 spho_subobj_t * spho; 293 picokfst_FST fst; 294 295 picoknow_kb_id_t myKbIds[PICOKNOW_MAX_NUM_SPHO_FSTS] = PICOKNOW_KBID_SPHO_ARRAY; 296 297 PICODBG_DEBUG(("init")); 298 299 if (NULL == this || NULL == this->subObj) { 300 return picoos_emRaiseException(this->common->em, 301 PICO_ERR_NULLPTR_ACCESS, NULL, NULL); 302 } 303 304 spho = (spho_subobj_t *) this->subObj; 305 306 spho->numFsts = 0; 307 308 spho->curFst = 0; 309 310 for (i = 0; i<PICOKNOW_MAX_NUM_SPHO_FSTS; i++) { 311 fst = picokfst_getFST(this->voice->kbArray[myKbIds[i]]); 312 if (NULL != fst) { 313 spho->fst[spho->numFsts++] = fst; 314 } 315 } 316 spho->fixedIds = picoktab_getFixedIds(this->voice->kbArray[PICOKNOW_KBID_FIXED_IDS]); 317 spho->phones = picoktab_getPhones(this->voice->kbArray[PICOKNOW_KBID_TAB_PHONES]); 318 319 spho->syllSepId = picoktab_getSyllboundID(spho->phones); 320 spho->primStressId = picoktab_getPrimstressID(spho->phones); 321 spho->secondStressId = picoktab_getSecstressID(spho->phones); 322 323 PICODBG_DEBUG(("got %i fsts", spho->numFsts)); 324 325 326 return sphoReset(this); 327 328 } 329 330 static picodata_step_result_t sphoStep(register picodata_ProcessingUnit this, 331 picoos_int16 mode, picoos_uint16 *numBytesOutput); 332 333 334 335 336 static pico_status_t sphoTerminate(register picodata_ProcessingUnit this) 337 { 338 return PICO_OK; 339 } 340 341 342 static pico_status_t sphoSubObjDeallocate(register picodata_ProcessingUnit this, 343 picoos_MemoryManager mm) 344 { 345 spho_subobj_t * spho; 346 347 spho = (spho_subobj_t *) this->subObj; 348 349 if (NULL != this) { 350 if (NULL != this->subObj) { 351 spho = (spho_subobj_t *) (this->subObj); 352 picotrns_deallocate_alt_desc_buf(spho->common->mm,&spho->altDescBuf); 353 picoos_deallocate(mm, (void *) &this->subObj); 354 } 355 } 356 return PICO_OK; 357 } 358 359 picodata_ProcessingUnit picospho_newSentPhoUnit(picoos_MemoryManager mm, 360 picoos_Common common, picodata_CharBuffer cbIn, 361 picodata_CharBuffer cbOut, picorsrc_Voice voice) 362 { 363 spho_subobj_t * spho; 364 365 picodata_ProcessingUnit this = picodata_newProcessingUnit(mm, common, cbIn, cbOut, voice); 366 if (this == NULL) { 367 return NULL; 368 } 369 370 this->initialize = sphoInitialize; 371 this->step = sphoStep; 372 this->terminate = sphoTerminate; 373 this->subDeallocate = sphoSubObjDeallocate; 374 375 this->subObj = picoos_allocate(mm, sizeof(spho_subobj_t)); 376 if (this->subObj == NULL) { 377 picoos_deallocate(mm, (void **)(void*)&this); 378 return NULL; 379 } 380 spho = (spho_subobj_t *) this->subObj; 381 382 spho->common = this->common; 383 384 /* these are given by the pre-allocated array sizes */ 385 spho->outBufSize = PICODATA_BUFSIZE_DEFAULT; 386 387 388 spho->altDescBuf = picotrns_allocate_alt_desc_buf(spho->common->mm, SPHO_MAX_ALTDESC_SIZE, &spho->maxAltDescLen); 389 if (NULL == spho->altDescBuf) { 390 picotrns_deallocate_alt_desc_buf(spho->common->mm,&spho->altDescBuf); 391 picoos_emRaiseException(spho->common->em,PICO_EXC_OUT_OF_MEM, NULL,NULL); 392 return NULL; 393 } 394 395 sphoInitialize(this, PICO_RESET_FULL); 396 return this; 397 } 398 399 400 /* ***********************************************************************/ 401 /* process buffered item list */ 402 /* ***********************************************************************/ 403 404 405 /* shift relevant data in headx/'cbuf' (between 'readPos' incl and writePos non-incl) to 'start'. 406 * modify read/writePos accordingly */ 407 static picoos_int16 shift_range_left_1(spho_subobj_t *spho, picoos_int16 * from, picoos_int16 to) 408 { 409 410 /* remember shift parameters for cbuf */ 411 picoos_uint16 412 c_i, 413 c_j, 414 c_diff, 415 c_writePos, 416 i, 417 j, 418 diff, 419 writePos; 420 i = to; 421 j = *from; 422 diff = j-i; 423 writePos = spho->headxWritePos; 424 c_i = spho->headx[to].cind; 425 if (j < writePos) { 426 c_j = spho->headx[j].cind; 427 } else { 428 c_j = spho->cbufWritePos; 429 } 430 c_diff = c_j - c_i; 431 c_writePos = spho->cbufWritePos; 432 433 PICODBG_DEBUG(( 434 "shifting buffer region [%i,%i[ down to %i",*from, writePos, to 435 )); 436 437 438 /* PICODBG_ASSERT((i<j)); */ 439 if (i > j) { 440 return -1; 441 } 442 /* shift cbuf */ 443 while (c_j < c_writePos) { 444 spho->cbuf[c_i++] = spho->cbuf[c_j++]; 445 } 446 /* shift headx */ 447 while (j < writePos) { 448 spho->headx[j].cind -= c_diff; 449 spho->headx[i++] = spho->headx[j++]; 450 } 451 spho->headxWritePos -= diff; 452 *from = to; 453 spho->cbufWritePos -= c_diff; 454 /* */ 455 PICODBG_DEBUG(( 456 "readPos,WritePos are now [%i,%i[, returning shift amount %i",*from, spho->headxWritePos, diff 457 )); 458 return diff; 459 } 460 461 static pico_status_t sphoAddPhoneme(register spho_subobj_t *spho, picoos_int16 pos, picoos_int16 sym) { 462 picoos_uint8 plane, unshifted; 463 /* just for debuging */ 464 unshifted = picotrns_unplane(sym,&plane); 465 PICODBG_TRACE(("adding %i/%i (%c on plane %i) at phonBuf[%i]",pos,sym,unshifted,plane,spho->phonWritePos)); 466 if (2* PICOTRNS_MAX_NUM_POSSYM <= spho->phonWritePos) { 467 /* not an error! */ 468 PICODBG_DEBUG(("couldn't add because phon buffer full")); 469 return PICO_EXC_BUF_OVERFLOW; 470 } else { 471 spho->phonBuf[spho->phonWritePos].pos = pos; 472 spho->phonBuf[spho->phonWritePos].sym = sym; 473 spho->phonWritePos++; 474 return PICO_OK; 475 } 476 } 477 478 static pico_status_t sphoAddStartPhoneme(register spho_subobj_t *spho) { 479 return sphoAddPhoneme(spho, PICOTRNS_POS_IGNORE, 480 (PICOKFST_PLANE_INTERN << 8) + spho->fixedIds->phonStartId); 481 } 482 483 static pico_status_t sphoAddTermPhonemes(register spho_subobj_t *spho, picoos_uint16 pos) { 484 return sphoAddPhoneme(spho, pos, 485 (PICOKFST_PLANE_PB_STRENGTHS << 8) + PICODATA_ITEMINFO1_BOUND_SEND) 486 && sphoAddPhoneme(spho, PICOTRNS_POS_IGNORE, 487 (PICOKFST_PLANE_INTERN << 8) + spho->fixedIds->phonTermId); 488 } 489 490 /* return "syllable accent" (or prominence) symbol, given "word accent" symbol 'wacc' and stress value (no=0, primary=1, secondary=2) */ 491 static picoos_uint16 sphoGetSylAccent(register spho_subobj_t *spho, 492 picoos_uint8 wacc, picoos_uint8 sylStress) 493 { 494 PICODBG_ASSERT(sylStress <= 2); 495 496 spho = spho; /* avoid warning "var not used in this function"*/ 497 498 switch (sylStress) { 499 case 0: /* non-stressed syllable gets no prominence */ 500 /* return spho->fixedIds->accId[0]; */ 501 return PICODATA_ACC0; 502 break; 503 case 1: /* primary-stressed syllable gets word prominence */ 504 return wacc; 505 break; 506 case 2: /* secondary-stressed syllable gets no prominence or secondary stress prom. (4) */ 507 return (PICODATA_ACC0 == wacc) ? PICODATA_ACC0 508 : PICODATA_ACC4; 509 /*return (spho->fixedIds->accId[0] == wacc) ? spho->fixedIds->accId[0] 510 : spho->fixedIds->accId[4]; */ 511 break; 512 default: 513 /* never occurs :-) */ 514 return PICODATA_ACC0; 515 break; 516 } 517 } 518 519 520 /* ***********************************************************************/ 521 /* extract phonemes of an item into a phonBuf */ 522 /* ***********************************************************************/ 523 static pico_status_t sphoExtractPhonemes(register picodata_ProcessingUnit this, 524 register spho_subobj_t *spho, picoos_uint16 pos, 525 picoos_uint8 convertAccents, picoos_uint8 * suppressWB) 526 { 527 pico_status_t rv = PICO_OK; 528 picoos_uint16 i, j; 529 picoos_int16 fstSymbol; 530 picoos_uint8 curStress; 531 picotrns_possym_t tmpPosSym; 532 picoos_uint16 oldPos, curPos; 533 picodata_itemhead_t * head; 534 picoos_uint8* content; 535 536 #if defined(PICO_DEBUG) 537 picoos_char msgstr[SPHO_MSGSTR_SIZE]; 538 #endif 539 540 541 /* 542 Items considered in a transduction are a BOUND or a WORDPHON item. its starting offset within the 543 headxBuf is given as 'pos'. 544 Elements that go into the transduction receive "their" position in the buffer. 545 */ 546 547 oldPos = spho->phonWritePos; 548 549 head = &(spho->headx[pos].head); 550 content = spho->cbuf + spho->headx[pos].cind; 551 552 PICODBG_TRACE(("doing item %s\n", 553 picodata_head_to_string(head,msgstr,SPHO_MSGSTR_SIZE))); 554 555 switch (head->type) { 556 case PICODATA_ITEM_BOUND: 557 /* map SBEG, SEND and TERM (as sentence closing) to SEND */ 558 fstSymbol = (PICODATA_ITEMINFO1_BOUND_SBEG == head->info1 || PICODATA_ITEMINFO1_BOUND_TERM == head->info1) ? PICODATA_ITEMINFO1_BOUND_SEND : head->info1; 559 PICODBG_TRACE(("found bound of type %c\n",head->info1)); 560 /* BOUND(<bound strength><phrase type>) */ 561 /* insert bound strength */ 562 PICODBG_TRACE(("inserting phrase bound phoneme %c and setting suppresWB=1\n",fstSymbol)); 563 fstSymbol += (PICOKFST_PLANE_PB_STRENGTHS << 8); 564 rv = sphoAddPhoneme(spho,pos,fstSymbol); 565 /* phrase type not used */ 566 /* suppress next word boundary */ 567 (*suppressWB) = 1; 568 break; 569 570 case PICODATA_ITEM_WORDPHON: 571 /* WORDPHON(POS,WACC)phon */ 572 PICODBG_TRACE(("found WORDPHON")); 573 /* insert word boundary if not suppressed */ 574 if (!(*suppressWB)) { 575 fstSymbol = (PICOKFST_PLANE_PB_STRENGTHS << 8) + PICODATA_ITEMINFO1_BOUND_PHR0; 576 PICODBG_TRACE(("adding word boundary phone")); 577 rv = sphoAddPhoneme(spho,pos,fstSymbol); 578 } 579 (*suppressWB) = 0; 580 /* for the time being, we force to use POS so we can transduce all fsts in a row without reconsulting the items */ 581 582 583 /* If 'convertAccents' then the accentuation is not directly encoded. It rather influences the mapping of 584 * the word accent symbol to the actual accent phoneme which is put after the syllable separator. */ 585 if (convertAccents) { 586 PICODBG_TRACE(("converting accents")); 587 /* extracting phonemes IN REVERSE order replacing syllable symbols with prominence symbols */ 588 curPos = spho->phonWritePos; 589 curStress = 0; /* no stress */ 590 for (i = head->len; i > 0 ;) { 591 i--; 592 if (spho->primStressId == content[i]) { 593 curStress = 1; 594 PICODBG_DEBUG(("skipping primary stress at pos %i (in 1 .. %i)",i, head->len)); 595 continue; /* skip primary stress symbol */ 596 } else if (spho->secondStressId == content[i]) { 597 curStress = 2; 598 PICODBG_DEBUG(("skipping secondary stress at pos %i (in 1 .. %i)",i, head->len)); 599 continue; /* skip secundary stress symbol */ 600 } else if (spho->syllSepId == content[i]) { 601 fstSymbol = (PICOKFST_PLANE_POS << 8) + head->info1; 602 rv = sphoAddPhoneme(spho, pos, fstSymbol); 603 /* replace syllSepId by combination of syllable stress and word prominence */ 604 fstSymbol = sphoGetSylAccent(spho,head->info2,curStress); 605 curStress = 0; 606 /* add accent */ 607 fstSymbol += (PICOKFST_PLANE_ACCENTS << 8); 608 rv = sphoAddPhoneme(spho,pos,fstSymbol); 609 if (PICO_OK != rv) { 610 break; 611 } 612 /* and keep syllable boundary */ 613 fstSymbol = (PICOKFST_PLANE_PHONEMES << 8) + content[i]; 614 } else { 615 /* normal phoneme */ 616 fstSymbol = (PICOKFST_PLANE_PHONEMES << 8) + content[i]; 617 } 618 if (PICO_OK == rv) { 619 rv = sphoAddPhoneme(spho,pos,fstSymbol); 620 } 621 } 622 if (PICO_OK == rv) { 623 /* bug 366: we position the "head" into the item header and not on the first phoneme 624 * because there might be no phonemes at all */ 625 /* insert head of the first syllable of a word */ 626 fstSymbol = (PICOKFST_PLANE_POS << 8) + head->info1; 627 rv = sphoAddPhoneme(spho,pos,fstSymbol); 628 fstSymbol = sphoGetSylAccent(spho,head->info2,curStress); 629 curStress = 0; 630 fstSymbol += (PICOKFST_PLANE_ACCENTS << 8); 631 rv = sphoAddPhoneme(spho,pos,fstSymbol); 632 } 633 if (PICO_OK == rv) { 634 /* invert sympos portion */ 635 i = curPos; 636 j=spho->phonWritePos-1; 637 while (i < j) { 638 tmpPosSym.pos = spho->phonBuf[i].pos; 639 tmpPosSym.sym = spho->phonBuf[i].sym; 640 spho->phonBuf[i].pos = spho->phonBuf[j].pos; 641 spho->phonBuf[i].sym = spho->phonBuf[j].sym; 642 spho->phonBuf[j].pos = tmpPosSym.pos; 643 spho->phonBuf[j].sym = tmpPosSym.sym; 644 i++; 645 j--; 646 } 647 } 648 } else { /* convertAccents */ 649 for (i = 0; i <head->len; i++) { 650 fstSymbol = (PICOKFST_PLANE_PHONEMES << 8) + content[i]; 651 rv = sphoAddPhoneme(spho,pos,fstSymbol); 652 } 653 } 654 break; 655 default: 656 picoos_emRaiseException(this->common->em,rv,NULL,NULL); 657 break; 658 } /* switch(head->type) */ 659 if (PICO_OK != rv) { 660 spho->phonWritePos = oldPos; 661 } 662 return rv; 663 } 664 665 666 667 668 669 #define SPHO_POSSYM_OK 0 670 #define SPHO_POSSYM_OUT_OF_RANGE 1 671 #define SPHO_POSSYM_END 2 672 #define SPHO_POSSYM_INVALID -3 673 /* *readPos is the next position in phonBuf to be read, and *writePos is the first position not to be read (may be outside 674 * buf). 675 * 'rangeEnd' is the first possym position outside the desired range. 676 * Possible return values: 677 * SPHO_POSSYM_OK : 'pos' and 'sym' are set to the read possym, *readPos is advanced 678 * SPHO_POSSYM_OUT_OF_RANGE : pos is out of range. 'pos' is set to that of the read possym, 'sym' is undefined 679 * SPHO_POSSYM_UNDERFLOW : no more data in buf. 'pos' is set to PICOTRNS_POS_INVALID, 'sym' is undefined 680 * SPHO_POSSYM_INVALID : "strange" pos. 'pos' is set to PICOTRNS_POS_INVALID, 'sym' is undefined 681 */ 682 static pico_status_t getNextPosSym(spho_subobj_t * spho, picoos_int16 * pos, picoos_int16 * sym, 683 picoos_int16 rangeEnd) { 684 /* skip POS_IGNORE */ 685 while ((spho->phonReadPos < spho->phonWritePos) && (PICOTRNS_POS_IGNORE == spho->phonBuf[spho->phonReadPos].pos)) { 686 PICODBG_DEBUG(("ignoring phone at spho->phonBuf[%i] because it has pos==IGNORE",spho->phonReadPos)); 687 spho->phonReadPos++; 688 } 689 if ((spho->phonReadPos < spho->phonWritePos)) { 690 *pos = spho->phonBuf[spho->phonReadPos].pos; 691 if ((PICOTRNS_POS_INSERT == *pos) || ((0 <= *pos) && (*pos < rangeEnd))) { 692 *sym = spho->phonBuf[spho->phonReadPos++].sym; 693 return SPHO_POSSYM_OK; 694 } else if (*pos < 0){ /* *pos is "strange" (e.g. POS_INVALID) */ 695 return SPHO_POSSYM_INVALID; 696 } else { 697 return SPHO_POSSYM_OUT_OF_RANGE; 698 } 699 } else { 700 /* no more possyms to read */ 701 *pos = PICOTRNS_POS_INVALID; 702 return SPHO_POSSYM_END; 703 } 704 } 705 706 707 708 /** Calculate bound strength modified by transduction 709 * 710 * Given the original bound strength 'orig' and the desired target strength 'target' (suggested by fst), 711 * calculate the modified bound strength. 712 * 713 * @param orig original bound strength 714 * @param target target bound strength 715 * @return resulting bound strength 716 */ 717 static picoos_uint8 fstModifiedBoundStrength(picoos_uint8 orig, picoos_uint8 target) 718 { 719 switch (orig) { 720 case PICODATA_ITEMINFO1_BOUND_PHR1: 721 case PICODATA_ITEMINFO1_BOUND_PHR2: 722 /* don't allow primary phrase bounds to be demoted to word bound */ 723 if (PICODATA_ITEMINFO1_BOUND_PHR0 == target) { 724 return PICODATA_ITEMINFO1_BOUND_PHR3; 725 } 726 case PICODATA_ITEMINFO1_BOUND_PHR0: 727 case PICODATA_ITEMINFO1_BOUND_PHR3: 728 return target; 729 break; 730 default: 731 /* don't allow bounds other than phrase or word bounds to be changed */ 732 return orig; 733 break; 734 } 735 } 736 737 /** Calculate bound strength modified by a \<break> command 738 * 739 * Given the original (predicted and possibly fst-modified) bound strength, and a time value from an 740 * overwriding \<break> command, calculate the modified bound strength. 741 * 742 * @param orig original bound strength 743 * @param time time given as property of \<break> command 744 * @param wasPrimary 745 * @return modified bound strength 746 */ 747 static picoos_uint8 breakModifiedBoundStrength(picoos_uint8 orig, picoos_uint16 time, picoos_bool wasPrimary) 748 { 749 picoos_uint8 modified = (0 == time) ? PICODATA_ITEMINFO1_BOUND_PHR3 : 750 (50 < time) ? PICODATA_ITEMINFO1_BOUND_PHR1 : PICODATA_ITEMINFO1_BOUND_PHR2; 751 switch (orig) { 752 /* for word and phrase breaks, return 'modified', unless a non-silence gets time==0, in which 753 * case return no break (word break) */ 754 case PICODATA_ITEMINFO1_BOUND_PHR0: 755 if (0 == time) { 756 return PICODATA_ITEMINFO1_BOUND_PHR0; 757 } 758 case PICODATA_ITEMINFO1_BOUND_PHR3: 759 if (!wasPrimary && (0 == time)) { 760 return PICODATA_ITEMINFO1_BOUND_PHR0; 761 } 762 case PICODATA_ITEMINFO1_BOUND_PHR1: 763 case PICODATA_ITEMINFO1_BOUND_PHR2: 764 return modified; 765 break; 766 default: 767 return orig; 768 break; 769 } 770 } 771 772 static picoos_bool breakStateInterrupting(picodata_itemhead_t * head, 773 picoos_bool * breakBefore, picoos_bool * breakAfter) { 774 775 picoos_bool result = 1; 776 777 *breakBefore = 0; 778 *breakAfter = 0; 779 780 if (PICODATA_ITEM_WORDPHON == head->type) { 781 782 } else if (PICODATA_ITEM_CMD == head->type) { 783 if ((PICODATA_ITEMINFO1_CMD_PLAY == head->info1) 784 || (PICODATA_ITEMINFO1_CMD_SAVE == head->info1) 785 || (PICODATA_ITEMINFO1_CMD_UNSAVE == head->info1)) { 786 *breakBefore = 1; 787 *breakAfter = 1; 788 } else if (PICODATA_ITEMINFO1_CMD_SAVE == head->info1) { 789 *breakBefore = 1; 790 } else if (PICODATA_ITEMINFO1_CMD_UNSAVE == head->info1) { 791 *breakAfter = 1; 792 } else if (PICODATA_ITEMINFO1_CMD_IGNSIG == head->info1) { 793 if (PICODATA_ITEMINFO2_CMD_START == head->info2) { 794 *breakBefore = 1; 795 } else { 796 *breakAfter = 1; 797 } 798 } 799 } else { 800 result = 0; 801 } 802 return result; 803 } 804 805 806 static void putSideBoundToOutput(spho_subobj_t * spho) 807 { 808 809 picodata_itemhead_t ohead; 810 picoos_uint8 ocontent[2*sizeof(picoos_uint16)]; 811 picoos_int16 sildur; 812 picoos_uint16 clen; 813 814 /* create boundary */ 815 ohead.type = PICODATA_ITEM_BOUND; 816 ohead.info1 = spho->headx[spho->outReadPos].boundstrength; 817 ohead.info2 = spho->headx[spho->outReadPos].phrasetype; 818 sildur = spho->headx[spho->outReadPos].sildur; 819 if ((sildur < 0) 820 || (PICODATA_ITEMINFO1_BOUND_PHR0 == ohead.info1) 821 || (PICODATA_ITEMINFO1_BOUND_PHR3 == ohead.info1)) { 822 PICODBG_DEBUG(("outputting a bound of strength '%c' and type '%c' without duration constraints",ohead.info1, ohead.info2)); 823 ohead.len = 0; 824 } else { 825 picoos_uint32 pos = 0; 826 picoos_write_mem_pi_uint16(ocontent,&pos,sildur); 827 picoos_write_mem_pi_uint16(ocontent,&pos,sildur); 828 PICODBG_DEBUG(("outputting a bound of strength '%c' and type '%c' with duration constraints [%i,%i]",ohead.info1, ohead.info2,sildur, sildur)); 829 ohead.len = pos; 830 } 831 picodata_put_itemparts(&ohead, ocontent, ohead.len, 832 spho->outBuf, spho->outBufSize, &clen); 833 /* disable side bound */ 834 spho->headx[spho->outReadPos].boundstrength = 0; 835 } 836 837 /** Set bound strength and sil dur. 838 * 839 * given the original bound strength 'orig_strength' and the fst-suggested bound strength 'fst_strength' 840 * and possibly being in a pending break state, calculate the resulting bound strength and set boundstrength 841 * and sildur of the current item (spho->headx[spho->outReadPos]) accordingly. 842 * if a boundstrength was set, also calculate the phrasetype and if necessary (and reachable), modify the phrase type 843 * of the previous phrase boundary. 844 * 845 * @param spho 846 * @param orig_strength 847 * @param orig_type 848 * @param fst_strength 849 */ 850 static void setSideBound(spho_subobj_t * spho, picoos_uint8 orig_strength, picoos_uint8 orig_type, picoos_uint8 fst_strength) { 851 picoos_uint8 strength; 852 853 /* insert modified bound according to transduction symbol, if any */ 854 if (PICODATA_ITEMINFO1_NA == orig_strength) { 855 /* no original/fst strength given */ 856 orig_strength = PICODATA_ITEMINFO1_BOUND_PHR0; 857 strength = PICODATA_ITEMINFO1_BOUND_PHR0; 858 } else { 859 strength = fstModifiedBoundStrength(orig_strength,fst_strength); 860 spho->headx[spho->outReadPos].boundstrength = strength; 861 spho->headx[spho->outReadPos].sildur = -1; 862 PICODBG_DEBUG(("setting bound strength to fst-suggested value %c (was %c)",strength, spho->headx[spho->outReadPos].boundstrength, spho->breakTime)); 863 } 864 865 /* insert modified bound according to pending break, if any */ 866 if (spho->breakPending) { 867 /* the calculation is based on the fst-modified value (because this is what the customer wants to 868 * override) 869 */ 870 strength = breakModifiedBoundStrength(strength, spho->breakTime, (PICODATA_ITEMINFO1_BOUND_PHR1 == orig_strength)); 871 PICODBG_DEBUG(("setting bound strength to break-imposed value %c (was %c) and time to %i",strength, spho->headx[spho->outReadPos].boundstrength, spho->breakTime)); 872 spho->headx[spho->outReadPos].boundstrength = strength; 873 spho->headx[spho->outReadPos].sildur = spho->breakTime; 874 spho->breakPending = FALSE; 875 } 876 if (spho->headx[spho->outReadPos].boundstrength) { 877 /* we did set a bound strength, possibly promoting or demoting a boundary; now set the phrase type 878 * possibly also changing the phrase type of the previous phrase bound 879 */ 880 picoos_uint8 fromPhrase = ((PICODATA_ITEMINFO1_BOUND_PHR0 != orig_strength)); 881 picoos_uint8 toPhrase = ((PICODATA_ITEMINFO1_BOUND_PHR0 != strength)); 882 883 PICODBG_DEBUG(("setting phrase type (wasPhrase=%i, isPhrase=%i)",fromPhrase,toPhrase)); 884 if (toPhrase) { 885 if (fromPhrase) { 886 spho->lastPhraseType = orig_type; 887 } else { /*promote */ 888 if (spho->activeStartPos <= spho->lastPhraseBoundPos) { 889 /* we still can change prev phrase bound */ 890 /* since a new phrase boundary is introduced, we have to 'invent' 891 * an additional phrase type here. For that, we have to use some of the 892 * knowledge that otherwise is handled in picoacph. 893 */ 894 spho->headx[spho->lastPhraseBoundPos].phrasetype 895 = PICODATA_ITEMINFO2_BOUNDTYPE_P; 896 } 897 } 898 spho->lastPhraseBoundPos = spho->outReadPos; 899 spho->headx[spho->lastPhraseBoundPos].phrasetype 900 = spho->lastPhraseType; 901 902 } else { 903 spho->headx[spho->outReadPos].phrasetype = PICODATA_ITEMINFO2_NA; 904 if (fromPhrase) { /* demote */ 905 spho->lastPhraseType = orig_type; 906 if (spho->activeStartPos <= spho->lastPhraseBoundPos) { 907 /* we still can change prev phrase bound */ 908 spho->headx[spho->lastPhraseBoundPos].phrasetype 909 = spho->lastPhraseType; 910 } 911 } 912 } 913 } 914 } 915 916 917 /* ***********************************************************************/ 918 /* sphoStep function */ 919 /* ***********************************************************************/ 920 921 922 static picodata_step_result_t sphoStep(register picodata_ProcessingUnit this, 923 picoos_int16 mode, picoos_uint16 * numBytesOutput) 924 { 925 926 register spho_subobj_t *spho; 927 pico_status_t rv= PICO_OK; 928 picoos_uint16 blen; 929 picodata_itemhead_t ihead, ohead; 930 picoos_uint8 *icontent; 931 picoos_uint16 nextInPos; 932 #if defined(PICO_DEBUG) 933 picoos_char msgstr[SPHO_MSGSTR_SIZE]; 934 #endif 935 936 /* used in FEED and FEED_SYM */ 937 picoos_uint16 clen; 938 picoos_int16 pos, sym, sylsym; 939 picoos_uint8 plane; 940 941 /* used in BOUNDS */ 942 picoos_bool breakBefore, breakAfter; 943 944 /* pico_status_t rvP= PICO_OK; */ 945 946 picoos_uint16 curPos /*, nextPos */; 947 picoos_uint16 remHeadxSize, remCbufSize; 948 949 950 if (NULL == this || NULL == this->subObj) { 951 return PICODATA_PU_ERROR; 952 } 953 spho = (spho_subobj_t *) this->subObj; 954 955 mode = mode; /* avoid warning "var not used in this function"*/ 956 957 *numBytesOutput = 0; 958 while (1) { /* exit via return */ 959 PICODBG_INFO(("doing state %i, headxReadPos: %d, headxWritePos: %d", 960 spho->procState, spho->headxReadPos, spho->headxWritePos)); 961 962 switch (spho->procState) { 963 964 case SPHO_STEPSTATE_INIT: 965 /* **********************************************************************/ 966 /* INIT */ 967 /* **********************************************************************/ 968 PICODBG_DEBUG(("INIT")); 969 /* (re)set values for PARSE */ 970 spho->penultima = SPHO_POS_INVALID; 971 spho->activeEndPos = SPHO_POS_INVALID; 972 spho->headxReadPos = 0; 973 spho->phonReadPos = 0; 974 spho->phonWritePos = 0; 975 spho->lastPhraseType = PICODATA_ITEMINFO2_NA; 976 spho->lastPhraseBoundPos = -1; 977 978 spho->procState = SPHO_STEPSTATE_COLLECT; 979 break; 980 981 982 case SPHO_STEPSTATE_COLLECT: 983 /* **********************************************************************/ 984 /* COLLECT */ 985 /* **********************************************************************/ 986 /* collect state: get items from charBuf and store in 987 * internal inBuf 988 */ 989 PICODBG_TRACE(("COLLECT")); 990 rv = PICO_OK; 991 remHeadxSize = spho->headxBufSize - spho->headxWritePos; 992 remCbufSize = spho->cbufBufSize - spho->cbufWritePos; 993 curPos = spho->headxWritePos; 994 while ((PICO_OK == rv) && (remHeadxSize > 0) && (remCbufSize > 0)) { 995 PICODBG_DEBUG(("COLLECT getting item at headxWritePos %i (remaining %i)",spho->headxWritePos, remHeadxSize)); 996 rv = picodata_cbGetItem(this->cbIn, spho->tmpbuf, PICODATA_MAX_ITEMSIZE, &blen); 997 if (PICO_OK == rv) { 998 rv = picodata_get_itemparts(spho->tmpbuf, 999 PICODATA_MAX_ITEMSIZE, &(spho->headx[spho->headxWritePos].head), 1000 &(spho->cbuf[spho->cbufWritePos]), remCbufSize, &blen); 1001 if (PICO_OK == rv) { 1002 spho->headx[spho->headxWritePos].cind = spho->cbufWritePos; 1003 spho->headx[spho->headxWritePos].boundstrength = 0; 1004 spho->headxWritePos++; 1005 remHeadxSize--; 1006 spho->cbufWritePos += blen; 1007 remCbufSize -= blen; 1008 } 1009 } 1010 } 1011 if ((PICO_OK == rv) && ((remHeadxSize <= 0) || (remCbufSize <= 0))) { 1012 rv = PICO_EXC_BUF_OVERFLOW; 1013 } 1014 1015 /* in normal circumstances, rv is either PICO_EOF (no more items in cbIn) or PICO_BUF_OVERFLOW 1016 * (if no more items fit into headx) */ 1017 if ((PICO_EOF != rv) && (PICO_EXC_BUF_OVERFLOW != rv)) { 1018 PICODBG_DEBUG(("COLLECT ** problem getting item, unhandled, rv: %i", rv)); 1019 picoos_emRaiseException(this->common->em, rv, 1020 NULL, NULL); 1021 return PICODATA_PU_ERROR; 1022 } 1023 if (PICO_EOF == rv) { /* there are no more items available */ 1024 if (curPos < spho->headxWritePos) { /* we did get some new items */ 1025 PICODBG_DEBUG(("COLLECT read %i items", 1026 spho->headxWritePos - curPos)); 1027 spho->needMoreInput = FALSE; 1028 } 1029 if (spho->needMoreInput) { /* not enough items to proceed */ 1030 PICODBG_DEBUG(("COLLECT need more data, returning IDLE")); 1031 return PICODATA_PU_IDLE; 1032 } else { 1033 spho->procState = SPHO_STEPSTATE_PROCESS_PARSE; 1034 /* uncomment next to split into two steps */ 1035 /* return PICODATA_PU_ATOMIC; */ 1036 } 1037 } else { /* input buffer full */ 1038 PICODBG_DEBUG(("COLLECT input buffer full")); 1039 if (spho->needMoreInput) { /* forced output because we can't get more data */ 1040 spho->needMoreInput = FALSE; 1041 spho->force = TRUE; 1042 } 1043 spho->procState = SPHO_STEPSTATE_PROCESS_PARSE; 1044 } 1045 break; 1046 1047 case SPHO_STEPSTATE_PROCESS_PARSE: 1048 1049 /* **********************************************************************/ 1050 /* PARSE: items -> input pos/phon pairs */ 1051 /* **********************************************************************/ 1052 1053 /* parse one item at a time */ 1054 /* If 1055 * - the item is a sentence end or 1056 * - it is the last item and force=1 or 1057 * - the phon buffer is full 1058 * then set inReadPos to 0 and go to TRANSDUCE 1059 * else advance by one item */ 1060 1061 /* look at the current item */ 1062 PICODBG_TRACE(("PARSE")); 1063 if (spho->headxReadPos >= spho->headxWritePos) { 1064 /* no more items in headx */ 1065 if (spho->force) { 1066 PICODBG_INFO(("no more items in headx but we are forced to transduce")); 1067 1068 /* headx is full; we are forced to transduce before reaching the sentence end */ 1069 spho->force = FALSE; 1070 if (SPHO_POS_INVALID == spho->activeEndPos) { 1071 spho->activeEndPos = spho->headxReadPos; 1072 } 1073 spho->procState = SPHO_STEPSTATE_PROCESS_TRANSDUCE; 1074 } else { 1075 /* we try to get more data */ 1076 PICODBG_INFO(("no more items in headx, try to collect more")); 1077 spho->needMoreInput = TRUE; 1078 spho->procState = SPHO_STEPSTATE_COLLECT; 1079 } 1080 break; 1081 } 1082 1083 ihead = spho->headx[spho->headxReadPos].head; 1084 icontent = spho->cbuf + spho->headx[spho->headxReadPos].cind; 1085 1086 PICODBG_DEBUG(("PARSE looking at item %s",picodata_head_to_string(&ihead,msgstr,SPHO_MSGSTR_SIZE))); 1087 /* treat header */ 1088 if (PICODATA_ITEM_BOUND == ihead.type) { 1089 /* see if it is a sentence end or termination boundary (flush) */ 1090 if ((PICODATA_ITEMINFO1_BOUND_SEND == ihead.info1) 1091 || (PICODATA_ITEMINFO1_BOUND_TERM == ihead.info1)) { 1092 PICODBG_INFO(("PARSE found sentence end or term BOUND")); 1093 1094 if (spho->sentenceStarted) { 1095 /* its the end of the sentence */ 1096 PICODBG_INFO(("PARSE found sentence end")); 1097 spho->sentenceStarted = 0; 1098 /* there is no need for a right context; move the active end to the end */ 1099 /* add sentence termination phonemes */ 1100 sphoAddTermPhonemes(spho, spho->headxReadPos); 1101 spho->headxReadPos++; 1102 spho->activeEndPos = spho->headxReadPos; 1103 /* we may discard all information up to activeEndPos, after processing of last 1104 * sentence part 1105 */ 1106 spho->penultima = spho->activeEndPos; 1107 1108 /* transduce */ 1109 spho->procState = SPHO_STEPSTATE_PROCESS_TRANSDUCE; 1110 /* uncomment to split */ 1111 /* return PICODATA_PU_BUSY; */ 1112 break; 1113 } else { 1114 if (PICODATA_ITEMINFO1_BOUND_TERM == ihead.info1) { 1115 /* its the end of input (flush) */ 1116 PICODBG_INFO(("PARSE forwarding input end (flush)")); 1117 /* copy item unmodified */ 1118 picodata_put_itemparts(&ihead, 1119 icontent, 1120 ihead.len, 1121 spho->outBuf, spho->outBufSize, 1122 &clen); 1123 1124 spho->headxReadPos++; 1125 spho->activeEndPos = spho->headxReadPos; 1126 spho->penultima = SPHO_POS_INVALID; 1127 spho->feedFollowState = SPHO_STEPSTATE_SHIFT; 1128 spho->procState = SPHO_STEPSTATE_FEED; 1129 break; 1130 } else { 1131 /* this should never happen */ 1132 /* eliminate bound */ 1133 spho->headxReadPos++; 1134 spho->activeEndPos = spho->headxReadPos; 1135 spho->penultima = SPHO_POS_INVALID; 1136 PICODBG_ERROR(("PARSE found a sentence end without a sentence start; eliminated")); 1137 } 1138 } 1139 } else if (PICODATA_ITEMINFO1_BOUND_SBEG == ihead.info1) { 1140 /* its the start of the sentence */ 1141 PICODBG_INFO(("PARSE found sentence start")); 1142 /* add sentence starting phoneme */ 1143 sphoAddStartPhoneme(spho); 1144 1145 spho->sentenceStarted = 1; 1146 } 1147 } 1148 1149 if ((PICODATA_ITEM_WORDPHON == ihead.type) 1150 || (PICODATA_ITEM_BOUND == ihead.type)) { 1151 /* if it is a word or a bound try to extract phonemes */ 1152 PICODBG_INFO(("PARSE found WORD phon or phrase BOUND")); 1153 rv = sphoExtractPhonemes(this, spho, spho->headxReadPos, 1154 TRUE /* convertAccents */, 1155 &spho->suppressParseWordBound); 1156 if (PICO_OK == rv) { 1157 PICODBG_INFO(("PARSE successfully returned from phoneme extraction")); 1158 /* replace activeEndPos if the new item is a word, or activeEndPos was not set yet, or 1159 * activeEndPos was a bound */ 1160 if ((spho->activeStartPos <= spho->headxReadPos) && ((PICODATA_ITEM_WORDPHON == ihead.type) 1161 || (SPHO_POS_INVALID == spho->activeEndPos) 1162 || (PICODATA_ITEM_BOUND == spho->headx[spho->activeEndPos].head.type))) { 1163 PICODBG_INFO(("PARSE found new activeEndPos: %i,%i -> %i,%i", 1164 spho->penultima,spho->activeEndPos,spho->activeEndPos,spho->headxReadPos)); 1165 spho->penultima = spho->activeEndPos; 1166 spho->activeEndPos = spho->headxReadPos; 1167 } 1168 1169 } else if (PICO_EXC_BUF_OVERFLOW == rv) { 1170 /* phoneme buffer cannot take this item anymore; 1171 if the phoneme buffer has some contents, we are forced to transduce before reaching the sentence end 1172 else we skip the (too long word) */ 1173 PICODBG_INFO(("PARSE returned from phoneme extraction with overflow, number of phonemes in phonBuf: %i; forced to TRANSDUCE", spho->phonWritePos)); 1174 if ((SPHO_POS_INVALID == spho->activeEndPos) || (spho->activeStartPos == spho->activeEndPos)) { 1175 spho->activeEndPos = spho->headxReadPos; 1176 } 1177 spho->procState = SPHO_STEPSTATE_PROCESS_TRANSDUCE; 1178 break; 1179 } else { 1180 PICODBG_ERROR(("PARSE returned from phoneme extraction with exception %i",rv)); 1181 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, 1182 PICO_ERR_OTHER, NULL, NULL); 1183 } 1184 } else { 1185 PICODBG_INFO(("PARSE found other item, passing over")); 1186 /* it is "other" item, ignore */ 1187 } 1188 /* set pos at next item */ 1189 PICODBG_INFO(("PARSE going to next item: %i -> %i",spho->headxReadPos, spho->headxReadPos + 1)); 1190 spho->headxReadPos++; 1191 break; 1192 1193 case SPHO_STEPSTATE_PROCESS_TRANSDUCE: 1194 1195 /* **********************************************************************/ 1196 /* TRANSDUCE: transduction input pos/phon pairs to output pos/phon pairs */ 1197 /* **********************************************************************/ 1198 PICODBG_DEBUG(("TRANSDUCE (%i-th of %i fsts",spho->curFst+1, spho->numFsts)); 1199 1200 /* termination condition first */ 1201 if (spho->curFst >= spho->numFsts) { 1202 1203 #if defined(PICO_DEBUG) 1204 { 1205 PICODBG_INFO_CTX(); 1206 PICODBG_INFO_MSG(("result of all transductions: ")); 1207 PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], spho->phonBufOut, spho->phonWritePos); 1208 PICODBG_INFO_MSG(("\n")); 1209 } 1210 #endif 1211 1212 /* reset for next transduction */ 1213 spho->curFst = 0; 1214 /* prepare BOUNDS */ 1215 spho->outReadPos = 0; 1216 spho->phonReadPos = 0; 1217 1218 spho->procState = SPHO_STEPSTATE_PROCESS_BOUNDS; 1219 break; 1220 } 1221 1222 /* transduce from phonBufIn to PhonBufOut */ 1223 { 1224 1225 picoos_uint32 nrSteps; 1226 #if defined(PICO_DEBUG) 1227 { 1228 PICODBG_INFO_CTX(); 1229 PICODBG_INFO_MSG(("spho trying to transduce: ")); 1230 PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], spho->phonBuf, spho->phonWritePos); 1231 PICODBG_INFO_MSG(("\n")); 1232 } 1233 #endif 1234 rv = picotrns_transduce(spho->fst[spho->curFst], FALSE, 1235 picotrns_printSolution, spho->phonBuf, spho->phonWritePos, spho->phonBufOut, 1236 &spho->phonWritePos, 1237 4*PICOTRNS_MAX_NUM_POSSYM, spho->altDescBuf, 1238 spho->maxAltDescLen, &nrSteps); 1239 if (PICO_OK == rv) { 1240 #if defined(PICO_DEBUG) 1241 { 1242 PICODBG_INFO_CTX(); 1243 PICODBG_INFO_MSG(("result of transduction: (output symbols: %i)", spho->phonWritePos)); 1244 PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], spho->phonBufOut, spho->phonWritePos); 1245 PICODBG_INFO_MSG(("\n")); 1246 } 1247 #endif 1248 PICODBG_TRACE(("number of steps done in tranduction: %i", nrSteps)); 1249 } else { 1250 picoos_emRaiseWarning(this->common->em, PICO_WARN_FALLBACK,NULL,(picoos_char *)"phon buffer full"); 1251 } 1252 } 1253 /* eliminate deep epsilons */ 1254 picotrns_eliminate_epsilons(spho->phonBufOut, spho->phonWritePos, spho->phonBuf, 1255 &spho->phonWritePos,4*PICOTRNS_MAX_NUM_POSSYM); 1256 1257 spho->curFst++; 1258 1259 /* return PICODATA_PU_ATOMIC */ 1260 break; 1261 1262 1263 case SPHO_STEPSTATE_PROCESS_BOUNDS: 1264 /* ************************************************************************/ 1265 /* BOUNDS: combine input item with pos/phon pairs to insert/modify bounds */ 1266 /* ************************************************************************/ 1267 1268 PICODBG_INFO(("BOUNDS")); 1269 1270 /* get the suppressRecombWordBound in the left context */ 1271 spho->suppressRecombWordBound = FALSE; 1272 while (spho->outReadPos < spho->activeStartPos) { 1273 /* look at the current item */ 1274 ihead = spho->headx[spho->outReadPos].head; 1275 /* icontent = spho->cbuf + spho->headx[spho->outReadPos].cind; */ 1276 PICODBG_INFO(("in position %i, looking at item %s",spho->outReadPos,picodata_head_to_string(&ihead,msgstr,SPHO_MSGSTR_SIZE))); 1277 if (PICODATA_ITEM_BOUND == ihead.type) { 1278 spho->suppressRecombWordBound = TRUE; 1279 } else if (PICODATA_ITEM_WORDPHON == ihead.type) { 1280 spho->suppressRecombWordBound = FALSE; 1281 } 1282 spho->outReadPos++; 1283 } 1284 /* spho->outReadPos point now to the active region */ 1285 1286 /* advance the phone reading pos to the active range */ 1287 spho->phonReadPos = 0; 1288 while (SPHO_POSSYM_OK == (rv = getNextPosSym(spho, &pos, &sym, 1289 spho->activeStartPos))) { 1290 /* ignore */ 1291 } 1292 PICODBG_INFO(("skipping left context phones results in %s", (SPHO_POSSYM_OUT_OF_RANGE==rv) ? "OUT_OF_RANGE" : (SPHO_POSSYM_END ==rv) ? "END" : "OTHER")); 1293 1294 /* 1295 * Align input items with transduced phones and note bound stregth changes and break commands 1296 */ 1297 1298 while (spho->outReadPos < spho->activeEndPos) { 1299 1300 /* look at the current item */ 1301 ihead = spho->headx[spho->outReadPos].head; 1302 icontent = spho->cbuf + spho->headx[spho->outReadPos].cind; 1303 nextInPos = spho->outReadPos + 1; 1304 /* */ 1305 PICODBG_INFO(("in position %i, looking at item %s",spho->outReadPos,picodata_head_to_string(&ihead,msgstr,SPHO_MSGSTR_SIZE))); 1306 1307 if ((PICODATA_ITEM_BOUND == ihead.type) 1308 || ((PICODATA_ITEM_WORDPHON == ihead.type) 1309 && (!spho->suppressRecombWordBound))) { 1310 /* there was a boundary originally */ 1311 picoos_uint8 orig_strength, orig_type; 1312 if (PICODATA_ITEM_BOUND == ihead.type) { 1313 orig_strength = ihead.info1; 1314 orig_type = ihead.info2; 1315 spho->suppressRecombWordBound = TRUE; 1316 } else { 1317 orig_strength = PICODATA_ITEMINFO1_BOUND_PHR0; 1318 orig_type = PICODATA_ITEMINFO2_NA; 1319 } 1320 /* i expect a boundary phone here */ 1321 /* consume FST bound phones, consider pending break and set the side-bound */ 1322 PICODBG_INFO(("got BOUND or WORDPHON item and expects corresponding phone")); 1323 rv = getNextPosSym(spho, &pos, &sym, nextInPos); 1324 if (SPHO_POSSYM_OK != rv) { 1325 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list (%s)", (SPHO_POSSYM_OUT_OF_RANGE==rv) ? "OUT_OF_RANGE" : (SPHO_POSSYM_END ==rv) ? "END" :"OTHER")); 1326 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, 1327 PICO_ERR_OTHER, NULL, NULL); 1328 } 1329 sym = picotrns_unplane(sym, &plane); 1330 /* */ 1331 PICODBG_ASSERT((PICOKFST_PLANE_PB_STRENGTHS == plane)); 1332 1333 /* insert modified bound according to transduction and possibly pending break */ 1334 setSideBound(spho, orig_strength, orig_type, 1335 (picoos_uint8) sym); 1336 } else if ((PICODATA_ITEM_CMD == ihead.type) 1337 && (PICODATA_ITEMINFO1_CMD_SIL == ihead.info1)) { 1338 /* it's a SIL (break) command */ 1339 picoos_uint16 time; 1340 picoos_uint32 pos = 0; 1341 picoos_read_mem_pi_uint16(icontent, &pos, &time); 1342 if (spho->breakPending) { 1343 spho->breakTime += time; 1344 } else { 1345 spho->breakTime = time; 1346 spho->breakPending = TRUE; 1347 } 1348 } else if ((PICODATA_ITEM_CMD == ihead.type) && (PICODATA_ITEMINFO1_CMD_PLAY == ihead.info1)) { 1349 /* insert break of at least one ms */ 1350 if (!spho->breakPending || (spho->breakTime <= 0)) { 1351 spho->breakTime = SPHO_SMALLEST_SIL_DUR; 1352 spho->breakPending = TRUE; 1353 } 1354 setSideBound(spho, PICODATA_ITEMINFO1_NA, 1355 PICODATA_ITEMINFO2_NA, PICODATA_ITEMINFO1_NA); 1356 /* force following break to be at least one ms */ 1357 spho->breakTime = SPHO_SMALLEST_SIL_DUR; 1358 spho->breakPending = TRUE; 1359 } else if (breakStateInterrupting(&ihead, &breakBefore, &breakAfter)) { 1360 1361 if (breakBefore &&(!spho->breakPending || (spho->breakTime <= 0))) { 1362 spho->breakTime = SPHO_SMALLEST_SIL_DUR; 1363 spho->breakPending = TRUE; 1364 } 1365 setSideBound(spho, PICODATA_ITEMINFO1_NA, 1366 PICODATA_ITEMINFO2_NA, PICODATA_ITEMINFO1_NA); 1367 1368 if (breakAfter) { 1369 spho->breakTime = SPHO_SMALLEST_SIL_DUR; 1370 spho->breakPending = TRUE; 1371 } 1372 if (PICODATA_ITEM_WORDPHON == ihead.type) { 1373 spho->suppressRecombWordBound = FALSE; 1374 } 1375 } 1376 1377 /* skip phones of that item */ 1378 while (SPHO_POSSYM_OK == (rv = getNextPosSym(spho, &pos, 1379 &sym, nextInPos))) { 1380 /* ignore */ 1381 } 1382 spho->outReadPos++; 1383 } 1384 1385 /* reset for RECOMB */ 1386 spho->outReadPos = 0; 1387 spho->phonReadPos = 0; 1388 spho->suppressRecombWordBound = FALSE; 1389 1390 spho->procState = SPHO_STEPSTATE_PROCESS_RECOMB; 1391 return PICODATA_PU_ATOMIC; 1392 1393 break; 1394 1395 case SPHO_STEPSTATE_PROCESS_RECOMB: 1396 /* **********************************************************************/ 1397 /* RECOMB: combine input item with pos/phon pairs to output item */ 1398 /* **********************************************************************/ 1399 1400 PICODBG_TRACE(("RECOMB")); 1401 1402 /* default place to come after feed: here */ 1403 spho->feedFollowState = SPHO_STEPSTATE_PROCESS_RECOMB; 1404 1405 /* check termination condition first */ 1406 if (spho->outReadPos >= spho->activeEndPos) { 1407 PICODBG_DEBUG(("RECOMB reached active region's end at %i",spho->outReadPos)); 1408 spho->procState = SPHO_STEPSTATE_SHIFT; 1409 break; 1410 } 1411 1412 /* look at the current item */ 1413 ihead = spho->headx[spho->outReadPos].head; 1414 icontent = spho->cbuf + spho->headx[spho->outReadPos].cind; 1415 1416 PICODBG_DEBUG(("RECOMB looking at item %s",picodata_head_to_string(&ihead,msgstr,SPHO_MSGSTR_SIZE))); 1417 1418 nextInPos = spho->outReadPos + 1; 1419 1420 PICODBG_DEBUG(("RECOMB treating item in headx at pos %i",spho->outReadPos)); 1421 if (nextInPos <= spho->activeStartPos) { /* we're in the (passive) left context. Just skip it */ 1422 PICODBG_DEBUG(("RECOMB skipping item in the left context (%i <= %i)",nextInPos, spho->activeStartPos)); 1423 if (PICODATA_ITEM_BOUND == ihead.type) { 1424 spho->suppressRecombWordBound = 1; 1425 } else if (PICODATA_ITEM_WORDPHON == ihead.type) { 1426 spho->suppressRecombWordBound = 0; 1427 } 1428 1429 /* consume possyms */ 1430 while (SPHO_POSSYM_OK == (rv = getNextPosSym(spho,&pos,&sym,nextInPos))) { 1431 /* ignore */ 1432 } 1433 if (rv == SPHO_POSSYM_INVALID) { 1434 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, 1435 PICO_ERR_OTHER, NULL, NULL); 1436 } 1437 spho->outReadPos = nextInPos; 1438 } else { /* active region */ 1439 if (spho->headx[spho->outReadPos].boundstrength) { 1440 /* ***************** "side-bound" *********************/ 1441 /* copy to outbuf */ 1442 putSideBoundToOutput(spho); 1443 /* mark as processed */ 1444 spho->headx[spho->outReadPos].boundstrength = 0; 1445 /* output it */ 1446 spho->procState = SPHO_STEPSTATE_FEED; 1447 } else if (PICODATA_ITEM_BOUND == ihead.type) { 1448 /* ***************** BOUND *********************/ 1449 /* expect a boundary phone here */ 1450 PICODBG_DEBUG(("RECOMB got BOUND item and expects corresponding phone")); 1451 rv = getNextPosSym(spho, &pos, &sym, nextInPos); 1452 if (SPHO_POSSYM_OK != rv) { 1453 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list")); 1454 return (picodata_step_result_t)picoos_emRaiseException( 1455 this->common->em, PICO_ERR_OTHER, NULL, 1456 NULL); 1457 } 1458 sym = picotrns_unplane(sym, &plane); 1459 /* */ 1460 PICODBG_ASSERT((PICOKFST_PLANE_PB_STRENGTHS == plane)); 1461 1462 spho->suppressRecombWordBound = TRUE; /* if word following, don't need word boundary */ 1463 /* just consume item and come back here*/ 1464 spho->outReadPos = nextInPos; 1465 1466 } else if (PICODATA_ITEM_WORDPHON == ihead.type) { 1467 /* ***************** WORDPHON *********************/ 1468 spho->wordStarted = TRUE; 1469 /* i expect a word boundary symbol in this range unless a phrase boundary was encountered before */ 1470 if (spho->suppressRecombWordBound) { 1471 PICODBG_DEBUG(("RECOMB got WORDPHON item but skips expecting BOUND")); 1472 spho->suppressRecombWordBound = FALSE; 1473 } else { 1474 PICODBG_DEBUG(("RECOMB got WORDPHON item and expects corresponding bound phone")); 1475 rv = getNextPosSym(spho, &pos, &sym, nextInPos); 1476 if (SPHO_POSSYM_OK != rv) { 1477 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list")); 1478 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, 1479 PICO_ERR_OTHER, NULL, NULL); 1480 } 1481 } 1482 spho->procState = SPHO_STEPSTATE_PROCESS_SYL; 1483 } else if ((PICODATA_ITEM_CMD == ihead.type) && (PICODATA_ITEMINFO1_CMD_SIL == ihead.info1)) { 1484 /* ***************** BREAK COMMAND *********************/ 1485 /* just consume and come back here */ 1486 PICODBG_DEBUG(("RECOMB consuming item from inBuf %i -> %i",spho->outReadPos, nextInPos)); 1487 spho->outReadPos = nextInPos; 1488 } else { 1489 /* ***************** OTHER *********************/ 1490 /* just copy item */ 1491 PICODBG_DEBUG(("RECOMB found other item, just copying")); 1492 picodata_put_itemparts(&ihead, icontent, ihead.len, 1493 spho->outBuf, spho->outBufSize, &clen); 1494 PICODBG_DEBUG(("RECOMB consuming item from inBuf %i -> %i",spho->outReadPos, nextInPos)); 1495 spho->outReadPos = nextInPos; 1496 /* and output it */ 1497 spho->procState = SPHO_STEPSTATE_FEED; 1498 } /* if (ihead.type) */ 1499 1500 } 1501 1502 /* return PICODATA_PU_BUSY; */ 1503 break; 1504 1505 case SPHO_STEPSTATE_PROCESS_SYL: 1506 /* **********************************************************************/ 1507 /* SYL: combine input word item with pos/phon pairs to syl output item */ 1508 /* **********************************************************************/ 1509 1510 /* consume all transduced phonemes with pos in in the range [spho->outReadPos,nextInPos[ */ 1511 PICODBG_DEBUG(("SYL")); 1512 1513 spho->feedFollowState = SPHO_STEPSTATE_PROCESS_SYL; 1514 1515 /* look at the current item */ 1516 ihead = spho->headx[spho->outReadPos].head; 1517 icontent = spho->cbuf + spho->headx[spho->outReadPos].cind; 1518 nextInPos = spho->outReadPos + 1; 1519 PICODBG_DEBUG(("SYL (1) treating item in headx at pos %i",spho->outReadPos)); 1520 /* create syllable item in ohead (head) and sylBuf (contents) */ 1521 ohead.type = PICODATA_ITEM_SYLLPHON; 1522 1523 PICODBG_TRACE(("SYL expects accent at phonBuf[%i] = (%i,%i) (outReadPos=%i)", spho->phonReadPos, spho->phonBuf[spho->phonReadPos].pos, spho->phonBuf[spho->phonReadPos].sym,spho->outReadPos)); 1524 rv = getNextPosSym(spho,&pos,&sym,nextInPos); 1525 if (SPHO_POSSYM_OK != rv) { 1526 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list (%i)",rv)); 1527 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL); 1528 } 1529 ohead.info2 = picotrns_unplane(sym, &plane); 1530 PICODBG_ASSERT((PICOKFST_PLANE_ACCENTS == plane)); 1531 PICODBG_DEBUG(("SYL sets accent to %c", sym)); 1532 1533 /* for the time being, we force to use POS so we can transduce all fsts in a row without reconsulting the items */ 1534 PICODBG_TRACE(("SYL expects POS")); 1535 PICODBG_DEBUG(("SYL (2) treating item in inBuf range [%i,%i[",spho->outReadPos,nextInPos)); 1536 rv = getNextPosSym(spho,&pos,&sym,nextInPos); 1537 if (SPHO_POSSYM_OK != rv) { 1538 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list")); 1539 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL); 1540 } 1541 if (spho->wordStarted) { 1542 spho->wordStarted = FALSE; 1543 ohead.info1 = picotrns_unplane(sym, &plane); 1544 /* */ 1545 PICODBG_ASSERT(PICOKFST_PLANE_POS == plane); 1546 /* */ 1547 PICODBG_DEBUG(("SYL setting POS to %c", ohead.info1)); 1548 } else { 1549 ohead.info1 = PICODATA_ITEMINFO1_NA; 1550 } 1551 1552 PICODBG_DEBUG(("SYL (3) treating item in inBuf range [%i,%i[",spho->outReadPos,nextInPos)); 1553 /* get phonemes of that syllable; stop if syllable boundary or outside word */ 1554 sylsym = (PICOKFST_PLANE_PHONEMES << 8) 1555 + spho->syllSepId; 1556 PICODBG_DEBUG(("collecting syllable phonemes before headx position %i",nextInPos)); 1557 spho->sylWritePos = 0; 1558 while (SPHO_POSSYM_OK == (rv = getNextPosSym(spho,&pos,&sym,nextInPos)) && (sym != sylsym)) { 1559 spho->sylBuf[spho->sylWritePos++] = picotrns_unplane(sym, &plane); 1560 /* */ 1561 PICODBG_TRACE(("SYL adding phoneme to syllable: (pos %i,sym %i)[plane %i,sym %c]",pos,sym,plane,sym & 0xFF)); 1562 PICODBG_ASSERT((PICOKFST_PLANE_PHONEMES == plane)); 1563 } 1564 PICODBG_DEBUG(("SYL (4) treating item in inBuf range [%i,%i[",spho->outReadPos,nextInPos)); 1565 ohead.len = spho->sylWritePos; 1566 if (SPHO_POS_INVALID == rv) { 1567 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list")); 1568 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, PICO_WARN_INCOMPLETE, NULL, NULL); 1569 } else if ((SPHO_POSSYM_OUT_OF_RANGE == rv) || (SPHO_POSSYM_END == rv)) { 1570 PICODBG_DEBUG(("SYL arrived at end of word and/or end of phon buffer, go to next word")); 1571 spho->outReadPos = nextInPos; /* advance to next item */ 1572 spho->feedFollowState = SPHO_STEPSTATE_PROCESS_RECOMB; /* go to RECOMB after feed */ 1573 } else { 1574 PICODBG_ASSERT((sym == sylsym)); 1575 } 1576 PICODBG_DEBUG(("SYL (5) treating item in inBuf range [%i,%i[",spho->outReadPos,nextInPos)); 1577 1578 if (ohead.len > 0) { 1579 /* prepare syllable output */ 1580 picodata_put_itemparts(&ohead, spho->sylBuf, 1581 PICODATA_BUFSIZE_DEFAULT, spho->outBuf, 1582 spho->outBufSize, &clen); 1583 1584 spho->procState = SPHO_STEPSTATE_FEED; 1585 } else { /* skip feeding output of empty syllable */ 1586 spho->procState = spho->feedFollowState; 1587 } 1588 break; 1589 1590 case SPHO_STEPSTATE_FEED: 1591 /* **********************************************************************/ 1592 /* FEED: output output item and proceed to feedFollowState */ 1593 /* **********************************************************************/ 1594 1595 PICODBG_DEBUG(("FEED")); 1596 1597 PICODBG_DEBUG(("FEED putting outBuf item into cb")); 1598 1599 /*feeding items to PU output buffer*/ 1600 rv = picodata_cbPutItem(this->cbOut, spho->outBuf, 1601 spho->outBufSize, &clen); 1602 1603 PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG], 1604 (picoos_uint8 *)"spho: ", 1605 spho->outBuf, spho->outBufSize); 1606 1607 if (PICO_EXC_BUF_OVERFLOW == rv) { 1608 /* we have to redo this item */ 1609 PICODBG_DEBUG(("FEED got overflow, returning ICODATA_PU_OUT_FULL")); 1610 return PICODATA_PU_OUT_FULL; 1611 } else if (PICO_OK == rv) { 1612 *numBytesOutput += clen; 1613 spho->procState = spho->feedFollowState; 1614 PICODBG_DEBUG(("FEED ok, going back to procState %i", spho->procState)); 1615 return PICODATA_PU_BUSY; 1616 } else { 1617 PICODBG_DEBUG(("FEED got exception %i when trying to output item",rv)); 1618 spho->procState = spho->feedFollowState; 1619 return (picodata_step_result_t)rv; 1620 } 1621 break; 1622 1623 case SPHO_STEPSTATE_SHIFT: 1624 /* **********************************************************************/ 1625 /* SHIFT */ 1626 /* **********************************************************************/ 1627 /* If there exists a valid penultima, it should replace any left context (from 0 to activeStartPos) 1628 * else discard the current active range (from activeStartPos to activeEndPos), leaving the current 1629 * left context intact. Often, PARSE would move activeStartPos to 0, so that there is no left context 1630 * after the shift. 1631 */ 1632 1633 PICODBG_DEBUG(("SHIFT")); 1634 1635 if (spho->penultima != SPHO_POS_INVALID) { 1636 picoos_int16 shift; 1637 /* set penultima as new left context and set activeStartPos to the shifted activeEndPos */ 1638 PICODBG_DEBUG(( 1639 "SHIFT shifting penultima from %i to 0", 1640 spho->penultima)); 1641 shift = shift_range_left_1(spho, &spho->penultima, 0); 1642 if (shift < 0) { 1643 picoos_emRaiseException(this->common->em,PICO_ERR_OTHER,NULL,NULL); 1644 return PICODATA_PU_ERROR; 1645 } 1646 spho->activeStartPos = spho->activeEndPos 1647 - shift; 1648 spho->lastPhraseBoundPos -= shift; 1649 spho->suppressParseWordBound = FALSE; 1650 spho->suppressRecombWordBound = FALSE; 1651 1652 } else { 1653 picoos_int16 shift; 1654 picoos_bool lastPhraseBoundActive; 1655 if (spho->activeStartPos == spho->activeEndPos) { 1656 /* no items consumed; we have to abandon left context */ 1657 spho->activeStartPos = 0; 1658 } 1659 lastPhraseBoundActive = (spho->lastPhraseBoundPos >= spho->activeStartPos); 1660 /* dummy comment */ 1661 PICODBG_DEBUG(("SHIFT shift active end from %i to %i", 1662 spho->activeEndPos, spho->activeStartPos)); 1663 shift = shift_range_left_1(spho, &spho->activeEndPos, spho->activeStartPos); 1664 if (shift < 0) { 1665 picoos_emRaiseException(this->common->em,PICO_ERR_OTHER,NULL,NULL); 1666 return PICODATA_PU_ERROR; 1667 } 1668 if (lastPhraseBoundActive) { 1669 spho->lastPhraseBoundPos -= shift; 1670 } 1671 } 1672 1673 spho->procState = SPHO_STEPSTATE_INIT; 1674 break; 1675 1676 default: 1677 picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL); 1678 return PICODATA_PU_ERROR; 1679 break; 1680 1681 } /* switch (spho->procState) */ 1682 1683 } /* while (1) */ 1684 1685 /* should be never reached */ 1686 picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL); 1687 return PICODATA_PU_ERROR; 1688 } 1689 1690 #ifdef __cplusplus 1691 } 1692 #endif 1693 1694 /* end picospho.c */ 1695
