1 /* ANTLRTokenBuffer.cpp 2 * 3 * SOFTWARE RIGHTS 4 * 5 * We reserve no LEGAL rights to the Purdue Compiler Construction Tool 6 * Set (PCCTS) -- PCCTS is in the public domain. An individual or 7 * company may do whatever they wish with source code distributed with 8 * PCCTS or the code generated by PCCTS, including the incorporation of 9 * PCCTS, or its output, into commerical software. 10 * 11 * We encourage users to develop software with PCCTS. However, we do ask 12 * that credit is given to us for developing PCCTS. By "credit", 13 * we mean that if you incorporate our source code into one of your 14 * programs (commercial product, research project, or otherwise) that you 15 * acknowledge this fact somewhere in the documentation, research report, 16 * etc... If you like PCCTS and have developed a nice tool with the 17 * output, please mention that you developed it using PCCTS. In 18 * addition, we ask that this header remain intact in our source code. 19 * As long as these guidelines are kept, we expect to continue enhancing 20 * this system and expect to make other tools available as they are 21 * completed. 22 * 23 * ANTLR 1.33 24 * Terence Parr 25 * Parr Research Corporation 26 * with Purdue University and AHPCRC, University of Minnesota 27 * 1989-2000 28 */ 29 30 typedef int ANTLRTokenType; // fool AToken.h into compiling 31 32 class ANTLRParser; /* MR1 */ 33 34 #define ANTLR_SUPPORT_CODE 35 36 #include "pcctscfg.h" 37 38 #include ATOKENBUFFER_H 39 #include APARSER_H // MR23 40 41 typedef ANTLRAbstractToken *_ANTLRTokenPtr; 42 43 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW) 44 static unsigned char test[1000]; 45 #endif 46 47 #ifdef DBG_REFCOUNTTOKEN 48 int ANTLRRefCountToken::ctor = 0; /* MR23 */ 49 int ANTLRRefCountToken::dtor = 0; /* MR23 */ 50 #endif 51 52 ANTLRTokenBuffer:: 53 ANTLRTokenBuffer(ANTLRTokenStream *_input, int _k, int _chunk_size_formal) /* MR14 */ 54 { 55 this->input = _input; 56 this->k = _k; 57 buffer_size = chunk_size = _chunk_size_formal; 58 buffer = (_ANTLRTokenPtr *) 59 calloc(chunk_size+1,sizeof(_ANTLRTokenPtr )); 60 if ( buffer == NULL ) { 61 panic("cannot alloc token buffer"); 62 } 63 buffer++; // leave the first elem empty so tp-1 is valid ptr 64 65 tp = &buffer[0]; 66 last = tp-1; 67 next = &buffer[0]; 68 num_markers = 0; 69 end_of_buffer = &buffer[buffer_size-1]; 70 threshold = &buffer[(int)(buffer_size/2)]; // MR23 - Used to be 1.0/2.0 ! 71 _deleteTokens = 1; // assume we delete tokens 72 parser=NULL; // MR5 - uninitialized reference 73 } 74 75 static void f() {;} 76 ANTLRTokenBuffer:: 77 ~ANTLRTokenBuffer() 78 { 79 f(); 80 // Delete all remaining tokens (from 0..last inclusive) 81 if ( _deleteTokens ) 82 { 83 _ANTLRTokenPtr *z; 84 for (z=buffer; z<=last; z++) 85 { 86 (*z)->deref(); 87 // z->deref(); 88 #ifdef DBG_REFCOUNTTOKEN 89 /* MR23 */ printMessage(stderr, "##########dtor: deleting token '%s' (ref %d)\n", 90 ((ANTLRCommonToken *)*z)->getText(), (*z)->nref()); 91 #endif 92 if ( (*z)->nref()==0 ) 93 { 94 delete (*z); 95 } 96 } 97 } 98 99 if ( buffer!=NULL ) free((char *)(buffer-1)); 100 } 101 102 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW) 103 #include "pccts_stdio.h" 104 PCCTS_NAMESPACE_STD 105 #endif 106 107 _ANTLRTokenPtr ANTLRTokenBuffer:: 108 getToken() 109 { 110 if ( tp <= last ) // is there any buffered lookahead still to be read? 111 { 112 return *tp++; // read buffered lookahead 113 } 114 // out of buffered lookahead, get some more "real" 115 // input from getANTLRToken() 116 if ( num_markers==0 ) 117 { 118 if( next > threshold ) 119 { 120 #ifdef DBG_TBUF 121 /* MR23 */ printMessage(stderr,"getToken: next > threshold (high water is %d)\n", threshold-buffer); 122 #endif 123 makeRoom(); 124 } 125 } 126 else { 127 if ( next > end_of_buffer ) 128 { 129 #ifdef DBG_TBUF 130 /* MR23 */ printMessage(stderr,"getToken: next > end_of_buffer (size is %d)\n", buffer_size); 131 #endif 132 extendBuffer(); 133 } 134 } 135 *next = getANTLRToken(); 136 (*next)->ref(); // say we have a copy of this pointer in buffer 137 last = next; 138 next++; 139 tp = last; 140 return *tp++; 141 } 142 143 void ANTLRTokenBuffer:: 144 rewind(int pos) 145 { 146 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW) 147 /* MR23 */ printMessage(stderr, "rewind(%d)[nm=%d,from=%d,%d.n=%d]\n", pos, num_markers, tp-buffer,pos,test[pos]); 148 test[pos]--; 149 #endif 150 tp = &buffer[pos]; 151 num_markers--; 152 } 153 154 /* 155 * This function is used to specify that the token pointers read 156 * by the ANTLRTokenBuffer should be buffered up (to be reused later). 157 */ 158 int ANTLRTokenBuffer:: 159 mark() 160 { 161 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW) 162 test[tp-buffer]++; 163 /* MR23 */ printMessage(stderr,"mark(%d)[nm=%d,%d.n=%d]\n",tp-buffer,num_markers+1,tp-buffer,test[tp-buffer]); 164 #endif 165 num_markers++; 166 return tp - buffer; 167 } 168 169 /* 170 * returns the token pointer n positions ahead. 171 * This implies that bufferedToken(1) gets the NEXT symbol of lookahead. 172 * This is used in conjunction with the ANTLRParser lookahead buffer. 173 * 174 * No markers are set or anything. A bunch of input is buffered--that's all. 175 * The tp pointer is left alone as the lookahead has not been advanced 176 * with getToken(). The next call to getToken() will find a token 177 * in the buffer and won't have to call getANTLRToken(). 178 * 179 * If this is called before a consume() is done, how_many_more_i_need is 180 * set to 'n'. 181 */ 182 _ANTLRTokenPtr ANTLRTokenBuffer:: 183 bufferedToken(int n) 184 { 185 // int how_many_more_i_need = (last-tp < 0) ? n : n-(last-tp)-1; 186 int how_many_more_i_need = (tp > last) ? n : n-(last-tp)-1; 187 // Make sure that at least n tokens are available in the buffer 188 #ifdef DBG_TBUF 189 /* MR23 */ printMessage(stderr, "bufferedToken(%d)\n", n); 190 #endif 191 for (int i=1; i<=how_many_more_i_need; i++) 192 { 193 if ( next > end_of_buffer ) // buffer overflow? 194 { 195 extendBuffer(); 196 } 197 *next = getANTLRToken(); 198 (*next)->ref(); // say we have a copy of this pointer in buffer 199 last = next; 200 next++; 201 } 202 return tp[n - 1]; 203 } 204 205 /* If no markers are set, the none of the input needs to be saved (except 206 * for the lookahead Token pointers). We save only k-1 token pointers as 207 * we are guaranteed to do a getANTLRToken() right after this because otherwise 208 * we wouldn't have needed to extend the buffer. 209 * 210 * If there are markers in the buffer, we need to save things and so 211 * extendBuffer() is called. 212 */ 213 void ANTLRTokenBuffer:: 214 makeRoom() 215 { 216 #ifdef DBG_TBUF 217 /* MR23 */ printMessage(stderr, "in makeRoom.................\n"); 218 /* MR23 */ printMessage(stderr, "num_markers==%d\n", num_markers); 219 #endif 220 /* 221 if ( num_markers == 0 ) 222 { 223 */ 224 #ifdef DBG_TBUF 225 /* MR23 */ printMessage(stderr, "moving lookahead and resetting next\n"); 226 227 _ANTLRTokenPtr *r; 228 /* MR23 */ printMessage(stderr, "tbuf = ["); 229 for (r=buffer; r<=last; r++) 230 { 231 if ( *r==NULL ) /* MR23 */ printMessage(stderr, " xxx"); 232 else /* MR23 */ printMessage(stderr, " '%s'", ((ANTLRCommonToken *)*r)->getText()); 233 } 234 /* MR23 */ printMessage(stderr, " ]\n"); 235 236 /* MR23 */ printMessage(stderr, 237 "before: tp=%d, last=%d, next=%d, threshold=%d\n",tp-buffer,last-buffer,next-buffer,threshold-buffer); 238 #endif 239 240 // Delete all tokens from 0..last-(k-1) inclusive 241 if ( _deleteTokens ) 242 { 243 _ANTLRTokenPtr *z; 244 for (z=buffer; z<=last-(k-1); z++) 245 { 246 (*z)->deref(); 247 // z->deref(); 248 #ifdef DBG_REFCOUNTTOKEN 249 /* MR23 */ printMessage(stderr, "##########makeRoom: deleting token '%s' (ref %d)\n", 250 ((ANTLRCommonToken *)*z)->getText(), (*z)->nref()); 251 #endif 252 if ( (*z)->nref()==0 ) 253 { 254 delete (*z); 255 } 256 } 257 } 258 259 // reset the buffer to initial conditions, but move k-1 symbols 260 // to the beginning of buffer and put new input symbol at k 261 _ANTLRTokenPtr *p = buffer, *q = last-(k-1)+1; 262 // ANTLRAbstractToken **p = buffer, **q = end_of_buffer-(k-1)+1; 263 #ifdef DBG_TBUF 264 /* MR23 */ printMessage(stderr, "lookahead buffer = ["); 265 #endif 266 for (int i=1; i<=(k-1); i++) 267 { 268 *p++ = *q++; 269 #ifdef DBG_TBUF 270 /* MR23 */ printMessage(stderr, 271 " '%s'", ((ANTLRCommonToken *)buffer[i-1])->getText()); 272 #endif 273 } 274 #ifdef DBG_TBUF 275 /* MR23 */ printMessage(stderr, " ]\n"); 276 #endif 277 next = &buffer[k-1]; 278 tp = &buffer[k-1]; // tp points to what will be filled in next 279 last = tp-1; 280 #ifdef DBG_TBUF 281 /* MR23 */ printMessage(stderr, 282 "after: tp=%d, last=%d, next=%d\n", 283 tp-buffer, last-buffer, next-buffer); 284 #endif 285 /* 286 } 287 else { 288 extendBuffer(); 289 } 290 */ 291 } 292 293 /* This function extends 'buffer' by chunk_size and returns with all 294 * pointers at the same relative positions in the buffer (the buffer base 295 * address could have changed in realloc()) except that 'next' comes 296 * back set to where the next token should be stored. All other pointers 297 * are untouched. 298 */ 299 void 300 ANTLRTokenBuffer:: 301 extendBuffer() 302 { 303 int save_last = last-buffer, save_tp = tp-buffer, save_next = next-buffer; 304 #ifdef DBG_TBUF 305 /* MR23 */ printMessage(stderr, "extending physical buffer\n"); 306 #endif 307 buffer_size += chunk_size; 308 buffer = (_ANTLRTokenPtr *) 309 realloc((char *)(buffer-1), 310 (buffer_size+1)*sizeof(_ANTLRTokenPtr )); 311 if ( buffer == NULL ) { 312 panic("cannot alloc token buffer"); 313 } 314 buffer++; // leave the first elem empty so tp-1 is valid ptr 315 316 tp = buffer + save_tp; // put the pointers back to same relative position 317 last = buffer + save_last; 318 next = buffer + save_next; 319 end_of_buffer = &buffer[buffer_size-1]; 320 threshold = &buffer[(int)(buffer_size*(1.0/2.0))]; 321 322 /* 323 // zero out new token ptrs so we'll know if something to delete in buffer 324 ANTLRAbstractToken **p = end_of_buffer-chunk_size+1; 325 for (; p<=end_of_buffer; p++) *p = NULL; 326 */ 327 } 328 329 ANTLRParser * ANTLRTokenBuffer:: // MR1 330 setParser(ANTLRParser *p) { // MR1 331 ANTLRParser *old=parser; // MR1 332 parser=p; // MR1 333 input->setParser(p); // MR1 334 return old; // MR1 335 } // MR1 336 // MR1 337 ANTLRParser * ANTLRTokenBuffer:: // MR1 338 getParser() { // MR1 339 return parser; // MR1 340 } // MR1 341 342 void ANTLRTokenBuffer::panic(const char *msg) // MR23 343 { 344 if (parser) //MR23 345 parser->panic(msg); //MR23 346 else //MR23 347 exit(PCCTS_EXIT_FAILURE); 348 } 349 350 //MR23 351 int ANTLRTokenBuffer::printMessage(FILE* pFile, const char* pFormat, ...) 352 { 353 va_list marker; 354 va_start( marker, pFormat ); 355 356 int iRet = 0; 357 if (parser) 358 parser->printMessageV(pFile, pFormat, marker); 359 else 360 iRet = vfprintf(pFile, pFormat, marker); 361 362 va_end( marker ); 363 return iRet; 364 } 365 366 /* to avoid having to link in another file just for the smart token ptr 367 * stuff, we include it here. Ugh. 368 * 369 * MR23 This causes nothing but problems for IDEs. 370 * Change from .cpp to .h 371 * 372 */ 373 374 #include ATOKPTR_IMPL_H 375
