Home | History | Annotate | Download | only in preprocessor 
      1 /****************************************************************************\
      2 Copyright (c) 2002, NVIDIA Corporation.
      3 
      4 NVIDIA Corporation("NVIDIA") supplies this software to you in
      5 consideration of your agreement to the following terms, and your use,
      6 installation, modification or redistribution of this NVIDIA software
      7 constitutes acceptance of these terms.  If you do not agree with these
      8 terms, please do not use, install, modify or redistribute this NVIDIA
      9 software.
     10 
     11 In consideration of your agreement to abide by the following terms, and
     12 subject to these terms, NVIDIA grants you a personal, non-exclusive
     13 license, under NVIDIA's copyrights in this original NVIDIA software (the
     14 "NVIDIA Software"), to use, reproduce, modify and redistribute the
     15 NVIDIA Software, with or without modifications, in source and/or binary
     16 forms; provided that if you redistribute the NVIDIA Software, you must
     17 retain the copyright notice of NVIDIA, this notice and the following
     18 text and disclaimers in all such redistributions of the NVIDIA Software.
     19 Neither the name, trademarks, service marks nor logos of NVIDIA
     20 Corporation may be used to endorse or promote products derived from the
     21 NVIDIA Software without specific prior written permission from NVIDIA.
     22 Except as expressly stated in this notice, no other rights or licenses
     23 express or implied, are granted by NVIDIA herein, including but not
     24 limited to any patent rights that may be infringed by your derivative
     25 works or by other works in which the NVIDIA Software may be
     26 incorporated. No hardware is licensed hereunder.
     27 
     28 THE NVIDIA SOFTWARE IS BEING PROVIDED ON AN "AS IS" BASIS, WITHOUT
     29 WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED,
     30 INCLUDING WITHOUT LIMITATION, WARRANTIES OR CONDITIONS OF TITLE,
     31 NON-INFRINGEMENT, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR
     32 ITS USE AND OPERATION EITHER ALONE OR IN COMBINATION WITH OTHER
     33 PRODUCTS.
     34 
     35 IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT,
     36 INCIDENTAL, EXEMPLARY, CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
     37 TO, LOST PROFITS; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
     38 USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) OR ARISING IN ANY WAY
     39 OUT OF THE USE, REPRODUCTION, MODIFICATION AND/OR DISTRIBUTION OF THE
     40 NVIDIA SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT,
     41 TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF
     42 NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     43 \****************************************************************************/
     44 //
     45 // tokens.c
     46 //
     47 
     48 #include <stdlib.h>
     49 #include <stdio.h>
     50 #include <string.h>
     51 #include <ctype.h>
     52 
     53 #include "compiler/debug.h"
     54 #include "compiler/preprocessor/slglobals.h"
     55 #include "compiler/util.h"
     56 
     57 ///////////////////////////////////////////////////////////////////////////////////////////////
     58 //////////////////////// Preprocessor and Token Recorder and Playback: ////////////////////////
     59 ///////////////////////////////////////////////////////////////////////////////////////////////
     60 
     61 /*
     62  * idstr()
     63  * Copy a string to a malloc'ed block and convert it into something suitable
     64  * for an ID
     65  *
     66  */
     67 
     68 static char *idstr(const char *fstr, MemoryPool *pool)
     69 {
     70     size_t len;
     71     char *str, *t;
     72     const char *f;
     73 
     74     len = strlen(fstr);
     75     if (!pool)
     76         str = (char *) malloc(len + 1);
     77     else
     78         str = (char *) mem_Alloc(pool, len + 1);
     79 
     80     for (f=fstr, t=str; *f; f++) {
     81         if (isalnum(*f)) *t++ = *f;
     82         else if (*f == '.' || *f == '/') *t++ = '_';
     83     }
     84     *t = 0;
     85     return str;
     86 } // idstr
     87 
     88 
     89 /*
     90  * lNewBlock()
     91  *
     92  */
     93 
     94 static TokenBlock *lNewBlock(TokenStream *fTok, MemoryPool *pool)
     95 {
     96     TokenBlock *lBlock;
     97 
     98     if (!pool)
     99         lBlock = (TokenBlock *) malloc(sizeof(TokenBlock) + 256);
    100     else
    101         lBlock = (TokenBlock *) mem_Alloc(pool, sizeof(TokenBlock) + 256);
    102     lBlock->count = 0;
    103     lBlock->current = 0;
    104     lBlock->data = (unsigned char *) lBlock + sizeof(TokenBlock);
    105     lBlock->max = 256;
    106     lBlock->next = NULL;
    107     if (fTok->head) {
    108         fTok->current->next = lBlock;
    109     } else {
    110         fTok->head = lBlock;
    111     }
    112     fTok->current = lBlock;
    113     return lBlock;
    114 } // lNewBlock
    115 
    116 /*
    117  * lAddByte()
    118  *
    119  */
    120 
    121 static void lAddByte(TokenStream *fTok, unsigned char fVal)
    122 {
    123     TokenBlock *lBlock;
    124     lBlock = fTok->current;
    125     if (lBlock->count >= lBlock->max)
    126         lBlock = lNewBlock(fTok, 0);
    127     lBlock->data[lBlock->count++] = fVal;
    128 } // lAddByte
    129 
    130 
    131 
    132 /*
    133  * lReadByte() - Get the next byte from a stream.
    134  *
    135  */
    136 
    137 static int lReadByte(TokenStream *pTok)
    138 {
    139     TokenBlock *lBlock;
    140     int lval = -1;
    141 
    142     lBlock = pTok->current;
    143     if (lBlock) {
    144         if (lBlock->current >= lBlock->count) {
    145             lBlock = lBlock->next;
    146             if (lBlock)
    147                 lBlock->current = 0;
    148             pTok->current = lBlock;
    149         }
    150         if (lBlock)
    151             lval = lBlock->data[lBlock->current++];
    152     }
    153     return lval;
    154 } // lReadByte
    155 
    156 /////////////////////////////////////// Global Functions://////////////////////////////////////
    157 
    158 /*
    159  * NewTokenStream()
    160  *
    161  */
    162 
    163 TokenStream *NewTokenStream(const char *name, MemoryPool *pool)
    164 {
    165     TokenStream *pTok;
    166 
    167     if (!pool)
    168         pTok = (TokenStream *) malloc(sizeof(TokenStream));
    169     else
    170         pTok = (TokenStream*)mem_Alloc(pool, sizeof(TokenStream));
    171     pTok->next = NULL;
    172     pTok->name = idstr(name, pool);
    173     pTok->head = NULL;
    174     pTok->current = NULL;
    175     lNewBlock(pTok, pool);
    176     return pTok;
    177 } // NewTokenStream
    178 
    179 /*
    180  * DeleteTokenStream()
    181  *
    182  */
    183 
    184 void DeleteTokenStream(TokenStream *pTok)
    185 {
    186     TokenBlock *pBlock, *nBlock;
    187 
    188     if (pTok) {
    189         pBlock = pTok->head;
    190         while (pBlock) {
    191             nBlock = pBlock->next;
    192             free(pBlock);
    193             pBlock = nBlock;
    194         }
    195         if (pTok->name)
    196             free(pTok->name);
    197         free(pTok);
    198     }
    199 } // DeleteTokenStream
    200 
    201 /*
    202  * RecordToken() - Add a token to the end of a list for later playback or printout.
    203  *
    204  */
    205 
    206 void RecordToken(TokenStream *pTok, int token, yystypepp * yylvalpp)
    207 {
    208     const char *s;
    209     char *str=NULL;
    210 
    211     if (token > 256)
    212         lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80));
    213     else
    214         lAddByte(pTok, (unsigned char)(token & 0x7f));
    215     switch (token) {
    216     case CPP_IDENTIFIER:
    217     case CPP_TYPEIDENTIFIER:
    218     case CPP_STRCONSTANT:
    219         s = GetAtomString(atable, yylvalpp->sc_ident);
    220         while (*s)
    221             lAddByte(pTok, (unsigned char) *s++);
    222         lAddByte(pTok, 0);
    223         break;
    224     case CPP_FLOATCONSTANT:
    225     case CPP_INTCONSTANT:
    226          str=yylvalpp->symbol_name;
    227          while (*str){
    228             lAddByte(pTok, (unsigned char) *str++);
    229          }
    230          lAddByte(pTok, 0);
    231          break;
    232     case '(':
    233         lAddByte(pTok, (unsigned char)(yylvalpp->sc_int ? 1 : 0));
    234     default:
    235         break;
    236     }
    237 } // RecordToken
    238 
    239 /*
    240  * RewindTokenStream() - Reset a token stream in preperation for reading.
    241  *
    242  */
    243 
    244 void RewindTokenStream(TokenStream *pTok)
    245 {
    246     if (pTok->head) {
    247         pTok->current = pTok->head;
    248         pTok->current->current = 0;
    249     }
    250 } // RewindTokenStream
    251 
    252 /*
    253  * ReadToken() - Read the next token from a stream.
    254  *
    255  */
    256 
    257 int ReadToken(TokenStream *pTok, yystypepp * yylvalpp)
    258 {
    259     char symbol_name[MAX_SYMBOL_NAME_LEN + 1];
    260     char string_val[MAX_STRING_LEN + 1];
    261     int ltoken, len;
    262     char ch;
    263 
    264     ltoken = lReadByte(pTok);
    265     if (ltoken >= 0) {
    266         if (ltoken > 127)
    267             ltoken += 128;
    268         switch (ltoken) {
    269         case CPP_IDENTIFIER:
    270         case CPP_TYPEIDENTIFIER:
    271             len = 0;
    272             ch = lReadByte(pTok);
    273             while ((ch >= 'a' && ch <= 'z') ||
    274                      (ch >= 'A' && ch <= 'Z') ||
    275                      (ch >= '0' && ch <= '9') ||
    276                      ch == '_')
    277             {
    278                 if (len < MAX_SYMBOL_NAME_LEN) {
    279                     symbol_name[len++] = ch;
    280                     ch = lReadByte(pTok);
    281                 }
    282             }
    283             symbol_name[len] = '\0';
    284             assert(ch == '\0');
    285             yylvalpp->sc_ident = LookUpAddString(atable, symbol_name);
    286             return CPP_IDENTIFIER;
    287             break;
    288         case CPP_STRCONSTANT:
    289             len = 0;
    290             while ((ch = lReadByte(pTok)) != 0)
    291                 if (len < MAX_STRING_LEN)
    292                     string_val[len++] = ch;
    293             string_val[len] = '\0';
    294             yylvalpp->sc_ident = LookUpAddString(atable, string_val);
    295             break;
    296         case CPP_FLOATCONSTANT:
    297             len = 0;
    298             ch = lReadByte(pTok);
    299             while ((ch >= '0' && ch <= '9')||(ch=='e'||ch=='E'||ch=='.')||(ch=='+'||ch=='-'))
    300             {
    301                 if (len < MAX_SYMBOL_NAME_LEN) {
    302                     symbol_name[len++] = ch;
    303                     ch = lReadByte(pTok);
    304                 }
    305             }
    306             symbol_name[len] = '\0';
    307             assert(ch == '\0');
    308             strcpy(yylvalpp->symbol_name,symbol_name);
    309             yylvalpp->sc_fval=(float)atof_dot(yylvalpp->symbol_name);
    310             break;
    311         case CPP_INTCONSTANT:
    312             len = 0;
    313             ch = lReadByte(pTok);
    314             while ((ch >= '0' && ch <= '9'))
    315             {
    316                 if (len < MAX_SYMBOL_NAME_LEN) {
    317                     symbol_name[len++] = ch;
    318                     ch = lReadByte(pTok);
    319                 }
    320             }
    321             symbol_name[len] = '\0';
    322             assert(ch == '\0');
    323             strcpy(yylvalpp->symbol_name,symbol_name);
    324             yylvalpp->sc_int=atoi(yylvalpp->symbol_name);
    325             break;
    326         case '(':
    327             yylvalpp->sc_int = lReadByte(pTok);
    328             break;
    329         }
    330         return ltoken;
    331     }
    332     return EOF_SY;
    333 } // ReadToken
    334 
    335 typedef struct TokenInputSrc {
    336     InputSrc            base;
    337     TokenStream         *tokens;
    338     int                 (*final)(CPPStruct *);
    339 } TokenInputSrc;
    340 
    341 static int scan_token(TokenInputSrc *in, yystypepp * yylvalpp)
    342 {
    343     int token = ReadToken(in->tokens, yylvalpp);
    344     int (*final)(CPPStruct *);
    345     cpp->tokenLoc->file = cpp->currentInput->name;
    346     cpp->tokenLoc->line = cpp->currentInput->line;
    347     if (token == '\n') {
    348         in->base.line++;
    349         return token;
    350     }
    351     if (token > 0) return token;
    352     cpp->currentInput = in->base.prev;
    353     final = in->final;
    354     free(in);
    355     if (final && !final(cpp)) return -1;
    356     return cpp->currentInput->scan(cpp->currentInput, yylvalpp);
    357 }
    358 
    359 int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(CPPStruct *))
    360 {
    361     TokenInputSrc *in = malloc(sizeof(TokenInputSrc));
    362     memset(in, 0, sizeof(TokenInputSrc));
    363     in->base.name = name;
    364     in->base.prev = cpp->currentInput;
    365     in->base.scan = (int (*)(InputSrc *, yystypepp *))scan_token;
    366     in->base.line = 1;
    367     in->tokens = ts;
    368     in->final = final;
    369     RewindTokenStream(ts);
    370     cpp->currentInput = &in->base;
    371     return 1;
    372 }
    373 
    374 typedef struct UngotToken {
    375     InputSrc    base;
    376     int         token;
    377     yystypepp     lval;
    378 } UngotToken;
    379 
    380 static int reget_token(UngotToken *t, yystypepp * yylvalpp)
    381 {
    382     int token = t->token;
    383     *yylvalpp = t->lval;
    384     cpp->currentInput = t->base.prev;
    385     free(t);
    386     return token;
    387 }
    388 
    389 void UngetToken(int token, yystypepp * yylvalpp) {
    390     UngotToken *t = malloc(sizeof(UngotToken));
    391     memset(t, 0, sizeof(UngotToken));
    392     t->token = token;
    393     t->lval = *yylvalpp;
    394     t->base.scan = (void *)reget_token;
    395     t->base.prev = cpp->currentInput;
    396     t->base.name = cpp->currentInput->name;
    397     t->base.line = cpp->currentInput->line;
    398     cpp->currentInput = &t->base;
    399 }
    400 
    401 
    402 void DumpTokenStream(FILE *fp, TokenStream *s, yystypepp * yylvalpp) {
    403     int token;
    404     char str[100];
    405 
    406     if (fp == 0) fp = stdout;
    407     RewindTokenStream(s);
    408     while ((token = ReadToken(s, yylvalpp)) > 0) {
    409         switch (token) {
    410         case CPP_IDENTIFIER:
    411         case CPP_TYPEIDENTIFIER:
    412             sprintf(str, "%s ", GetAtomString(atable, yylvalpp->sc_ident));
    413             break;
    414         case CPP_STRCONSTANT:
    415             sprintf(str, "\"%s\"", GetAtomString(atable, yylvalpp->sc_ident));
    416             break;
    417         case CPP_FLOATCONSTANT:
    418             //printf("%g9.6 ", yylvalpp->sc_fval);
    419             break;
    420         case CPP_INTCONSTANT:
    421             //printf("%d ", yylvalpp->sc_int);
    422             break;
    423         default:
    424             if (token >= 127)
    425                 sprintf(str, "%s ", GetAtomString(atable, token));
    426             else
    427                 sprintf(str, "%c", token);
    428             break;
    429         }
    430         CPPDebugLogMsg(str);
    431     }
    432 }
    433 
    434 ///////////////////////////////////////////////////////////////////////////////////////////////
    435 /////////////////////////////////////// End of tokens.c ///////////////////////////////////////
    436 ///////////////////////////////////////////////////////////////////////////////////////////////
    437