Home | History | Annotate | Download | only in toolutil 
      1 /*
      2 *******************************************************************************
      3 *
      4 *   Copyright (C) 2000-2010, International Business Machines
      5 *   Corporation and others.  All Rights Reserved.
      6 *
      7 *******************************************************************************
      8 *   file name:  uparse.c
      9 *   encoding:   US-ASCII
     10 *   tab size:   8 (not used)
     11 *   indentation:4
     12 *
     13 *   created on: 2000apr18
     14 *   created by: Markus W. Scherer
     15 *
     16 *   This file provides a parser for files that are delimited by one single
     17 *   character like ';' or TAB. Example: the Unicode Character Properties files
     18 *   like UnicodeData.txt are semicolon-delimited.
     19 */
     20 
     21 #include "unicode/utypes.h"
     22 #include "cstring.h"
     23 #include "filestrm.h"
     24 #include "uparse.h"
     25 #include "unicode/uchar.h"
     26 #include "unicode/ustring.h"
     27 #include "ustr_imp.h"
     28 
     29 #include <stdio.h>
     30 
     31 /* Is c a whitespace character? */
     32 #define IS_INV_WHITESPACE(c) ((c)==' ' || (c)=='\t' || (c)=='\r' || (c)=='\n')
     33 
     34 U_CAPI const char * U_EXPORT2
     35 u_skipWhitespace(const char *s) {
     36     while(IS_INV_WHITESPACE(*s)) {
     37         ++s;
     38     }
     39     return s;
     40 }
     41 
     42 U_CAPI char * U_EXPORT2
     43 u_rtrim(char *s) {
     44     char *end=uprv_strchr(s, 0);
     45     while(s<end && IS_INV_WHITESPACE(*(end-1))) {
     46         *--end = 0;
     47     }
     48     return end;
     49 }
     50 
     51 /*
     52  * If the string starts with # @missing: then return the pointer to the
     53  * following non-whitespace character.
     54  * Otherwise return the original pointer.
     55  * Unicode 5.0 adds such lines in some data files to document
     56  * default property values.
     57  * Poor man's regex for variable amounts of white space.
     58  */
     59 static const char *
     60 getMissingLimit(const char *s) {
     61     const char *s0=s;
     62     if(
     63         *(s=u_skipWhitespace(s))=='#' &&
     64         *(s=u_skipWhitespace(s+1))=='@' &&
     65         0==strncmp((s=u_skipWhitespace(s+1)), "missing", 7) &&
     66         *(s=u_skipWhitespace(s+7))==':'
     67     ) {
     68         return u_skipWhitespace(s+1);
     69     } else {
     70         return s0;
     71     }
     72 }
     73 
     74 U_CAPI void U_EXPORT2
     75 u_parseDelimitedFile(const char *filename, char delimiter,
     76                      char *fields[][2], int32_t fieldCount,
     77                      UParseLineFn *lineFn, void *context,
     78                      UErrorCode *pErrorCode) {
     79     FileStream *file;
     80     char line[300];
     81     char *start, *limit;
     82     int32_t i, length;
     83 
     84     if(U_FAILURE(*pErrorCode)) {
     85         return;
     86     }
     87 
     88     if(fields==NULL || lineFn==NULL || fieldCount<=0) {
     89         *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
     90         return;
     91     }
     92 
     93     if(filename==NULL || *filename==0 || (*filename=='-' && filename[1]==0)) {
     94         filename=NULL;
     95         file=T_FileStream_stdin();
     96     } else {
     97         file=T_FileStream_open(filename, "r");
     98     }
     99     if(file==NULL) {
    100         *pErrorCode=U_FILE_ACCESS_ERROR;
    101         return;
    102     }
    103 
    104     while(T_FileStream_readLine(file, line, sizeof(line))!=NULL) {
    105         /* remove trailing newline characters */
    106         length=(int32_t)(u_rtrim(line)-line);
    107 
    108         /*
    109          * detect a line with # @missing:
    110          * start parsing after that, or else from the beginning of the line
    111          * set the default warning for @missing lines
    112          */
    113         start=(char *)getMissingLimit(line);
    114         if(start==line) {
    115             *pErrorCode=U_ZERO_ERROR;
    116         } else {
    117             *pErrorCode=U_USING_DEFAULT_WARNING;
    118         }
    119 
    120         /* skip this line if it is empty or a comment */
    121         if(*start==0 || *start=='#') {
    122             continue;
    123         }
    124 
    125         /* remove in-line comments */
    126         limit=uprv_strchr(start, '#');
    127         if(limit!=NULL) {
    128             /* get white space before the pound sign */
    129             while(limit>start && IS_INV_WHITESPACE(*(limit-1))) {
    130                 --limit;
    131             }
    132 
    133             /* truncate the line */
    134             *limit=0;
    135         }
    136 
    137         /* skip lines with only whitespace */
    138         if(u_skipWhitespace(start)[0]==0) {
    139             continue;
    140         }
    141 
    142         /* for each field, call the corresponding field function */
    143         for(i=0; i<fieldCount; ++i) {
    144             /* set the limit pointer of this field */
    145             limit=start;
    146             while(*limit!=delimiter && *limit!=0) {
    147                 ++limit;
    148             }
    149 
    150             /* set the field start and limit in the fields array */
    151             fields[i][0]=start;
    152             fields[i][1]=limit;
    153 
    154             /* set start to the beginning of the next field, if any */
    155             start=limit;
    156             if(*start!=0) {
    157                 ++start;
    158             } else if(i+1<fieldCount) {
    159                 *pErrorCode=U_PARSE_ERROR;
    160                 limit=line+length;
    161                 i=fieldCount;
    162                 break;
    163             }
    164         }
    165 
    166         /* error in a field function? */
    167         if(U_FAILURE(*pErrorCode)) {
    168             break;
    169         }
    170 
    171         /* call the field function */
    172         lineFn(context, fields, fieldCount, pErrorCode);
    173         if(U_FAILURE(*pErrorCode)) {
    174             break;
    175         }
    176     }
    177 
    178     if(filename!=NULL) {
    179         T_FileStream_close(file);
    180     }
    181 }
    182 
    183 /*
    184  * parse a list of code points
    185  * store them as a UTF-32 string in dest[destCapacity]
    186  * return the number of code points
    187  */
    188 U_CAPI int32_t U_EXPORT2
    189 u_parseCodePoints(const char *s,
    190                   uint32_t *dest, int32_t destCapacity,
    191                   UErrorCode *pErrorCode) {
    192     char *end;
    193     uint32_t value;
    194     int32_t count;
    195 
    196     if(U_FAILURE(*pErrorCode)) {
    197         return 0;
    198     }
    199     if(s==NULL || destCapacity<0 || (destCapacity>0 && dest==NULL)) {
    200         *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
    201         return 0;
    202     }
    203 
    204     count=0;
    205     for(;;) {
    206         s=u_skipWhitespace(s);
    207         if(*s==';' || *s==0) {
    208             return count;
    209         }
    210 
    211         /* read one code point */
    212         value=(uint32_t)uprv_strtoul(s, &end, 16);
    213         if(end<=s || (!IS_INV_WHITESPACE(*end) && *end!=';' && *end!=0) || value>=0x110000) {
    214             *pErrorCode=U_PARSE_ERROR;
    215             return 0;
    216         }
    217 
    218         /* append it to the destination array */
    219         if(count<destCapacity) {
    220             dest[count++]=value;
    221         } else {
    222             *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
    223         }
    224 
    225         /* go to the following characters */
    226         s=end;
    227     }
    228 }
    229 
    230 /*
    231  * parse a list of code points
    232  * store them as a string in dest[destCapacity]
    233  * set the first code point in *pFirst
    234  * @return The length of the string in numbers of UChars.
    235  */
    236 U_CAPI int32_t U_EXPORT2
    237 u_parseString(const char *s,
    238               UChar *dest, int32_t destCapacity,
    239               uint32_t *pFirst,
    240               UErrorCode *pErrorCode) {
    241     char *end;
    242     uint32_t value;
    243     int32_t destLength;
    244 
    245     if(U_FAILURE(*pErrorCode)) {
    246         return 0;
    247     }
    248     if(s==NULL || destCapacity<0 || (destCapacity>0 && dest==NULL)) {
    249         *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
    250     }
    251 
    252     if(pFirst!=NULL) {
    253         *pFirst=0xffffffff;
    254     }
    255 
    256     destLength=0;
    257     for(;;) {
    258         s=u_skipWhitespace(s);
    259         if(*s==';' || *s==0) {
    260             if(destLength<destCapacity) {
    261                 dest[destLength]=0;
    262             } else if(destLength==destCapacity) {
    263                 *pErrorCode=U_STRING_NOT_TERMINATED_WARNING;
    264             } else {
    265                 *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
    266             }
    267             return destLength;
    268         }
    269 
    270         /* read one code point */
    271         value=(uint32_t)uprv_strtoul(s, &end, 16);
    272         if(end<=s || (!IS_INV_WHITESPACE(*end) && *end!=';' && *end!=0) || value>=0x110000) {
    273             *pErrorCode=U_PARSE_ERROR;
    274             return 0;
    275         }
    276 
    277         /* store the first code point */
    278         if(pFirst!=NULL) {
    279             *pFirst=value;
    280             pFirst=NULL;
    281         }
    282 
    283         /* append it to the destination array */
    284         if((destLength+U16_LENGTH(value))<=destCapacity) {
    285             U16_APPEND_UNSAFE(dest, destLength, value);
    286         } else {
    287             destLength+=U16_LENGTH(value);
    288         }
    289 
    290         /* go to the following characters */
    291         s=end;
    292     }
    293 }
    294 
    295 /* read a range like start or start..end */
    296 U_CAPI int32_t U_EXPORT2
    297 u_parseCodePointRangeAnyTerminator(const char *s,
    298                                    uint32_t *pStart, uint32_t *pEnd,
    299                                    const char **terminator,
    300                                    UErrorCode *pErrorCode) {
    301     char *end;
    302     uint32_t value;
    303 
    304     if(U_FAILURE(*pErrorCode)) {
    305         return 0;
    306     }
    307     if(s==NULL || pStart==NULL || pEnd==NULL) {
    308         *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
    309         return 0;
    310     }
    311 
    312     /* read the start code point */
    313     s=u_skipWhitespace(s);
    314     value=(uint32_t)uprv_strtoul(s, &end, 16);
    315     if(end<=s || value>=0x110000) {
    316         *pErrorCode=U_PARSE_ERROR;
    317         return 0;
    318     }
    319     *pStart=*pEnd=value;
    320 
    321     /* is there a "..end"? */
    322     s=u_skipWhitespace(end);
    323     if(*s!='.' || s[1]!='.') {
    324         *terminator=end;
    325         return 1;
    326     }
    327     s=u_skipWhitespace(s+2);
    328 
    329     /* read the end code point */
    330     value=(uint32_t)uprv_strtoul(s, &end, 16);
    331     if(end<=s || value>=0x110000) {
    332         *pErrorCode=U_PARSE_ERROR;
    333         return 0;
    334     }
    335     *pEnd=value;
    336 
    337     /* is this a valid range? */
    338     if(value<*pStart) {
    339         *pErrorCode=U_PARSE_ERROR;
    340         return 0;
    341     }
    342 
    343     *terminator=end;
    344     return value-*pStart+1;
    345 }
    346 
    347 U_CAPI int32_t U_EXPORT2
    348 u_parseCodePointRange(const char *s,
    349                       uint32_t *pStart, uint32_t *pEnd,
    350                       UErrorCode *pErrorCode) {
    351     const char *terminator;
    352     int32_t rangeLength=
    353         u_parseCodePointRangeAnyTerminator(s, pStart, pEnd, &terminator, pErrorCode);
    354     if(U_SUCCESS(*pErrorCode)) {
    355         terminator=u_skipWhitespace(terminator);
    356         if(*terminator!=';' && *terminator!=0) {
    357             *pErrorCode=U_PARSE_ERROR;
    358             return 0;
    359         }
    360     }
    361     return rangeLength;
    362 }
    363 
    364 U_CAPI int32_t U_EXPORT2
    365 u_parseUTF8(const char *source, int32_t sLen, char *dest, int32_t destCapacity, UErrorCode *status) {
    366     const char *read = source;
    367     int32_t i = 0;
    368     unsigned int value = 0;
    369     if(sLen == -1) {
    370         sLen = (int32_t)strlen(source);
    371     }
    372 
    373     while(read < source+sLen) {
    374         sscanf(read, "%2x", &value);
    375         if(i < destCapacity) {
    376             dest[i] = (char)value;
    377         }
    378         i++;
    379         read += 2;
    380     }
    381     return u_terminateChars(dest, destCapacity, i, status);
    382 }
    383