Home | History | Annotate | Download | only in ubrkperf
      1 /***********************************************************************
      2  * Copyright (C) 2016 and later: Unicode, Inc. and others.
      3  * License & terms of use: http://www.unicode.org/copyright.html#License
      4  *
      5  ***********************************************************************
      6  ***********************************************************************
      7  * COPYRIGHT:
      8  * Copyright (C) 2001-2012 IBM, Inc.   All Rights Reserved.
      9  *
     10  ***********************************************************************/
     11 /********************************************************************************
     12 *
     13 * File ubrkperf.cpp
     14 *
     15 * Modification History:
     16 *        Name                     Description
     17 *     Vladimir Weinstein          First Version, based on collperf
     18 *
     19 *********************************************************************************
     20 */
     21 
     22 //
     23 //  This program tests break iterator performance
     24 //      Currently we test only ICU APIs with the future possibility of testing *nix & win32 APIs
     25 //      (if any)
     26 //      A text file is required as input.  It must be in utf-8 or utf-16 format,
     27 //      and include a byte order mark.  Either LE or BE format is OK.
     28 //
     29 
     30 const char gUsageString[] =
     31  "usage:  ubrkperf options...\n"
     32     "-help                      Display this message.\n"
     33     "-file file_name            utf-16/utf-8 format file.\n"
     34     "-locale name               ICU locale to use.  Default is en_US\n"
     35     "-langid 0x1234             Windows Language ID number.  Default to value for -locale option\n"
     36     "                              see http://msdn.microsoft.com/library/psdk/winbase/nls_8xo3.htm\n"
     37     "-win                       Run test using Windows native services. (currently not working) (ICU is default)\n"
     38     "-unix                      Run test using Unix word breaking services. (currently not working) \n"
     39     "-mac                       Run test using MacOSX word breaking services.\n"
     40     "-uselen                    Use API with string lengths.  Default is null-terminated strings\n"
     41     "-char                      Use character break iterator\n"
     42     "-word                      Use word break iterator\n"
     43     "-line                      Use line break iterator\n"
     44     "-sentence                  Use sentence break iterator\n"
     45     "-loop nnnn                 Loopcount for test.  Adjust for reasonable total running time.\n"
     46     "-iloop n                   Inner Loop Count.  Default = 1.  Number of calls to function\n"
     47     "                               under test at each call point.  For measuring test overhead.\n"
     48     "-terse                     Terse numbers-only output.  Intended for use by scripts.\n"
     49     "-dump                      Display stuff.\n"
     50     "-capi                      Use C APIs instead of C++ APIs (currently not working)\n"
     51     "-next                      Do the next test\n"
     52     "-isBound                   Do the isBound test\n"
     53     ;
     54 
     55 
     56 #include <stdio.h>
     57 #include <string.h>
     58 #include <stdlib.h>
     59 #include <math.h>
     60 #include <locale.h>
     61 #include <errno.h>
     62 #include <sys/stat.h>
     63 
     64 #include <unicode/utypes.h>
     65 #include <unicode/ucol.h>
     66 #include <unicode/ucoleitr.h>
     67 #include <unicode/uloc.h>
     68 #include <unicode/ustring.h>
     69 #include <unicode/ures.h>
     70 #include <unicode/uchar.h>
     71 #include <unicode/ucnv.h>
     72 #include <unicode/utf8.h>
     73 
     74 #include <unicode/brkiter.h>
     75 
     76 
     77 #if U_PLATFORM_HAS_WIN32_API
     78 #include <windows.h>
     79 #else
     80 //
     81 //  Stubs for Windows API functions when building on UNIXes.
     82 //
     83 #include <sys/time.h>
     84 unsigned long timeGetTime() {
     85     struct timeval t;
     86     gettimeofday(&t, 0);
     87     unsigned long val = t.tv_sec * 1000;  // Let it overflow.  Who cares.
     88     val += t.tv_usec / 1000;
     89     return val;
     90 };
     91 #define MAKELCID(a,b) 0
     92 #endif
     93 
     94 
     95 //
     96 //  Command line option variables
     97 //     These global variables are set according to the options specified
     98 //     on the command line by the user.
     99 char * opt_fName      = 0;
    100 char * opt_locale     = "en_US";
    101 int    opt_langid     = 0;         // Defaults to value corresponding to opt_locale.
    102 char * opt_rules      = 0;
    103 UBool  opt_help       = FALSE;
    104 int    opt_time       = 0;
    105 int    opt_loopCount  = 0;
    106 int    opt_passesCount= 1;
    107 UBool  opt_terse      = FALSE;
    108 UBool  opt_icu        = TRUE;
    109 UBool  opt_win        = FALSE;      // Run with Windows native functions.
    110 UBool  opt_unix       = FALSE;      // Run with UNIX strcoll, strxfrm functions.
    111 UBool  opt_mac        = FALSE;      // Run with MacOSX word break services.
    112 UBool  opt_uselen     = FALSE;
    113 UBool  opt_dump       = FALSE;
    114 UBool  opt_char       = FALSE;
    115 UBool  opt_word       = FALSE;
    116 UBool  opt_line       = FALSE;
    117 UBool  opt_sentence   = FALSE;
    118 UBool  opt_capi       = FALSE;
    119 
    120 UBool  opt_next       = FALSE;
    121 UBool  opt_isBound    = FALSE;
    122 
    123 
    124 
    125 //
    126 //   Definitions for the command line options
    127 //
    128 struct OptSpec {
    129     const char *name;
    130     enum {FLAG, NUM, STRING} type;
    131     void *pVar;
    132 };
    133 
    134 OptSpec opts[] = {
    135     {"-file",        OptSpec::STRING, &opt_fName},
    136     {"-locale",      OptSpec::STRING, &opt_locale},
    137     {"-langid",      OptSpec::NUM,    &opt_langid},
    138     {"-win",         OptSpec::FLAG,   &opt_win},
    139     {"-unix",        OptSpec::FLAG,   &opt_unix},
    140     {"-mac",         OptSpec::FLAG,   &opt_mac},
    141     {"-uselen",      OptSpec::FLAG,   &opt_uselen},
    142     {"-loop",        OptSpec::NUM,    &opt_loopCount},
    143     {"-time",        OptSpec::NUM,    &opt_time},
    144     {"-passes",      OptSpec::NUM,    &opt_passesCount},
    145     {"-char",        OptSpec::FLAG,   &opt_char},
    146     {"-word",        OptSpec::FLAG,   &opt_word},
    147     {"-line",        OptSpec::FLAG,   &opt_line},
    148     {"-sentence",    OptSpec::FLAG,   &opt_sentence},
    149     {"-terse",       OptSpec::FLAG,   &opt_terse},
    150     {"-dump",        OptSpec::FLAG,   &opt_dump},
    151     {"-capi",        OptSpec::FLAG,   &opt_capi},
    152     {"-next",        OptSpec::FLAG,   &opt_next},
    153     {"-isBound",     OptSpec::FLAG,   &opt_isBound},
    154     {"-help",        OptSpec::FLAG,   &opt_help},
    155     {"-?",           OptSpec::FLAG,   &opt_help},
    156     {0, OptSpec::FLAG, 0}
    157 };
    158 
    159 
    160 //---------------------------------------------------------------------------
    161 //
    162 //  Global variables pointing to and describing the test file
    163 //
    164 //---------------------------------------------------------------------------
    165 
    166 //DWORD          gWinLCID;
    167 BreakIterator *brkit = NULL;
    168 UChar *text = NULL;
    169 int32_t textSize = 0;
    170 
    171 
    172 
    173 #if U_PLATFORM_IS_DARWIN_BASED
    174 #include <ApplicationServices/ApplicationServices.h>
    175 enum{
    176   kUCTextBreakAllMask = (kUCTextBreakClusterMask | kUCTextBreakWordMask | kUCTextBreakLineMask)
    177     };
    178 UCTextBreakType breakTypes[4] = {kUCTextBreakCharMask, kUCTextBreakClusterMask, kUCTextBreakWordMask, kUCTextBreakLineMask};
    179 TextBreakLocatorRef breakRef;
    180 UCTextBreakType macBreakType;
    181 
    182 void createMACBrkIt() {
    183   OSStatus status = noErr;
    184   LocaleRef lref;
    185   status = LocaleRefFromLocaleString(opt_locale, &lref);
    186   status = UCCreateTextBreakLocator(lref, 0, kUCTextBreakAllMask, (TextBreakLocatorRef*)&breakRef);
    187   if(opt_char == TRUE) {
    188     macBreakType = kUCTextBreakClusterMask;
    189   } else if(opt_word == TRUE) {
    190     macBreakType = kUCTextBreakWordMask;
    191   } else if(opt_line == TRUE) {
    192     macBreakType = kUCTextBreakLineMask;
    193   } else if(opt_sentence == TRUE) {
    194     // error
    195     // brkit = BreakIterator::createSentenceInstance(opt_locale, status);
    196   } else {
    197     // default is character iterator
    198     macBreakType = kUCTextBreakClusterMask;
    199       }
    200 }
    201 #endif
    202 
    203 void createICUBrkIt() {
    204   //
    205   //  Set up an ICU break iterator
    206   //
    207   UErrorCode          status = U_ZERO_ERROR;
    208   if(opt_char == TRUE) {
    209     brkit = BreakIterator::createCharacterInstance(opt_locale, status);
    210   } else if(opt_word == TRUE) {
    211     brkit = BreakIterator::createWordInstance(opt_locale, status);
    212   } else if(opt_line == TRUE) {
    213     brkit = BreakIterator::createLineInstance(opt_locale, status);
    214   } else if(opt_sentence == TRUE) {
    215     brkit = BreakIterator::createSentenceInstance(opt_locale, status);
    216   } else {
    217     // default is character iterator
    218     brkit = BreakIterator::createCharacterInstance(opt_locale, status);
    219   }
    220   if (status==U_USING_DEFAULT_WARNING && opt_terse==FALSE) {
    221     fprintf(stderr, "Warning, U_USING_DEFAULT_WARNING for %s\n", opt_locale);
    222   }
    223   if (status==U_USING_FALLBACK_WARNING && opt_terse==FALSE) {
    224     fprintf(stderr, "Warning, U_USING_FALLBACK_ERROR for %s\n", opt_locale);
    225   }
    226 
    227 }
    228 
    229 //---------------------------------------------------------------------------
    230 //
    231 //  ProcessOptions()    Function to read the command line options.
    232 //
    233 //---------------------------------------------------------------------------
    234 UBool ProcessOptions(int argc, const char **argv, OptSpec opts[])
    235 {
    236     int         i;
    237     int         argNum;
    238     const char  *pArgName;
    239     OptSpec    *pOpt;
    240 
    241     for (argNum=1; argNum<argc; argNum++) {
    242         pArgName = argv[argNum];
    243         for (pOpt = opts;  pOpt->name != 0; pOpt++) {
    244             if (strcmp(pOpt->name, pArgName) == 0) {
    245                 switch (pOpt->type) {
    246                 case OptSpec::FLAG:
    247                     *(UBool *)(pOpt->pVar) = TRUE;
    248                     break;
    249                 case OptSpec::STRING:
    250                     argNum ++;
    251                     if (argNum >= argc) {
    252                         fprintf(stderr, "value expected for \"%s\" option.\n", pOpt->name);
    253                         return FALSE;
    254                     }
    255                     *(const char **)(pOpt->pVar)  = argv[argNum];
    256                     break;
    257                 case OptSpec::NUM:
    258                     argNum ++;
    259                     if (argNum >= argc) {
    260                         fprintf(stderr, "value expected for \"%s\" option.\n", pOpt->name);
    261                         return FALSE;
    262                     }
    263                     char *endp;
    264                     i = strtol(argv[argNum], &endp, 0);
    265                     if (endp == argv[argNum]) {
    266                         fprintf(stderr, "integer value expected for \"%s\" option.\n", pOpt->name);
    267                         return FALSE;
    268                     }
    269                     *(int *)(pOpt->pVar) = i;
    270                 }
    271                 break;
    272             }
    273         }
    274         if (pOpt->name == 0)
    275         {
    276             fprintf(stderr, "Unrecognized option \"%s\"\n", pArgName);
    277             return FALSE;
    278         }
    279     }
    280 return TRUE;
    281 }
    282 
    283 
    284 void doForwardTest() {
    285   if (opt_terse == FALSE) {
    286     printf("Doing the forward test\n");
    287   }
    288   int32_t noBreaks = 0;
    289   int32_t i = 0;
    290   unsigned long startTime = timeGetTime();
    291   unsigned long elapsedTime = 0;
    292   if(opt_icu) {
    293     createICUBrkIt();
    294     brkit->setText(UnicodeString(text, textSize));
    295     brkit->first();
    296     if (opt_terse == FALSE) {
    297       printf("Warmup\n");
    298     }
    299     int j;
    300     while((j = brkit->next()) != BreakIterator::DONE) {
    301       noBreaks++;
    302       //fprintf(stderr, "%d ", j);
    303     }
    304 
    305     if (opt_terse == FALSE) {
    306       printf("Measure\n");
    307     }
    308     startTime = timeGetTime();
    309     for(i = 0; i < opt_loopCount; i++) {
    310       brkit->first();
    311       while(brkit->next() != BreakIterator::DONE) {
    312       }
    313     }
    314 
    315     elapsedTime = timeGetTime()-startTime;
    316   } else if(opt_mac) {
    317 #if U_PLATFORM_IS_DARWIN_BASED
    318     createMACBrkIt();
    319     UniChar* filePtr = text;
    320     OSStatus status = noErr;
    321     UniCharCount startOffset = 0, breakOffset = 0, numUniChars = textSize;
    322     startOffset = 0;
    323     //printf("\t---Search forward--\n");
    324 
    325     while (startOffset < numUniChars)
    326     {
    327 	status = UCFindTextBreak(breakRef, macBreakType, kUCTextBreakLeadingEdgeMask, filePtr, numUniChars,
    328                                startOffset, &breakOffset);
    329       //require_action(status == noErr, EXIT, printf( "**UCFindTextBreak failed: startOffset %d, status %d\n", (int)startOffset, (int)status));
    330       //require_action((breakOffset <= numUniChars),EXIT, printf("**UCFindTextBreak breakOffset too big: startOffset %d, breakOffset %d\n", (int)startOffset, (int)breakOffset));
    331 
    332       // Output break
    333       //printf("\t%d\n", (int)breakOffset);
    334 
    335       // Increment counters
    336 	noBreaks++;
    337       startOffset = breakOffset;
    338     }
    339     startTime = timeGetTime();
    340     for(i = 0; i < opt_loopCount; i++) {
    341       startOffset = 0;
    342 
    343       while (startOffset < numUniChars)
    344 	{
    345 	  status = UCFindTextBreak(breakRef, macBreakType, kUCTextBreakLeadingEdgeMask, filePtr, numUniChars,
    346 				   startOffset, &breakOffset);
    347 	  // Increment counters
    348 	  startOffset = breakOffset;
    349 	}
    350     }
    351     elapsedTime = timeGetTime()-startTime;
    352     UCDisposeTextBreakLocator(&breakRef);
    353 #endif
    354 
    355 
    356   }
    357 
    358 
    359   if (opt_terse == FALSE) {
    360   int32_t loopTime = (int)(float(1000) * ((float)elapsedTime/(float)opt_loopCount));
    361       int32_t timePerCU = (int)(float(1000) * ((float)loopTime/(float)textSize));
    362       int32_t timePerBreak = (int)(float(1000) * ((float)loopTime/(float)noBreaks));
    363       printf("forward break iteration average loop time %d\n", loopTime);
    364       printf("number of code units %d average time per code unit %d\n", textSize, timePerCU);
    365       printf("number of breaks %d average time per break %d\n", noBreaks, timePerBreak);
    366   } else {
    367       printf("time=%d\nevents=%d\nsize=%d\n", elapsedTime, noBreaks, textSize);
    368   }
    369 
    370 
    371 }
    372 
    373 void doIsBoundTest() {
    374   int32_t noBreaks = 0, hit = 0;
    375   int32_t i = 0, j = 0;
    376   unsigned long startTime = timeGetTime();
    377   unsigned long elapsedTime = 0;
    378   createICUBrkIt();
    379   brkit->setText(UnicodeString(text, textSize));
    380   brkit->first();
    381   for(j = 0; j < textSize; j++) {
    382     if(brkit->isBoundary(j)) {
    383       noBreaks++;
    384       //fprintf(stderr, "%d ", j);
    385     }
    386   }
    387   /*
    388   while(brkit->next() != BreakIterator::DONE) {
    389     noBreaks++;
    390   }
    391   */
    392 
    393   startTime = timeGetTime();
    394   for(i = 0; i < opt_loopCount; i++) {
    395     for(j = 0; j < textSize; j++) {
    396       if(brkit->isBoundary(j)) {
    397         hit++;
    398       }
    399     }
    400   }
    401 
    402   elapsedTime = timeGetTime()-startTime;
    403   int32_t loopTime = (int)(float(1000) * ((float)elapsedTime/(float)opt_loopCount));
    404   if (opt_terse == FALSE) {
    405       int32_t timePerCU = (int)(float(1000) * ((float)loopTime/(float)textSize));
    406       int32_t timePerBreak = (int)(float(1000) * ((float)loopTime/(float)noBreaks));
    407       printf("forward break iteration average loop time %d\n", loopTime);
    408       printf("number of code units %d average time per code unit %d\n", textSize, timePerCU);
    409       printf("number of breaks %d average time per break %d\n", noBreaks, timePerBreak);
    410   } else {
    411       printf("time=%d\nevents=%d\nsize=%d\n", elapsedTime, noBreaks, textSize);
    412   }
    413 }
    414 
    415 //----------------------------------------------------------------------------------------
    416 //
    417 //   UnixConvert   -- Convert the lines of the file to the encoding for UNIX
    418 //                    Since it appears that Unicode support is going in the general
    419 //                    direction of the use of UTF-8 locales, that is the approach
    420 //                    that is used here.
    421 //
    422 //----------------------------------------------------------------------------------------
    423 void  UnixConvert() {
    424 #if 0
    425     int    line;
    426 
    427     UConverter   *cvrtr;    // An ICU code page converter.
    428     UErrorCode    status = U_ZERO_ERROR;
    429 
    430 
    431     cvrtr = ucnv_open("utf-8", &status);    // we are just doing UTF-8 locales for now.
    432     if (U_FAILURE(status)) {
    433         fprintf(stderr, "ICU Converter open failed.: %d\n", &status);
    434         exit(-1);
    435     }
    436     // redo for unix
    437     for (line=0; line < gNumFileLines; line++) {
    438         int sizeNeeded = ucnv_fromUChars(cvrtr,
    439                                          0,            // ptr to target buffer.
    440                                          0,            // length of target buffer.
    441                                          gFileLines[line].name,
    442                                          -1,           //  source is null terminated
    443                                          &status);
    444         if (status != U_BUFFER_OVERFLOW_ERROR && status != U_ZERO_ERROR) {
    445             fprintf(stderr, "Conversion from Unicode, something is wrong.\n");
    446             exit(-1);
    447         }
    448         status = U_ZERO_ERROR;
    449         gFileLines[line].unixName = new char[sizeNeeded+1];
    450         sizeNeeded = ucnv_fromUChars(cvrtr,
    451                                          gFileLines[line].unixName, // ptr to target buffer.
    452                                          sizeNeeded+1, // length of target buffer.
    453                                          gFileLines[line].name,
    454                                          -1,           //  source is null terminated
    455                                          &status);
    456         if (U_FAILURE(status)) {
    457             fprintf(stderr, "ICU Conversion Failed.: %d\n", status);
    458             exit(-1);
    459         }
    460         gFileLines[line].unixName[sizeNeeded] = 0;
    461     };
    462     ucnv_close(cvrtr);
    463 #endif
    464 }
    465 
    466 
    467 //----------------------------------------------------------------------------------------
    468 //
    469 //  class UCharFile   Class to hide all the gorp to read a file in
    470 //                    and produce a stream of UChars.
    471 //
    472 //----------------------------------------------------------------------------------------
    473 class UCharFile {
    474 public:
    475     UCharFile(const char *fileName);
    476     ~UCharFile();
    477     UChar   get();
    478     UBool   eof() {return fEof;};
    479     UBool   error() {return fError;};
    480     int32_t size() { return fFileSize; };
    481 
    482 private:
    483     UCharFile (const UCharFile &other) {};                         // No copy constructor.
    484     UCharFile & operator = (const UCharFile &other) {return *this;};   // No assignment op
    485 
    486     FILE         *fFile;
    487     const char   *fName;
    488     UBool        fEof;
    489     UBool        fError;
    490     UChar        fPending2ndSurrogate;
    491     int32_t      fFileSize;
    492 
    493     enum {UTF16LE, UTF16BE, UTF8} fEncoding;
    494 };
    495 
    496 UCharFile::UCharFile(const char * fileName) {
    497     fEof                 = FALSE;
    498     fError               = FALSE;
    499     fName                = fileName;
    500     struct stat buf;
    501     int32_t result = stat(fileName, &buf);
    502     if(result != 0) {
    503       fprintf(stderr, "Error getting info\n");
    504       fFileSize = -1;
    505     } else {
    506       fFileSize = buf.st_size;
    507     }
    508     fFile                = fopen(fName, "rb");
    509     fPending2ndSurrogate = 0;
    510     if (fFile == NULL) {
    511         fprintf(stderr, "Can not open file \"%s\"\n", opt_fName);
    512         fError = TRUE;
    513         return;
    514     }
    515     //
    516     //  Look for the byte order mark at the start of the file.
    517     //
    518     int BOMC1, BOMC2, BOMC3;
    519     BOMC1 = fgetc(fFile);
    520     BOMC2 = fgetc(fFile);
    521 
    522     if (BOMC1 == 0xff && BOMC2 == 0xfe) {
    523         fEncoding = UTF16LE; }
    524     else if (BOMC1 == 0xfe && BOMC2 == 0xff) {
    525         fEncoding = UTF16BE; }
    526     else if (BOMC1 == 0xEF && BOMC2 == 0xBB && (BOMC3 = fgetc(fFile)) == 0xBF ) {
    527         fEncoding = UTF8; }
    528     else
    529     {
    530         fprintf(stderr, "collperf:  file \"%s\" encoding must be UTF-8 or UTF-16, and "
    531             "must include a BOM.\n", fileName);
    532         fError = true;
    533         return;
    534     }
    535 }
    536 
    537 
    538 UCharFile::~UCharFile() {
    539     fclose(fFile);
    540 }
    541 
    542 
    543 
    544 UChar UCharFile::get() {
    545     UChar   c;
    546     switch (fEncoding) {
    547     case UTF16LE:
    548         {
    549             int  cL, cH;
    550             cL = fgetc(fFile);
    551             cH = fgetc(fFile);
    552             c  = cL  | (cH << 8);
    553             if (cH == EOF) {
    554                 c   = 0;
    555                 fEof = TRUE;
    556             }
    557             break;
    558         }
    559     case UTF16BE:
    560         {
    561             int  cL, cH;
    562             cH = fgetc(fFile);
    563             cL = fgetc(fFile);
    564             c  = cL  | (cH << 8);
    565             if (cL == EOF) {
    566                 c   = 0;
    567                 fEof = TRUE;
    568             }
    569             break;
    570         }
    571     case UTF8:
    572         {
    573             if (fPending2ndSurrogate != 0) {
    574                 c = fPending2ndSurrogate;
    575                 fPending2ndSurrogate = 0;
    576                 break;
    577             }
    578 
    579             int ch = fgetc(fFile);   // Note:  c and ch are separate cause eof test doesn't work on UChar type.
    580             if (ch == EOF) {
    581                 c = 0;
    582                 fEof = TRUE;
    583                 break;
    584             }
    585 
    586             if (ch <= 0x7f) {
    587                 // It's ascii.  No further utf-8 conversion.
    588                 c = ch;
    589                 break;
    590             }
    591 
    592             // Figure out the lenght of the char and read the rest of the bytes
    593             //   into a temp array.
    594             int nBytes;
    595             if (ch >= 0xF0) {nBytes=4;}
    596             else if (ch >= 0xE0) {nBytes=3;}
    597             else if (ch >= 0xC0) {nBytes=2;}
    598             else {
    599                 fprintf(stderr, "not likely utf-8 encoded file %s contains corrupt data at offset %d.\n", fName, ftell(fFile));
    600                 fError = TRUE;
    601                 return 0;
    602             }
    603 
    604             unsigned char  bytes[10];
    605             bytes[0] = (unsigned char)ch;
    606             int i;
    607             for (i=1; i<nBytes; i++) {
    608                 bytes[i] = fgetc(fFile);
    609                 if (bytes[i] < 0x80 || bytes[i] >= 0xc0) {
    610                     fprintf(stderr, "utf-8 encoded file %s contains corrupt data at offset %d. Expected %d bytes, byte %d is invalid. First byte is %02X\n", fName, ftell(fFile), nBytes, i, ch);
    611                     fError = TRUE;
    612                     return 0;
    613                 }
    614             }
    615 
    616             // Convert the bytes from the temp array to a Unicode char.
    617             i = 0;
    618             uint32_t  cp;
    619             U8_NEXT_UNSAFE(bytes, i, cp);
    620             c = (UChar)cp;
    621 
    622             if (cp >= 0x10000) {
    623                 // The code point needs to be broken up into a utf-16 surrogate pair.
    624                 //  Process first half this time through the main loop, and
    625                 //   remember the other half for the next time through.
    626                 UChar utf16Buf[3];
    627                 i = 0;
    628                 UTF16_APPEND_CHAR_UNSAFE(utf16Buf, i, cp);
    629                 fPending2ndSurrogate = utf16Buf[1];
    630                 c = utf16Buf[0];
    631             }
    632             break;
    633         };
    634     }
    635     return c;
    636 }
    637 
    638 
    639 //----------------------------------------------------------------------------------------
    640 //
    641 //    Main   --  process command line, read in and pre-process the test file,
    642 //                 call other functions to do the actual tests.
    643 //
    644 //----------------------------------------------------------------------------------------
    645 int main(int argc, const char** argv) {
    646     if (ProcessOptions(argc, argv, opts) != TRUE || opt_help || opt_fName == 0) {
    647         printf(gUsageString);
    648         exit (1);
    649     }
    650     // Make sure that we've only got one API selected.
    651     if (opt_mac || opt_unix || opt_win) opt_icu = FALSE;
    652     if (opt_mac || opt_unix) opt_win = FALSE;
    653     if (opt_mac) opt_unix = FALSE;
    654 
    655     UErrorCode          status = U_ZERO_ERROR;
    656 
    657 
    658 
    659     //
    660     //  Set up a Windows LCID
    661     //
    662   /*
    663     if (opt_langid != 0) {
    664         gWinLCID = MAKELCID(opt_langid, SORT_DEFAULT);
    665     }
    666     else {
    667         gWinLCID = uloc_getLCID(opt_locale);
    668     }
    669   */
    670 
    671     //
    672     //  Set the UNIX locale
    673     //
    674     if (opt_unix) {
    675         if (setlocale(LC_ALL, opt_locale) == 0) {
    676             fprintf(stderr, "setlocale(LC_ALL, %s) failed.\n", opt_locale);
    677             exit(-1);
    678         }
    679     }
    680 
    681     // Read in  the input file.
    682     //   File assumed to be utf-16.
    683     //   Lines go onto heap buffers.  Global index array to line starts is created.
    684     //   Lines themselves are null terminated.
    685     //
    686 
    687     UCharFile f(opt_fName);
    688     if (f.error()) {
    689         exit(-1);
    690     }
    691     int32_t fileSize = f.size();
    692     const int STARTSIZE = 70000;
    693     int32_t bufSize = 0;
    694     int32_t charCount = 0;
    695     if(fileSize != -1) {
    696       text = (UChar *)malloc(fileSize*sizeof(UChar));
    697       bufSize = fileSize;
    698     } else {
    699       text = (UChar *)malloc(STARTSIZE*sizeof(UChar));
    700       bufSize = STARTSIZE;
    701     }
    702     if(text == NULL) {
    703       fprintf(stderr, "Allocating buffer failed\n");
    704       exit(-1);
    705     }
    706 
    707 
    708     //  Read the file, split into lines, and save in memory.
    709     //  Loop runs once per utf-16 value from the input file,
    710     //    (The number of bytes read from file per loop iteration depends on external encoding.)
    711     for (;;) {
    712 
    713         UChar c = f.get();
    714         if(f.eof()) {
    715           break;
    716         }
    717         if (f.error()){
    718           exit(-1);
    719         }
    720         // We now have a good UTF-16 value in c.
    721         text[charCount++] = c;
    722         if(charCount == bufSize) {
    723           text = (UChar *)realloc(text, 2*bufSize*sizeof(UChar));
    724           if(text == NULL) {
    725             fprintf(stderr, "Reallocating buffer failed\n");
    726             exit(-1);
    727           }
    728           bufSize *= 2;
    729         }
    730     }
    731 
    732 
    733     if (opt_terse == FALSE) {
    734         printf("file \"%s\", %d charCount code units.\n", opt_fName, charCount);
    735     }
    736 
    737     textSize = charCount;
    738 
    739 
    740 
    741 
    742     //
    743     //  Dump file contents if requested.
    744     //
    745     if (opt_dump) {
    746       // dump file, etc... possibly
    747     }
    748 
    749 
    750     //
    751     //  We've got the file read into memory.  Go do something with it.
    752     //
    753     int32_t i = 0;
    754     for(i = 0; i < opt_passesCount; i++) {
    755       if(opt_loopCount != 0) {
    756         if(opt_next) {
    757           doForwardTest();
    758         } else if(opt_isBound) {
    759           doIsBoundTest();
    760         } else {
    761           doForwardTest();
    762         }
    763       } else if(opt_time != 0) {
    764 
    765       }
    766     }
    767 
    768   if(text != NULL) {
    769     free(text);
    770   }
    771     if(brkit != NULL) {
    772       delete brkit;
    773     }
    774 
    775     return 0;
    776 }
    777