Home | History | Annotate | Download | only in cintltst
      1 /********************************************************************
      2  * Copyright (c) 1997-2011, International Business Machines
      3  * Corporation and others. All Rights Reserved.
      4  ********************************************************************
      5  *
      6  * File UCNVSELTST.C
      7  *
      8  * Modification History:
      9  *        Name                     Description
     10  *     MOHAMED ELDAWY               Creation
     11  ********************************************************************
     12  */
     13 
     14 /* C API AND FUNCTIONALITY TEST FOR CONVERTER SELECTOR (ucnvsel.h)*/
     15 
     16 #include "ucnvseltst.h"
     17 
     18 #include <stdio.h>
     19 
     20 #include "unicode/utypes.h"
     21 #include "unicode/ucnvsel.h"
     22 #include "unicode/ustring.h"
     23 #include "cmemory.h"
     24 #include "cstring.h"
     25 #include "propsvec.h"
     26 
     27 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
     28 
     29 #define FILENAME_BUFFER 1024
     30 
     31 #define TDSRCPATH  ".." U_FILE_SEP_STRING "test" U_FILE_SEP_STRING "testdata" U_FILE_SEP_STRING
     32 
     33 static void TestSelector(void);
     34 static void TestUPropsVector(void);
     35 void addCnvSelTest(TestNode** root);  /* Declaration required to suppress compiler warnings. */
     36 
     37 void addCnvSelTest(TestNode** root)
     38 {
     39     addTest(root, &TestSelector, "tsconv/ucnvseltst/TestSelector");
     40     addTest(root, &TestUPropsVector, "tsconv/ucnvseltst/TestUPropsVector");
     41 }
     42 
     43 static const char **gAvailableNames = NULL;
     44 static int32_t gCountAvailable = 0;
     45 
     46 static UBool
     47 getAvailableNames() {
     48   int32_t i;
     49   if (gAvailableNames != NULL) {
     50     return TRUE;
     51   }
     52   gCountAvailable = ucnv_countAvailable();
     53   if (gCountAvailable == 0) {
     54     log_data_err("No converters available.\n");
     55     return FALSE;
     56   }
     57   gAvailableNames = (const char **)uprv_malloc(gCountAvailable * sizeof(const char *));
     58   if (gAvailableNames == NULL) {
     59     log_err("unable to allocate memory for %ld available converter names\n",
     60             (long)gCountAvailable);
     61     return FALSE;
     62   }
     63   for (i = 0; i < gCountAvailable; ++i) {
     64     gAvailableNames[i] = ucnv_getAvailableName(i);
     65   }
     66   return TRUE;
     67 }
     68 
     69 static void
     70 releaseAvailableNames() {
     71   uprv_free((void *)gAvailableNames);
     72   gAvailableNames = NULL;
     73   gCountAvailable = 0;
     74 }
     75 
     76 static const char **
     77 getEncodings(int32_t start, int32_t step, int32_t count, int32_t *pCount) {
     78   const char **names;
     79   int32_t i;
     80 
     81   *pCount = 0;
     82   if (count <= 0) {
     83     return NULL;
     84   }
     85   names = (const char **)uprv_malloc(count * sizeof(char *));
     86   if (names == NULL) {
     87     log_err("memory allocation error for %ld pointers\n", (long)count);
     88     return NULL;
     89   }
     90   if (step == 0 && count > 0) {
     91     step = 1;
     92   }
     93   for (i = 0; i < count; ++i) {
     94     if (0 <= start && start < gCountAvailable) {
     95       names[i] = gAvailableNames[start];
     96       start += step;
     97       ++*pCount;
     98     }
     99   }
    100   return names;
    101 }
    102 
    103 #if 0
    104 /*
    105  * ucnvsel_open() does not support "no encodings":
    106  * Given 0 encodings it will open a selector for all available ones.
    107  */
    108 static const char **
    109 getNoEncodings(int32_t *pCount) {
    110   *pCount = 0;
    111   return NULL;
    112 }
    113 #endif
    114 
    115 static const char **
    116 getOneEncoding(int32_t *pCount) {
    117   return getEncodings(1, 0, 1, pCount);
    118 }
    119 
    120 static const char **
    121 getFirstEvenEncodings(int32_t *pCount) {
    122   return getEncodings(0, 2, 25, pCount);
    123 }
    124 
    125 static const char **
    126 getMiddleEncodings(int32_t *pCount) {
    127   return getEncodings(gCountAvailable - 12, 1, 22, pCount);
    128 }
    129 
    130 static const char **
    131 getLastEncodings(int32_t *pCount) {
    132   return getEncodings(gCountAvailable - 1, -1, 25, pCount);
    133 }
    134 
    135 static const char **
    136 getSomeEncodings(int32_t *pCount) {
    137   /* 20 evenly distributed */
    138   return getEncodings(5, (gCountAvailable + 19)/ 20, 20, pCount);
    139 }
    140 
    141 static const char **
    142 getEveryThirdEncoding(int32_t *pCount) {
    143   return getEncodings(2, 3, (gCountAvailable + 2 )/ 3, pCount);
    144 }
    145 
    146 static const char **
    147 getAllEncodings(int32_t *pCount) {
    148   return getEncodings(0, 1, gCountAvailable, pCount);
    149 }
    150 
    151 typedef const char **GetEncodingsFn(int32_t *);
    152 
    153 static GetEncodingsFn *const getEncodingsFns[] = {
    154   getOneEncoding,
    155   getFirstEvenEncodings,
    156   getMiddleEncodings,
    157   getLastEncodings,
    158   getSomeEncodings,
    159   getEveryThirdEncoding,
    160   getAllEncodings
    161 };
    162 
    163 static FILE *fopenOrError(const char *filename) {
    164     int32_t needLen;
    165     FILE *f;
    166     char fnbuf[FILENAME_BUFFER];
    167     const char* directory= ctest_dataSrcDir();
    168     needLen = uprv_strlen(directory)+uprv_strlen(TDSRCPATH)+uprv_strlen(filename)+1;
    169     if(needLen > FILENAME_BUFFER) {
    170         log_err("FAIL: Could not load %s. Filename buffer overflow, needed %d but buffer is %d\n",
    171                 filename, needLen, FILENAME_BUFFER);
    172         return NULL;
    173     }
    174 
    175     strcpy(fnbuf, directory);
    176     strcat(fnbuf, TDSRCPATH);
    177     strcat(fnbuf, filename);
    178 
    179     f = fopen(fnbuf, "rb");
    180 
    181     if(f == NULL) {
    182         log_data_err("FAIL: Could not load %s [%s]\n", fnbuf, filename);
    183     }
    184     return f;
    185 }
    186 
    187 typedef struct TestText {
    188   char *text, *textLimit;
    189   char *limit;
    190   int32_t number;
    191 } TestText;
    192 
    193 static void
    194 text_reset(TestText *tt) {
    195   tt->limit = tt->text;
    196   tt->number = 0;
    197 }
    198 
    199 static char *
    200 text_nextString(TestText *tt, int32_t *pLength) {
    201   char *s = tt->limit;
    202   if (s == tt->textLimit) {
    203     /* we already delivered the last string */
    204     return NULL;
    205   } else if (s == tt->text) {
    206     /* first string */
    207     if ((tt->textLimit - tt->text) >= 3 &&
    208         s[0] == (char)0xef && s[1] == (char)0xbb && s[2] == (char)0xbf
    209     ) {
    210       s += 3;  /* skip the UTF-8 signature byte sequence (U+FEFF) */
    211     }
    212   } else {
    213     /* skip the string terminator */
    214     ++s;
    215     ++tt->number;
    216   }
    217 
    218   /* find the end of this string */
    219   tt->limit = uprv_strchr(s, 0);
    220   *pLength = (int32_t)(tt->limit - s);
    221   return s;
    222 }
    223 
    224 static UBool
    225 text_open(TestText *tt) {
    226   FILE *f;
    227   char *s;
    228   int32_t length;
    229   uprv_memset(tt, 0, sizeof(TestText));
    230   f = fopenOrError("ConverterSelectorTestUTF8.txt");
    231   if(!f) {
    232     return FALSE;
    233   }
    234   fseek(f, 0, SEEK_END);
    235   length = (int32_t)ftell(f);
    236   fseek(f, 0, SEEK_SET);
    237   tt->text = (char *)uprv_malloc(length + 1);
    238   if (tt->text == NULL) {
    239     fclose(f);
    240     return FALSE;
    241   }
    242   if (length != fread(tt->text, 1, length, f)) {
    243     log_err("error reading %ld bytes from test text file\n", (long)length);
    244     length = 0;
    245     uprv_free(tt->text);
    246   }
    247   fclose(f);
    248   tt->textLimit = tt->text + length;
    249   *tt->textLimit = 0;
    250   /* replace all Unicode '#' (U+0023) with NUL */
    251   for(s = tt->text; (s = uprv_strchr(s, 0x23)) != NULL; *s++ = 0) {}
    252   text_reset(tt);
    253   return TRUE;
    254 }
    255 
    256 static void
    257 text_close(TestText *tt) {
    258   uprv_free(tt->text);
    259 }
    260 
    261 static int32_t findIndex(const char* converterName) {
    262   int32_t i;
    263   for (i = 0 ; i < gCountAvailable; i++) {
    264     if(ucnv_compareNames(gAvailableNames[i], converterName) == 0) {
    265       return i;
    266     }
    267   }
    268   return -1;
    269 }
    270 
    271 static UBool *
    272 getResultsManually(const char** encodings, int32_t num_encodings,
    273                    const char *utf8, int32_t length,
    274                    const USet* excludedCodePoints, const UConverterUnicodeSet whichSet) {
    275   UBool* resultsManually;
    276   int32_t i;
    277 
    278   resultsManually = (UBool*) uprv_malloc(gCountAvailable);
    279   uprv_memset(resultsManually, 0, gCountAvailable);
    280 
    281   for(i = 0 ; i < num_encodings ; i++) {
    282     UErrorCode status = U_ZERO_ERROR;
    283     /* get unicode set for that converter */
    284     USet* set;
    285     UConverter* test_converter;
    286     UChar32 cp;
    287     int32_t encIndex, offset;
    288 
    289     set = uset_openEmpty();
    290     test_converter = ucnv_open(encodings[i], &status);
    291     ucnv_getUnicodeSet(test_converter, set,
    292                        whichSet, &status);
    293     if (excludedCodePoints != NULL) {
    294       uset_addAll(set, excludedCodePoints);
    295     }
    296     uset_freeze(set);
    297     offset = 0;
    298     cp = 0;
    299 
    300     encIndex = findIndex(encodings[i]);
    301     /*
    302      * The following is almost, but not entirely, the same as
    303      * resultsManually[encIndex] =
    304      *   (UBool)(uset_spanUTF8(set, utf8, length, USET_SPAN_SIMPLE) == length);
    305      * They might be different if the set contains strings,
    306      * or if the utf8 string contains an illegal sequence.
    307      *
    308      * The UConverterSelector does not currently handle strings that can be
    309      * converted, and it treats an illegal sequence as convertible
    310      * while uset_spanUTF8() treats it like U+FFFD which may not be convertible.
    311      */
    312     resultsManually[encIndex] = TRUE;
    313     while(offset<length) {
    314       U8_NEXT(utf8, offset, length, cp);
    315       if (cp >= 0 && !uset_contains(set, cp)) {
    316         resultsManually[encIndex] = FALSE;
    317         break;
    318       }
    319     }
    320     uset_close(set);
    321     ucnv_close(test_converter);
    322   }
    323   return resultsManually;
    324 }
    325 
    326 /* closes res but does not free resultsManually */
    327 static void verifyResult(UEnumeration* res, const UBool *resultsManually) {
    328   UBool* resultsFromSystem = (UBool*) uprv_malloc(gCountAvailable * sizeof(UBool));
    329   const char* name;
    330   UErrorCode status = U_ZERO_ERROR;
    331   int32_t i;
    332 
    333   /* fill the bool for the selector results! */
    334   uprv_memset(resultsFromSystem, 0, gCountAvailable);
    335   while ((name = uenum_next(res,NULL, &status)) != NULL) {
    336     resultsFromSystem[findIndex(name)] = TRUE;
    337   }
    338   for(i = 0 ; i < gCountAvailable; i++) {
    339     if(resultsManually[i] != resultsFromSystem[i]) {
    340       log_err("failure in converter selector\n"
    341               "converter %s had conflicting results -- manual: %d, system %d\n",
    342               gAvailableNames[i], resultsManually[i], resultsFromSystem[i]);
    343     }
    344   }
    345   uprv_free(resultsFromSystem);
    346   uenum_close(res);
    347 }
    348 
    349 static UConverterSelector *
    350 serializeAndUnserialize(UConverterSelector *sel, char **buffer, UErrorCode *status) {
    351   char *new_buffer;
    352   int32_t ser_len, ser_len2;
    353   /* preflight */
    354   ser_len = ucnvsel_serialize(sel, NULL, 0, status);
    355   if (*status != U_BUFFER_OVERFLOW_ERROR) {
    356     log_err("ucnvsel_serialize(preflighting) failed: %s\n", u_errorName(*status));
    357     return sel;
    358   }
    359   new_buffer = (char *)uprv_malloc(ser_len);
    360   *status = U_ZERO_ERROR;
    361   ser_len2 = ucnvsel_serialize(sel, new_buffer, ser_len, status);
    362   if (U_FAILURE(*status) || ser_len != ser_len2) {
    363     log_err("ucnvsel_serialize() failed: %s\n", u_errorName(*status));
    364     uprv_free(new_buffer);
    365     return sel;
    366   }
    367   ucnvsel_close(sel);
    368   uprv_free(*buffer);
    369   *buffer = new_buffer;
    370   sel = ucnvsel_openFromSerialized(new_buffer, ser_len, status);
    371   if (U_FAILURE(*status)) {
    372     log_err("ucnvsel_openFromSerialized() failed: %s\n", u_errorName(*status));
    373     return NULL;
    374   }
    375   return sel;
    376 }
    377 
    378 static void TestSelector()
    379 {
    380   TestText text;
    381   USet* excluded_sets[3] = { NULL };
    382   int32_t i, testCaseIdx;
    383 
    384   if (!getAvailableNames()) {
    385     return;
    386   }
    387   if (!text_open(&text)) {
    388     releaseAvailableNames();;
    389   }
    390 
    391   excluded_sets[0] = uset_openEmpty();
    392   for(i = 1 ; i < 3 ; i++) {
    393     excluded_sets[i] = uset_open(i*30, i*30+500);
    394   }
    395 
    396   for(testCaseIdx = 0; testCaseIdx < LENGTHOF(getEncodingsFns); testCaseIdx++)
    397   {
    398     int32_t excluded_set_id;
    399     int32_t num_encodings;
    400     const char **encodings = getEncodingsFns[testCaseIdx](&num_encodings);
    401     if (getTestOption(QUICK_OPTION) && num_encodings > 25) {
    402       uprv_free((void *)encodings);
    403       continue;
    404     }
    405 
    406     /*
    407      * for(excluded_set_id = 0 ; excluded_set_id < 3 ; excluded_set_id++)
    408      *
    409      * This loop was replaced by the following statement because
    410      * the loop made the test run longer without adding to the code coverage.
    411      * The handling of the exclusion set is independent of the
    412      * set of encodings, so there is no need to test every combination.
    413      */
    414     excluded_set_id = testCaseIdx % LENGTHOF(excluded_sets);
    415     {
    416       UConverterSelector *sel_rt, *sel_fb;
    417       char *buffer_fb = NULL;
    418       UErrorCode status = U_ZERO_ERROR;
    419       sel_rt = ucnvsel_open(encodings, num_encodings,
    420                             excluded_sets[excluded_set_id],
    421                             UCNV_ROUNDTRIP_SET, &status);
    422       if (num_encodings == gCountAvailable) {
    423         /* test the special "all converters" parameter values */
    424         sel_fb = ucnvsel_open(NULL, 0,
    425                               excluded_sets[excluded_set_id],
    426                               UCNV_ROUNDTRIP_AND_FALLBACK_SET, &status);
    427       } else if (uset_isEmpty(excluded_sets[excluded_set_id])) {
    428         /* test that a NULL set gives the same results as an empty set */
    429         sel_fb = ucnvsel_open(encodings, num_encodings,
    430                               NULL,
    431                               UCNV_ROUNDTRIP_AND_FALLBACK_SET, &status);
    432       } else {
    433         sel_fb = ucnvsel_open(encodings, num_encodings,
    434                               excluded_sets[excluded_set_id],
    435                               UCNV_ROUNDTRIP_AND_FALLBACK_SET, &status);
    436       }
    437       if (U_FAILURE(status)) {
    438         log_err("ucnv_sel_open(encodings %ld) failed - %s\n", testCaseIdx, u_errorName(status));
    439         ucnvsel_close(sel_rt);
    440         uprv_free((void *)encodings);
    441         continue;
    442       }
    443 
    444       text_reset(&text);
    445       for (;;) {
    446         UBool *manual_rt, *manual_fb;
    447         static UChar utf16[10000];
    448         char *s;
    449         int32_t length8, length16;
    450 
    451         s = text_nextString(&text, &length8);
    452         if (s == NULL || (getTestOption(QUICK_OPTION) && text.number > 3)) {
    453           break;
    454         }
    455 
    456         manual_rt = getResultsManually(encodings, num_encodings,
    457                                        s, length8,
    458                                        excluded_sets[excluded_set_id],
    459                                        UCNV_ROUNDTRIP_SET);
    460         manual_fb = getResultsManually(encodings, num_encodings,
    461                                        s, length8,
    462                                        excluded_sets[excluded_set_id],
    463                                        UCNV_ROUNDTRIP_AND_FALLBACK_SET);
    464         /* UTF-8 with length */
    465         status = U_ZERO_ERROR;
    466         verifyResult(ucnvsel_selectForUTF8(sel_rt, s, length8, &status), manual_rt);
    467         verifyResult(ucnvsel_selectForUTF8(sel_fb, s, length8, &status), manual_fb);
    468         /* UTF-8 NUL-terminated */
    469         verifyResult(ucnvsel_selectForUTF8(sel_rt, s, -1, &status), manual_rt);
    470         verifyResult(ucnvsel_selectForUTF8(sel_fb, s, -1, &status), manual_fb);
    471 
    472         u_strFromUTF8(utf16, LENGTHOF(utf16), &length16, s, length8, &status);
    473         if (U_FAILURE(status)) {
    474           log_err("error converting the test text (string %ld) to UTF-16 - %s\n",
    475                   (long)text.number, u_errorName(status));
    476         } else {
    477           if (text.number == 0) {
    478             sel_fb = serializeAndUnserialize(sel_fb, &buffer_fb, &status);
    479           }
    480           if (U_SUCCESS(status)) {
    481             /* UTF-16 with length */
    482             verifyResult(ucnvsel_selectForString(sel_rt, utf16, length16, &status), manual_rt);
    483             verifyResult(ucnvsel_selectForString(sel_fb, utf16, length16, &status), manual_fb);
    484             /* UTF-16 NUL-terminated */
    485             verifyResult(ucnvsel_selectForString(sel_rt, utf16, -1, &status), manual_rt);
    486             verifyResult(ucnvsel_selectForString(sel_fb, utf16, -1, &status), manual_fb);
    487           }
    488         }
    489 
    490         uprv_free(manual_rt);
    491         uprv_free(manual_fb);
    492       }
    493       ucnvsel_close(sel_rt);
    494       ucnvsel_close(sel_fb);
    495       uprv_free(buffer_fb);
    496     }
    497     uprv_free((void *)encodings);
    498   }
    499 
    500   releaseAvailableNames();
    501   text_close(&text);
    502   for(i = 0 ; i < 3 ; i++) {
    503     uset_close(excluded_sets[i]);
    504   }
    505 }
    506 
    507 /* Improve code coverage of UPropsVectors */
    508 static void TestUPropsVector() {
    509     UErrorCode errorCode = U_ILLEGAL_ARGUMENT_ERROR;
    510     UPropsVectors *pv = upvec_open(100, &errorCode);
    511     if (pv != NULL) {
    512         log_err("Should have returned NULL if UErrorCode is an error.");
    513         return;
    514     }
    515     errorCode = U_ZERO_ERROR;
    516     pv = upvec_open(-1, &errorCode);
    517     if (pv != NULL || U_SUCCESS(errorCode)) {
    518         log_err("Should have returned NULL if column is less than 0.\n");
    519         return;
    520     }
    521     errorCode = U_ZERO_ERROR;
    522     pv = upvec_open(100, &errorCode);
    523     if (pv == NULL || U_FAILURE(errorCode)) {
    524         log_err("Unable to open UPropsVectors.\n");
    525         return;
    526     }
    527 
    528     if (upvec_getValue(pv, 0, 1) != 0) {
    529         log_err("upvec_getValue should return 0.\n");
    530     }
    531     if (upvec_getRow(pv, 0, NULL, NULL) == NULL) {
    532         log_err("upvec_getRow should not return NULL.\n");
    533     }
    534     if (upvec_getArray(pv, NULL, NULL) != NULL) {
    535         log_err("upvec_getArray should return NULL.\n");
    536     }
    537 
    538     upvec_close(pv);
    539 }
    540