1 // Copyright (C) 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * 6 * Copyright (C) 2003-2012, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************* 10 * file name: spreptst.c 11 * encoding: US-ASCII 12 * tab size: 8 (not used) 13 * indentation:4 14 * 15 * created on: 2003jul11 16 * created by: Ram Viswanadha 17 */ 18 #define USPREP_TYPE_NAMES_ARRAY 19 20 #include "unicode/utypes.h" 21 22 #if !UCONFIG_NO_IDNA 23 24 #include "unicode/ustring.h" 25 #include "unicode/putil.h" 26 #include "cintltst.h" 27 #include "unicode/usprep.h" 28 #include "unicode/utf16.h" 29 #include "sprpimpl.h" 30 #include "uparse.h" 31 #include "cmemory.h" 32 #include "ustr_imp.h" 33 #include "cstring.h" 34 35 static void 36 parseMappings(const char *filename, UStringPrepProfile* data, UBool reportError, UErrorCode *pErrorCode); 37 38 static void 39 compareMapping(UStringPrepProfile* data, uint32_t codepoint, uint32_t* mapping, int32_t mapLength, 40 UStringPrepType option); 41 42 static void 43 compareFlagsForRange(UStringPrepProfile* data, uint32_t start, uint32_t end,UStringPrepType option); 44 45 void 46 doStringPrepTest(const char* binFileName, const char* txtFileName, int32_t options, UErrorCode* errorCode); 47 48 static void U_CALLCONV 49 strprepProfileLineFn(void *context, 50 char *fields[][2], int32_t fieldCount, 51 UErrorCode *pErrorCode) { 52 uint32_t mapping[40]; 53 char *end, *map; 54 uint32_t code; 55 int32_t length; 56 UStringPrepProfile* data = (UStringPrepProfile*) context; 57 const char* typeName; 58 uint32_t rangeStart=0,rangeEnd =0; 59 60 typeName = fields[2][0]; 61 map = fields[1][0]; 62 63 if(strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){ 64 65 u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode); 66 67 /* store the range */ 68 compareFlagsForRange(data, rangeStart,rangeEnd,USPREP_UNASSIGNED); 69 70 }else if(strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){ 71 72 u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode); 73 74 /* store the range */ 75 compareFlagsForRange(data, rangeStart,rangeEnd,USPREP_PROHIBITED); 76 77 }else if(strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){ 78 /* get the character code, field 0 */ 79 code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16); 80 81 /* parse the mapping string */ 82 length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode); 83 84 /* compare the mapping */ 85 compareMapping(data, code,mapping, length,USPREP_MAP); 86 }else{ 87 *pErrorCode = U_INVALID_FORMAT_ERROR; 88 } 89 90 } 91 92 93 94 static void 95 parseMappings(const char *filename, UStringPrepProfile* data, UBool reportError, UErrorCode *pErrorCode) { 96 char *fields[3][2]; 97 98 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { 99 return; 100 } 101 102 u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)data, pErrorCode); 103 104 /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/ 105 106 if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) { 107 log_err( "testidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); 108 } 109 } 110 111 112 static UStringPrepType 113 getValues(uint32_t result, int32_t* value, UBool* isIndex){ 114 115 UStringPrepType type; 116 if(result == 0){ 117 /* 118 * Initial value stored in the mapping table 119 * just return USPREP_TYPE_LIMIT .. so that 120 * the source codepoint is copied to the destination 121 */ 122 type = USPREP_TYPE_LIMIT; 123 }else if(result >= _SPREP_TYPE_THRESHOLD){ 124 type = (UStringPrepType) (result - _SPREP_TYPE_THRESHOLD); 125 }else{ 126 /* get the type */ 127 type = USPREP_MAP; 128 /* ascertain if the value is index or delta */ 129 if(result & 0x02){ 130 *isIndex = TRUE; 131 *value = result >> 2; 132 133 }else{ 134 *isIndex = FALSE; 135 *value = (int16_t)result; 136 *value = (*value >> 2); 137 138 } 139 if((result>>2) == _SPREP_MAX_INDEX_VALUE){ 140 type = USPREP_DELETE; 141 isIndex =FALSE; 142 value = 0; 143 } 144 } 145 return type; 146 } 147 148 static void 149 compareMapping(UStringPrepProfile* data, uint32_t codepoint, uint32_t* mapping,int32_t mapLength, 150 UStringPrepType type){ 151 uint32_t result = 0; 152 int32_t length=0; 153 UBool isIndex = FALSE; 154 UStringPrepType retType; 155 int32_t value=0, idx=0, delta=0; 156 int32_t* indexes = data->indexes; 157 UTrie trie = data->sprepTrie; 158 const uint16_t* mappingData = data->mappingData; 159 int32_t realLength =0; 160 int32_t j=0; 161 int8_t i=0; 162 163 UTRIE_GET16(&trie, codepoint, result); 164 retType = getValues(result,&value,&isIndex); 165 166 167 if(type != retType && retType != USPREP_DELETE){ 168 169 log_err( "Did not get the assigned type for codepoint 0x%08X. Expected: %i Got: %i\n",codepoint, USPREP_MAP, type); 170 171 } 172 173 if(isIndex){ 174 idx = value; 175 if(idx >= indexes[_SPREP_ONE_UCHAR_MAPPING_INDEX_START] && 176 idx < indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START]){ 177 length = 1; 178 }else if(idx >= indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START] && 179 idx < indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START]){ 180 length = 2; 181 }else if(idx >= indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START] && 182 idx < indexes[_SPREP_FOUR_UCHARS_MAPPING_INDEX_START]){ 183 length = 3; 184 }else{ 185 length = mappingData[idx++]; 186 } 187 }else{ 188 delta = value; 189 length = (retType == USPREP_DELETE)? 0 : 1; 190 } 191 192 /* figure out the real length */ 193 for(j=0; j<mapLength; j++){ 194 if(mapping[j] > 0xFFFF){ 195 realLength +=2; 196 }else{ 197 realLength++; 198 } 199 } 200 201 if(realLength != length){ 202 log_err( "Did not get the expected length. Expected: %i Got: %i\n", mapLength, length); 203 } 204 205 if(isIndex){ 206 for(i =0; i< mapLength; i++){ 207 if(mapping[i] <= 0xFFFF){ 208 if(mappingData[idx+i] != (uint16_t)mapping[i]){ 209 log_err("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[i], mappingData[idx+i]); 210 } 211 }else{ 212 UChar lead = U16_LEAD(mapping[i]); 213 UChar trail = U16_TRAIL(mapping[i]); 214 if(mappingData[idx+i] != lead || 215 mappingData[idx+i+1] != trail){ 216 log_err( "Did not get the expected result. Expected: 0x%04X 0x%04X Got: 0x%04X 0x%04X\n", lead, trail, mappingData[idx+i], mappingData[idx+i+1]); 217 } 218 } 219 } 220 }else{ 221 if(retType!=USPREP_DELETE && (codepoint-delta) != (uint16_t)mapping[0]){ 222 log_err("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[0],(codepoint-delta)); 223 } 224 } 225 226 } 227 228 static void 229 compareFlagsForRange(UStringPrepProfile* data, 230 uint32_t start, uint32_t end, 231 UStringPrepType type){ 232 233 uint32_t result =0 ; 234 UStringPrepType retType; 235 UBool isIndex=FALSE; 236 int32_t value=0; 237 UTrie trie = data->sprepTrie; 238 /* 239 // supplementary code point 240 UChar __lead16=U16_LEAD(0x2323E); 241 int32_t __offset; 242 243 // get data for lead surrogate 244 (result)=_UTRIE_GET_RAW((&idnTrie), index, 0, (__lead16)); 245 __offset=(&idnTrie)->getFoldingOffset(result); 246 247 // get the real data from the folded lead/trail units 248 if(__offset>0) { 249 (result)=_UTRIE_GET_RAW((&idnTrie), index, __offset, (0x2323E)&0x3ff); 250 } else { 251 (result)=(uint32_t)((&idnTrie)->initialValue); 252 } 253 254 UTRIE_GET16(&idnTrie,0x2323E, result); 255 */ 256 while(start < end+1){ 257 UTRIE_GET16(&trie,start, result); 258 retType = getValues(result, &value, &isIndex); 259 if(result > _SPREP_TYPE_THRESHOLD){ 260 if(retType != type){ 261 log_err( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]); 262 } 263 }else{ 264 if(type == USPREP_PROHIBITED && ((result & 0x01) != 0x01)){ 265 log_err( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]); 266 } 267 } 268 269 start++; 270 } 271 272 } 273 274 void 275 doStringPrepTest(const char* binFileName, const char* txtFileName, int32_t options, UErrorCode* errorCode){ 276 277 const char *testdatapath = loadTestData(errorCode); 278 const char *srcdatapath = NULL; 279 const char *relativepath = NULL; 280 char *filename = NULL; 281 UStringPrepProfile* profile = NULL; 282 283 #ifdef U_TOPSRCDIR 284 srcdatapath = U_TOPSRCDIR; 285 relativepath = U_FILE_SEP_STRING"test"U_FILE_SEP_STRING"testdata"U_FILE_SEP_STRING; 286 #else 287 srcdatapath = ctest_dataOutDir(); 288 relativepath = ".."U_FILE_SEP_STRING".."U_FILE_SEP_STRING"test"U_FILE_SEP_STRING"testdata"U_FILE_SEP_STRING; 289 #endif 290 291 profile = usprep_open(testdatapath, binFileName, errorCode); 292 293 if(*errorCode == U_FILE_ACCESS_ERROR) { 294 log_data_err("Failed to load %s data file. Error: %s \n", binFileName, u_errorName(*errorCode)); 295 return; 296 } else if(U_FAILURE(*errorCode)){ 297 log_err("Failed to load %s data file. Error: %s \n", binFileName, u_errorName(*errorCode)); 298 return; 299 } 300 filename = (char*) malloc(strlen(srcdatapath)+strlen(relativepath)+strlen(txtFileName)+10 ); 301 /* open and load the txt file */ 302 strcpy(filename,srcdatapath); 303 strcat(filename,relativepath); 304 strcat(filename,txtFileName); 305 306 parseMappings(filename,profile, TRUE,errorCode); 307 308 free(filename); 309 } 310 #endif 311 /* 312 * Hey, Emacs, please set the following: 313 * 314 * Local Variables: 315 * indent-tabs-mode: nil 316 * End: 317 * 318 */ 319