1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 2003-2012, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * file name: gensprep.c 9 * encoding: US-ASCII 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 2003-02-06 14 * created by: Ram Viswanadha 15 * 16 * This program reads the Profile.txt files, 17 * parses them, and extracts the data for StringPrep profile. 18 * It then preprocesses it and writes a binary file for efficient use 19 * in various StringPrep conversion processes. 20 */ 21 22 #define USPREP_TYPE_NAMES_ARRAY 1 23 24 #include <stdio.h> 25 #include <stdlib.h> 26 27 #include "cmemory.h" 28 #include "cstring.h" 29 #include "unewdata.h" 30 #include "uoptions.h" 31 #include "uparse.h" 32 #include "sprpimpl.h" 33 34 #include "unicode/uclean.h" 35 #include "unicode/udata.h" 36 #include "unicode/utypes.h" 37 #include "unicode/putil.h" 38 39 40 U_CDECL_BEGIN 41 #include "gensprep.h" 42 U_CDECL_END 43 44 UBool beVerbose=FALSE, haveCopyright=TRUE; 45 46 #define NORM_CORRECTIONS_FILE_NAME "NormalizationCorrections.txt" 47 48 #define NORMALIZE_DIRECTIVE "normalize" 49 #define NORMALIZE_DIRECTIVE_LEN 9 50 #define CHECK_BIDI_DIRECTIVE "check-bidi" 51 #define CHECK_BIDI_DIRECTIVE_LEN 10 52 53 /* prototypes --------------------------------------------------------------- */ 54 55 static void 56 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode); 57 58 static void 59 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode); 60 61 62 /* -------------------------------------------------------------------------- */ 63 64 static UOption options[]={ 65 UOPTION_HELP_H, 66 UOPTION_HELP_QUESTION_MARK, 67 UOPTION_VERBOSE, 68 UOPTION_COPYRIGHT, 69 UOPTION_DESTDIR, 70 UOPTION_SOURCEDIR, 71 UOPTION_ICUDATADIR, 72 UOPTION_BUNDLE_NAME, 73 { "normalization", NULL, NULL, NULL, 'n', UOPT_REQUIRES_ARG, 0 }, 74 { "norm-correction", NULL, NULL, NULL, 'm', UOPT_REQUIRES_ARG, 0 }, 75 { "check-bidi", NULL, NULL, NULL, 'k', UOPT_NO_ARG, 0}, 76 { "unicode", NULL, NULL, NULL, 'u', UOPT_REQUIRES_ARG, 0 }, 77 }; 78 79 enum{ 80 HELP, 81 HELP_QUESTION_MARK, 82 VERBOSE, 83 COPYRIGHT, 84 DESTDIR, 85 SOURCEDIR, 86 ICUDATADIR, 87 BUNDLE_NAME, 88 NORMALIZE, 89 NORM_CORRECTION_DIR, 90 CHECK_BIDI, 91 UNICODE_VERSION 92 }; 93 94 static int printHelp(int argc, char* argv[]){ 95 /* 96 * Broken into chucks because the C89 standard says the minimum 97 * required supported string length is 509 bytes. 98 */ 99 fprintf(stderr, 100 "Usage: %s [-options] [file_name]\n" 101 "\n" 102 "Read the files specified and\n" 103 "create a binary file [package-name]_[bundle-name]." DATA_TYPE " with the StringPrep profile data\n" 104 "\n", 105 argv[0]); 106 fprintf(stderr, 107 "Options:\n" 108 "\t-h or -? or --help print this usage text\n" 109 "\t-v or --verbose verbose output\n" 110 "\t-c or --copyright include a copyright notice\n"); 111 fprintf(stderr, 112 "\t-d or --destdir destination directory, followed by the path\n" 113 "\t-s or --sourcedir source directory of ICU data, followed by the path\n" 114 "\t-b or --bundle-name generate the ouput data file with the name specified\n" 115 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" 116 "\t followed by path, defaults to %s\n", 117 u_getDataDirectory()); 118 fprintf(stderr, 119 "\t-n or --normalize turn on the option for normalization and include mappings\n" 120 "\t from NormalizationCorrections.txt from the given path,\n" 121 "\t e.g: /test/icu/source/data/unidata\n"); 122 fprintf(stderr, 123 "\t-m or --norm-correction use NormalizationCorrections.txt from the given path\n" 124 "\t when the input file contains a normalization directive.\n" 125 "\t unlike -n/--normalize, this option does not force the\n" 126 "\t normalization.\n"); 127 fprintf(stderr, 128 "\t-k or --check-bidi turn on the option for checking for BiDi in the profile\n" 129 "\t-u or --unicode version of Unicode to be used with this profile followed by the version\n" 130 ); 131 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; 132 } 133 134 135 extern int 136 main(int argc, char* argv[]) { 137 #if !UCONFIG_NO_IDNA 138 char* filename = NULL; 139 #endif 140 const char *srcDir=NULL, *destDir=NULL, *icuUniDataDir=NULL; 141 const char *bundleName=NULL, *inputFileName = NULL; 142 char *basename=NULL; 143 int32_t sprepOptions = 0; 144 145 UErrorCode errorCode=U_ZERO_ERROR; 146 147 U_MAIN_INIT_ARGS(argc, argv); 148 149 /* preset then read command line options */ 150 options[DESTDIR].value=u_getDataDirectory(); 151 options[SOURCEDIR].value=""; 152 options[UNICODE_VERSION].value="0"; /* don't assume the unicode version */ 153 options[BUNDLE_NAME].value = DATA_NAME; 154 options[NORMALIZE].value = ""; 155 156 argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options); 157 158 /* error handling, printing usage message */ 159 if(argc<0) { 160 fprintf(stderr, 161 "error in command line argument \"%s\"\n", 162 argv[-argc]); 163 } 164 if(argc<0 || options[HELP].doesOccur || options[HELP_QUESTION_MARK].doesOccur) { 165 return printHelp(argc, argv); 166 167 } 168 169 /* get the options values */ 170 beVerbose=options[VERBOSE].doesOccur; 171 haveCopyright=options[COPYRIGHT].doesOccur; 172 srcDir=options[SOURCEDIR].value; 173 destDir=options[DESTDIR].value; 174 bundleName = options[BUNDLE_NAME].value; 175 if(options[NORMALIZE].doesOccur) { 176 icuUniDataDir = options[NORMALIZE].value; 177 } else { 178 icuUniDataDir = options[NORM_CORRECTION_DIR].value; 179 } 180 181 if(argc<2) { 182 /* print the help message */ 183 return printHelp(argc, argv); 184 } else { 185 inputFileName = argv[1]; 186 } 187 if(!options[UNICODE_VERSION].doesOccur){ 188 return printHelp(argc, argv); 189 } 190 if(options[ICUDATADIR].doesOccur) { 191 u_setDataDirectory(options[ICUDATADIR].value); 192 } 193 #if UCONFIG_NO_IDNA 194 195 fprintf(stderr, 196 "gensprep writes dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE 197 " because UCONFIG_NO_IDNA is set, \n" 198 "see icu/source/common/unicode/uconfig.h\n"); 199 generateData(destDir, bundleName); 200 201 #else 202 203 setUnicodeVersion(options[UNICODE_VERSION].value); 204 filename = (char* ) uprv_malloc(uprv_strlen(srcDir) + 300); /* hopefully this should be enough */ 205 206 /* prepare the filename beginning with the source dir */ 207 if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL && uprv_strchr(srcDir,U_FILE_ALT_SEP_CHAR) == NULL){ 208 filename[0] = '.'; 209 filename[1] = U_FILE_SEP_CHAR; 210 uprv_strcpy(filename+2,srcDir); 211 }else{ 212 uprv_strcpy(filename, srcDir); 213 } 214 215 basename=filename+uprv_strlen(filename); 216 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { 217 *basename++=U_FILE_SEP_CHAR; 218 } 219 220 /* initialize */ 221 init(); 222 223 /* process the file */ 224 uprv_strcpy(basename,inputFileName); 225 parseMappings(filename,FALSE, &errorCode); 226 if(U_FAILURE(errorCode)) { 227 fprintf(stderr, "Could not open file %s for reading. Error: %s \n", filename, u_errorName(errorCode)); 228 return errorCode; 229 } 230 231 if(options[NORMALIZE].doesOccur){ /* this option might be set by @normalize;; in the source file */ 232 /* set up directory for NormalizationCorrections.txt */ 233 uprv_strcpy(filename,icuUniDataDir); 234 basename=filename+uprv_strlen(filename); 235 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { 236 *basename++=U_FILE_SEP_CHAR; 237 } 238 239 *basename++=U_FILE_SEP_CHAR; 240 uprv_strcpy(basename,NORM_CORRECTIONS_FILE_NAME); 241 242 parseNormalizationCorrections(filename,&errorCode); 243 if(U_FAILURE(errorCode)){ 244 fprintf(stderr,"Could not open file %s for reading \n", filename); 245 return errorCode; 246 } 247 sprepOptions |= _SPREP_NORMALIZATION_ON; 248 } 249 250 if(options[CHECK_BIDI].doesOccur){ /* this option might be set by @check-bidi;; in the source file */ 251 sprepOptions |= _SPREP_CHECK_BIDI_ON; 252 } 253 254 setOptions(sprepOptions); 255 256 /* process parsed data */ 257 if(U_SUCCESS(errorCode)) { 258 /* write the data file */ 259 generateData(destDir, bundleName); 260 261 cleanUpData(); 262 } 263 264 uprv_free(filename); 265 266 u_cleanup(); 267 268 #endif 269 270 return errorCode; 271 } 272 273 #if !UCONFIG_NO_IDNA 274 275 static void U_CALLCONV 276 normalizationCorrectionsLineFn(void *context, 277 char *fields[][2], int32_t fieldCount, 278 UErrorCode *pErrorCode) { 279 uint32_t mapping[40]; 280 char *end, *s; 281 uint32_t code; 282 int32_t length; 283 UVersionInfo version; 284 UVersionInfo thisVersion; 285 286 /* get the character code, field 0 */ 287 code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16); 288 if(U_FAILURE(*pErrorCode)) { 289 fprintf(stderr, "gensprep: error parsing NormalizationCorrections.txt mapping at %s\n", fields[0][0]); 290 exit(*pErrorCode); 291 } 292 /* Original (erroneous) decomposition */ 293 s = fields[1][0]; 294 295 /* parse the mapping string */ 296 length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode); 297 298 /* ignore corrected decomposition */ 299 300 u_versionFromString(version,fields[3][0] ); 301 u_versionFromString(thisVersion, "3.2.0"); 302 303 304 305 if(U_FAILURE(*pErrorCode)) { 306 fprintf(stderr, "gensprep error parsing NormalizationCorrections.txt of U+%04lx - %s\n", 307 (long)code, u_errorName(*pErrorCode)); 308 exit(*pErrorCode); 309 } 310 311 /* store the mapping */ 312 if( version[0] > thisVersion[0] || 313 ((version[0]==thisVersion[0]) && (version[1] > thisVersion[1])) 314 ){ 315 storeMapping(code,mapping, length, USPREP_MAP, pErrorCode); 316 } 317 setUnicodeVersionNC(version); 318 } 319 320 static void 321 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode) { 322 char *fields[4][2]; 323 324 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { 325 return; 326 } 327 328 u_parseDelimitedFile(filename, ';', fields, 4, normalizationCorrectionsLineFn, NULL, pErrorCode); 329 330 /* fprintf(stdout,"Number of code points that have NormalizationCorrections mapping with length >1 : %i\n",len); */ 331 332 if(U_FAILURE(*pErrorCode) && ( *pErrorCode!=U_FILE_ACCESS_ERROR)) { 333 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); 334 exit(*pErrorCode); 335 } 336 } 337 338 static void U_CALLCONV 339 strprepProfileLineFn(void *context, 340 char *fields[][2], int32_t fieldCount, 341 UErrorCode *pErrorCode) { 342 uint32_t mapping[40]; 343 char *end, *map; 344 uint32_t code; 345 int32_t length; 346 /*UBool* mapWithNorm = (UBool*) context;*/ 347 const char* typeName; 348 uint32_t rangeStart=0,rangeEnd =0; 349 const char* filename = (const char*) context; 350 const char *s; 351 352 s = u_skipWhitespace(fields[0][0]); 353 if (*s == '@') { 354 /* special directive */ 355 s++; 356 length = fields[0][1] - s; 357 if (length >= NORMALIZE_DIRECTIVE_LEN 358 && uprv_strncmp(s, NORMALIZE_DIRECTIVE, NORMALIZE_DIRECTIVE_LEN) == 0) { 359 options[NORMALIZE].doesOccur = TRUE; 360 return; 361 } 362 else if (length >= CHECK_BIDI_DIRECTIVE_LEN 363 && uprv_strncmp(s, CHECK_BIDI_DIRECTIVE, CHECK_BIDI_DIRECTIVE_LEN) == 0) { 364 options[CHECK_BIDI].doesOccur = TRUE; 365 return; 366 } 367 else { 368 fprintf(stderr, "gensprep error parsing a directive %s.", fields[0][0]); 369 } 370 } 371 372 typeName = fields[2][0]; 373 map = fields[1][0]; 374 375 if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){ 376 377 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode); 378 if(U_FAILURE(*pErrorCode)){ 379 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode)); 380 return; 381 } 382 383 /* store the range */ 384 storeRange(rangeStart,rangeEnd,USPREP_UNASSIGNED, pErrorCode); 385 386 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){ 387 388 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode); 389 if(U_FAILURE(*pErrorCode)){ 390 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode)); 391 return; 392 } 393 394 /* store the range */ 395 storeRange(rangeStart,rangeEnd,USPREP_PROHIBITED, pErrorCode); 396 397 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){ 398 399 /* get the character code, field 0 */ 400 code=(uint32_t)uprv_strtoul(s, &end, 16); 401 if(end<=s || end!=fields[0][1]) { 402 fprintf(stderr, "gensprep: syntax error in field 0 at %s\n", fields[0][0]); 403 *pErrorCode=U_PARSE_ERROR; 404 exit(U_PARSE_ERROR); 405 } 406 407 /* parse the mapping string */ 408 length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode); 409 410 /* store the mapping */ 411 storeMapping(code,mapping, length,USPREP_MAP, pErrorCode); 412 413 }else{ 414 *pErrorCode = U_INVALID_FORMAT_ERROR; 415 } 416 417 if(U_FAILURE(*pErrorCode)) { 418 fprintf(stderr, "gensprep error parsing %s line %s at %s. Error: %s\n",filename, 419 fields[0][0],fields[2][0],u_errorName(*pErrorCode)); 420 exit(*pErrorCode); 421 } 422 423 } 424 425 static void 426 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode) { 427 char *fields[3][2]; 428 429 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { 430 return; 431 } 432 433 u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode); 434 435 /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/ 436 437 if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) { 438 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); 439 exit(*pErrorCode); 440 } 441 } 442 443 444 #endif /* #if !UCONFIG_NO_IDNA */ 445 446 /* 447 * Hey, Emacs, please set the following: 448 * 449 * Local Variables: 450 * indent-tabs-mode: nil 451 * End: 452 * 453 */ 454
