1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 2009-2012, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * file name: gennorm2.cpp 9 * encoding: US-ASCII 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 2009nov25 14 * created by: Markus W. Scherer 15 * 16 * This program reads text files that define Unicode normalization, 17 * parses them, and builds a binary data file. 18 */ 19 20 #include "unicode/utypes.h" 21 #include "n2builder.h" 22 23 #include <stdio.h> 24 #include <stdlib.h> 25 #include <string.h> 26 #include "unicode/errorcode.h" 27 #include "unicode/localpointer.h" 28 #include "unicode/putil.h" 29 #include "unicode/uchar.h" 30 #include "unicode/unistr.h" 31 #include "charstr.h" 32 #include "normalizer2impl.h" 33 #include "toolutil.h" 34 #include "uoptions.h" 35 #include "uparse.h" 36 37 #if UCONFIG_NO_NORMALIZATION 38 #include "unewdata.h" 39 #endif 40 41 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) 42 43 U_NAMESPACE_BEGIN 44 45 UBool beVerbose=FALSE, haveCopyright=TRUE; 46 47 U_DEFINE_LOCAL_OPEN_POINTER(LocalStdioFilePointer, FILE, fclose); 48 49 #if !UCONFIG_NO_NORMALIZATION 50 void parseFile(FILE *f, Normalizer2DataBuilder &builder); 51 #endif 52 53 /* -------------------------------------------------------------------------- */ 54 55 enum { 56 HELP_H, 57 HELP_QUESTION_MARK, 58 VERBOSE, 59 COPYRIGHT, 60 SOURCEDIR, 61 OUTPUT_FILENAME, 62 UNICODE_VERSION, 63 OPT_FAST 64 }; 65 66 static UOption options[]={ 67 UOPTION_HELP_H, 68 UOPTION_HELP_QUESTION_MARK, 69 UOPTION_VERBOSE, 70 UOPTION_COPYRIGHT, 71 UOPTION_SOURCEDIR, 72 UOPTION_DEF("output", 'o', UOPT_REQUIRES_ARG), 73 UOPTION_DEF("unicode", 'u', UOPT_REQUIRES_ARG), 74 UOPTION_DEF("fast", '\1', UOPT_NO_ARG) 75 }; 76 77 extern "C" int 78 main(int argc, char* argv[]) { 79 U_MAIN_INIT_ARGS(argc, argv); 80 81 /* preset then read command line options */ 82 options[SOURCEDIR].value=""; 83 argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[HELP_H]), options); 84 85 /* error handling, printing usage message */ 86 if(argc<0) { 87 fprintf(stderr, 88 "error in command line argument \"%s\"\n", 89 argv[-argc]); 90 } 91 if(!options[OUTPUT_FILENAME].doesOccur) { 92 argc=-1; 93 } 94 if( argc<2 || 95 options[HELP_H].doesOccur || options[HELP_QUESTION_MARK].doesOccur 96 ) { 97 /* 98 * Broken into chunks because the C89 standard says the minimum 99 * required supported string length is 509 bytes. 100 */ 101 fprintf(stderr, 102 "Usage: %s [-options] infiles+ -o outputfilename\n" 103 "\n" 104 "Reads the infiles with normalization data and\n" 105 "creates a binary file (outputfilename) with the data.\n" 106 "\n", 107 argv[0]); 108 fprintf(stderr, 109 "Options:\n" 110 "\t-h or -? or --help this usage text\n" 111 "\t-v or --verbose verbose output\n" 112 "\t-c or --copyright include a copyright notice\n" 113 "\t-u or --unicode Unicode version, followed by the version like 5.2.0\n"); 114 fprintf(stderr, 115 "\t-s or --sourcedir source directory, followed by the path\n" 116 "\t-o or --output output filename\n"); 117 fprintf(stderr, 118 "\t --fast optimize the .nrm file for fast normalization,\n" 119 "\t which might increase its size (Writes fully decomposed\n" 120 "\t regular mappings instead of delta mappings.\n" 121 "\t You should measure the runtime speed to make sure that\n" 122 "\t this is a good trade-off.)\n"); 123 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; 124 } 125 126 beVerbose=options[VERBOSE].doesOccur; 127 haveCopyright=options[COPYRIGHT].doesOccur; 128 129 IcuToolErrorCode errorCode("gennorm2/main()"); 130 131 #if UCONFIG_NO_NORMALIZATION 132 133 fprintf(stderr, 134 "gennorm2 writes a dummy binary data file " 135 "because UCONFIG_NO_NORMALIZATION is set, \n" 136 "see icu/source/common/unicode/uconfig.h\n"); 137 udata_createDummy(NULL, NULL, options[OUTPUT_FILENAME].value, errorCode); 138 // Should not return an error since this is the expected behaviour if UCONFIG_NO_NORMALIZATION is on. 139 // return U_UNSUPPORTED_ERROR; 140 return 0; 141 142 #else 143 144 LocalPointer<Normalizer2DataBuilder> builder(new Normalizer2DataBuilder(errorCode)); 145 errorCode.assertSuccess(); 146 147 if(options[UNICODE_VERSION].doesOccur) { 148 builder->setUnicodeVersion(options[UNICODE_VERSION].value); 149 } 150 151 if(options[OPT_FAST].doesOccur) { 152 builder->setOptimization(Normalizer2DataBuilder::OPTIMIZE_FAST); 153 } 154 155 // prepare the filename beginning with the source dir 156 CharString filename(options[SOURCEDIR].value, errorCode); 157 int32_t pathLength=filename.length(); 158 if( pathLength>0 && 159 filename[pathLength-1]!=U_FILE_SEP_CHAR && 160 filename[pathLength-1]!=U_FILE_ALT_SEP_CHAR 161 ) { 162 filename.append(U_FILE_SEP_CHAR, errorCode); 163 pathLength=filename.length(); 164 } 165 166 for(int i=1; i<argc; ++i) { 167 printf("gennorm2: processing %s\n", argv[i]); 168 filename.append(argv[i], errorCode); 169 LocalStdioFilePointer f(fopen(filename.data(), "r")); 170 if(f==NULL) { 171 fprintf(stderr, "gennorm2 error: unable to open %s\n", filename.data()); 172 exit(U_FILE_ACCESS_ERROR); 173 } 174 builder->setOverrideHandling(Normalizer2DataBuilder::OVERRIDE_PREVIOUS); 175 parseFile(f.getAlias(), *builder); 176 filename.truncate(pathLength); 177 } 178 179 builder->writeBinaryFile(options[OUTPUT_FILENAME].value); 180 181 return errorCode.get(); 182 183 #endif 184 } 185 186 #if !UCONFIG_NO_NORMALIZATION 187 188 void parseFile(FILE *f, Normalizer2DataBuilder &builder) { 189 IcuToolErrorCode errorCode("gennorm2/parseFile()"); 190 char line[300]; 191 uint32_t startCP, endCP; 192 while(NULL!=fgets(line, (int)sizeof(line), f)) { 193 char *comment=(char *)strchr(line, '#'); 194 if(comment!=NULL) { 195 *comment=0; 196 } 197 u_rtrim(line); 198 if(line[0]==0) { 199 continue; // skip empty and comment-only lines 200 } 201 if(line[0]=='*') { 202 const char *s=u_skipWhitespace(line+1); 203 if(0==strncmp(s, "Unicode", 7)) { 204 s=u_skipWhitespace(s+7); 205 builder.setUnicodeVersion(s); 206 } 207 continue; // reserved syntax 208 } 209 const char *delimiter; 210 int32_t rangeLength= 211 u_parseCodePointRangeAnyTerminator(line, &startCP, &endCP, &delimiter, errorCode); 212 if(errorCode.isFailure()) { 213 fprintf(stderr, "gennorm2 error: parsing code point range from %s\n", line); 214 exit(errorCode.reset()); 215 } 216 delimiter=u_skipWhitespace(delimiter); 217 if(*delimiter==':') { 218 const char *s=u_skipWhitespace(delimiter+1); 219 char *end; 220 unsigned long value=strtoul(s, &end, 10); 221 if(end<=s || *u_skipWhitespace(end)!=0 || value>=0xff) { 222 fprintf(stderr, "gennorm2 error: parsing ccc from %s\n", line); 223 exit(U_PARSE_ERROR); 224 } 225 for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) { 226 builder.setCC(c, (uint8_t)value); 227 } 228 continue; 229 } 230 if(*delimiter=='-') { 231 if(*u_skipWhitespace(delimiter+1)!=0) { 232 fprintf(stderr, "gennorm2 error: parsing remove-mapping %s\n", line); 233 exit(U_PARSE_ERROR); 234 } 235 for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) { 236 builder.removeMapping(c); 237 } 238 continue; 239 } 240 if(*delimiter=='=' || *delimiter=='>') { 241 UChar uchars[Normalizer2Impl::MAPPING_LENGTH_MASK]; 242 int32_t length=u_parseString(delimiter+1, uchars, LENGTHOF(uchars), NULL, errorCode); 243 if(errorCode.isFailure()) { 244 fprintf(stderr, "gennorm2 error: parsing mapping string from %s\n", line); 245 exit(errorCode.reset()); 246 } 247 UnicodeString mapping(FALSE, uchars, length); 248 if(*delimiter=='=') { 249 if(rangeLength!=1) { 250 fprintf(stderr, 251 "gennorm2 error: round-trip mapping for more than 1 code point on %s\n", 252 line); 253 exit(U_PARSE_ERROR); 254 } 255 builder.setRoundTripMapping((UChar32)startCP, mapping); 256 } else { 257 for(UChar32 c=(UChar32)startCP; c<=(UChar32)endCP; ++c) { 258 builder.setOneWayMapping(c, mapping); 259 } 260 } 261 continue; 262 } 263 fprintf(stderr, "gennorm2 error: unrecognized data line %s\n", line); 264 exit(U_PARSE_ERROR); 265 } 266 } 267 268 #endif // !UCONFIG_NO_NORMALIZATION 269 270 U_NAMESPACE_END 271 272 /* 273 * Hey, Emacs, please set the following: 274 * 275 * Local Variables: 276 * indent-tabs-mode: nil 277 * End: 278 * 279 */ 280
