Home | History | Annotate | Download | only in src
      1 #!/usr/bin/python -u
      2 #
      3 # Original script modified in November 2003 to take advantage of
      4 # the character-validation range routines, and updated to the
      5 # current Unicode information (Version 4.0.1)
      6 #
      7 # NOTE: there is an 'alias' facility for blocks which are not present in
      8 #	the current release, but are needed for ABI compatibility.  This
      9 #	must be accomplished MANUALLY!  Please see the comments below under
     10 #     'blockAliases'
     11 #
     12 import sys
     13 import string
     14 import time
     15 
     16 webpage = "http://www.unicode.org/Public/4.0-Update1/UCD-4.0.1.html"
     17 sources = "Blocks-4.0.1.txt UnicodeData-4.0.1.txt"
     18 
     19 #
     20 # blockAliases is a small hack - it is used for mapping block names which
     21 # were were used in the 3.1 release, but are missing or changed in the current
     22 # release.  The format is "OldBlockName:NewBlockName1[,NewBlockName2[,...]]"
     23 blockAliases = []
     24 blockAliases.append("CombiningMarksforSymbols:CombiningDiacriticalMarksforSymbols")
     25 blockAliases.append("Greek:GreekandCoptic")
     26 blockAliases.append("PrivateUse:PrivateUseArea,SupplementaryPrivateUseArea-A," + 
     27 	"SupplementaryPrivateUseArea-B")
     28 
     29 # minTableSize gives the minimum number of ranges which must be present
     30 # before a range table is produced.  If there are less than this
     31 # number, inline comparisons are generated
     32 minTableSize = 8
     33 
     34 (blockfile, catfile) = string.split(sources)
     35 
     36 
     37 #
     38 # Now process the "blocks" file, reducing it to a dictionary
     39 # indexed by blockname, containing a tuple with the applicable
     40 # block range
     41 #
     42 BlockNames = {}
     43 try:
     44     blocks = open(blockfile, "r")
     45 except:
     46     print "Missing %s, aborting ..." % blockfile
     47     sys.exit(1)
     48 
     49 for line in blocks.readlines():
     50     if line[0] == '#':
     51         continue
     52     line = string.strip(line)
     53     if line == '':
     54         continue
     55     try:
     56         fields = string.split(line, ';')
     57         range = string.strip(fields[0])
     58         (start, end) = string.split(range, "..")
     59         name = string.strip(fields[1])
     60         name = string.replace(name, ' ', '')
     61     except:
     62         print "Failed to process line: %s" % (line)
     63         continue
     64     start = "0x" + start
     65     end = "0x" + end
     66     try:
     67         BlockNames[name].append((start, end))
     68     except:
     69         BlockNames[name] = [(start, end)]
     70 blocks.close()
     71 print "Parsed %d blocks descriptions" % (len(BlockNames.keys()))
     72 
     73 for block in blockAliases:
     74     alias = string.split(block,':')
     75     alist = string.split(alias[1],',')
     76     for comp in alist:
     77         if BlockNames.has_key(comp):
     78             if alias[0] not in BlockNames:
     79                 BlockNames[alias[0]] = []
     80             for r in BlockNames[comp]:
     81                 BlockNames[alias[0]].append(r)
     82         else:
     83             print "Alias %s: %s not in Blocks" % (alias[0], comp)
     84             continue
     85 
     86 #
     87 # Next process the Categories file. This is more complex, since
     88 # the file is in code sequence, and we need to invert it.  We use
     89 # a dictionary with index category-name, with each entry containing
     90 # all the ranges (codepoints) of that category.  Note that category
     91 # names comprise two parts - the general category, and the "subclass"
     92 # within that category.  Therefore, both "general category" (which is
     93 # the first character of the 2-character category-name) and the full
     94 # (2-character) name are entered into this dictionary.
     95 #
     96 try:
     97     data = open(catfile, "r")
     98 except:
     99     print "Missing %s, aborting ..." % catfile
    100     sys.exit(1)
    101 
    102 nbchar = 0;
    103 Categories = {}
    104 for line in data.readlines():
    105     if line[0] == '#':
    106         continue
    107     line = string.strip(line)
    108     if line == '':
    109         continue
    110     try:
    111         fields = string.split(line, ';')
    112         point = string.strip(fields[0])
    113         value = 0
    114         while point != '':
    115             value = value * 16
    116             if point[0] >= '0' and point[0] <= '9':
    117                 value = value + ord(point[0]) - ord('0')
    118             elif point[0] >= 'A' and point[0] <= 'F':
    119                 value = value + 10 + ord(point[0]) - ord('A')
    120             elif point[0] >= 'a' and point[0] <= 'f':
    121                 value = value + 10 + ord(point[0]) - ord('a')
    122             point = point[1:]
    123         name = fields[2]
    124     except:
    125         print "Failed to process line: %s" % (line)
    126         continue
    127     
    128     nbchar = nbchar + 1
    129     # update entry for "full name"
    130     try:
    131         Categories[name].append(value)
    132     except:
    133         try:
    134             Categories[name] = [value]
    135         except:
    136             print "Failed to process line: %s" % (line)
    137     # update "general category" name
    138     try:
    139         Categories[name[0]].append(value)
    140     except:
    141         try:
    142             Categories[name[0]] = [value]
    143         except:
    144             print "Failed to process line: %s" % (line)
    145 
    146 blocks.close()
    147 print "Parsed %d char generating %d categories" % (nbchar, len(Categories.keys()))
    148 
    149 #
    150 # The data is now all read.  Time to process it into a more useful form.
    151 #
    152 # reduce the number list into ranges
    153 for cat in Categories.keys():
    154     list = Categories[cat]
    155     start = -1
    156     prev = -1
    157     end = -1
    158     ranges = []
    159     for val in list:
    160         if start == -1:
    161             start = val
    162             prev = val
    163             continue
    164         elif val == prev + 1:
    165             prev = val
    166             continue
    167         elif prev == start:
    168             ranges.append((prev, prev))
    169             start = val
    170             prev = val
    171             continue
    172         else:
    173             ranges.append((start, prev))
    174             start = val
    175             prev = val
    176             continue
    177     if prev == start:
    178         ranges.append((prev, prev))
    179     else:
    180         ranges.append((start, prev))
    181     Categories[cat] = ranges
    182 
    183 #
    184 # Assure all data is in alphabetic order, since we will be doing binary
    185 # searches on the tables.
    186 #
    187 bkeys = BlockNames.keys()
    188 bkeys.sort()
    189 
    190 ckeys = Categories.keys()
    191 ckeys.sort()
    192 
    193 #
    194 # Generate the resulting files
    195 #
    196 try:
    197     header = open("include/libxml/xmlunicode.h", "w")
    198 except:
    199     print "Failed to open include/libxml/xmlunicode.h"
    200     sys.exit(1)
    201 
    202 try:
    203     output = open("xmlunicode.c", "w")
    204 except:
    205     print "Failed to open xmlunicode.c"
    206     sys.exit(1)
    207 
    208 date = time.asctime(time.localtime(time.time()))
    209 
    210 header.write(
    211 """/*
    212  * Summary: Unicode character APIs
    213  * Description: API for the Unicode character APIs
    214  *
    215  * This file is automatically generated from the
    216  * UCS description files of the Unicode Character Database
    217  * %s
    218  * using the genUnicode.py Python script.
    219  *
    220  * Generation date: %s
    221  * Sources: %s
    222  * Author: Daniel Veillard
    223  */
    224 
    225 #ifndef __XML_UNICODE_H__
    226 #define __XML_UNICODE_H__
    227 
    228 #include <libxml/xmlversion.h>
    229 
    230 #ifdef LIBXML_UNICODE_ENABLED
    231 
    232 #ifdef __cplusplus
    233 extern "C" {
    234 #endif
    235 
    236 """ % (webpage, date, sources));
    237 
    238 output.write(
    239 """/*
    240  * xmlunicode.c: this module implements the Unicode character APIs
    241  *
    242  * This file is automatically generated from the
    243  * UCS description files of the Unicode Character Database
    244  * %s
    245  * using the genUnicode.py Python script.
    246  *
    247  * Generation date: %s
    248  * Sources: %s
    249  * Daniel Veillard <veillard (at] redhat.com>
    250  */
    251 
    252 #define IN_LIBXML
    253 #include "libxml.h"
    254 
    255 #ifdef LIBXML_UNICODE_ENABLED
    256 
    257 #include <string.h>
    258 #include <libxml/xmlversion.h>
    259 #include <libxml/xmlunicode.h>
    260 #include <libxml/chvalid.h>
    261 
    262 typedef int (xmlIntFunc)(int);	/* just to keep one's mind untwisted */
    263 
    264 typedef struct {
    265     const char *rangename;
    266     xmlIntFunc *func;
    267 } xmlUnicodeRange;
    268 
    269 typedef struct {
    270     xmlUnicodeRange *table;
    271     int		    numentries;
    272 } xmlUnicodeNameTable;
    273 
    274 
    275 static xmlIntFunc *xmlUnicodeLookup(xmlUnicodeNameTable *tptr, const char *tname);
    276 
    277 static xmlUnicodeRange xmlUnicodeBlocks[] = {
    278 """ % (webpage, date, sources));
    279 
    280 flag = 0
    281 for block in bkeys:
    282     name = string.replace(block, '-', '')
    283     if flag:
    284         output.write(',\n')
    285     else:
    286         flag = 1
    287     output.write('  {"%s", xmlUCSIs%s}' % (block, name))
    288 output.write('};\n\n')
    289 
    290 output.write('static xmlUnicodeRange xmlUnicodeCats[] = {\n')
    291 flag = 0;
    292 for name in ckeys:
    293     if flag:
    294         output.write(',\n')
    295     else:
    296         flag = 1
    297     output.write('  {"%s", xmlUCSIsCat%s}' % (name, name))
    298 output.write('};\n\n')
    299 
    300 #
    301 # For any categories with more than minTableSize ranges we generate
    302 # a range table suitable for xmlCharInRange
    303 #
    304 for name in ckeys:
    305   if len(Categories[name]) > minTableSize:
    306     numshort = 0
    307     numlong = 0
    308     ranges = Categories[name]
    309     sptr = "NULL"
    310     lptr = "NULL"
    311     for range in ranges:
    312       (low, high) = range
    313       if high < 0x10000:
    314         if numshort == 0:
    315           pline = "static const xmlChSRange xml%sS[] = {" % name
    316           sptr = "xml%sS" % name
    317         else:
    318           pline += ", "
    319         numshort += 1
    320       else:
    321         if numlong == 0:
    322           if numshort > 0:
    323             output.write(pline + " };\n")
    324           pline = "static const xmlChLRange xml%sL[] = {" % name
    325           lptr = "xml%sL" % name
    326         else:
    327           pline += ", "
    328         numlong += 1
    329       if len(pline) > 60:
    330         output.write(pline + "\n")
    331         pline = "    "
    332       pline += "{%s, %s}" % (hex(low), hex(high))
    333     output.write(pline + " };\nstatic xmlChRangeGroup xml%sG = {%s,%s,%s,%s};\n\n"
    334          % (name, numshort, numlong, sptr, lptr))
    335 
    336 
    337 output.write(
    338 """static xmlUnicodeNameTable xmlUnicodeBlockTbl = {xmlUnicodeBlocks, %s};
    339 static xmlUnicodeNameTable xmlUnicodeCatTbl = {xmlUnicodeCats, %s};
    340 
    341 /**
    342  * xmlUnicodeLookup:
    343  * @tptr: pointer to the name table
    344  * @name: name to be found
    345  *
    346  * binary table lookup for user-supplied name
    347  *
    348  * Returns pointer to range function if found, otherwise NULL
    349  */
    350 static xmlIntFunc
    351 *xmlUnicodeLookup(xmlUnicodeNameTable *tptr, const char *tname) {
    352     int low, high, mid, cmp;
    353     xmlUnicodeRange *sptr;
    354 
    355     if ((tptr == NULL) || (tname == NULL)) return(NULL);
    356 
    357     low = 0;
    358     high = tptr->numentries - 1;
    359     sptr = tptr->table;
    360     while (low <= high) {
    361 	mid = (low + high) / 2;
    362 	if ((cmp=strcmp(tname, sptr[mid].rangename)) == 0)
    363 	    return (sptr[mid].func);
    364 	if (cmp < 0)
    365 	    high = mid - 1;
    366 	else
    367 	    low = mid + 1;
    368     }
    369     return (NULL);    
    370 }
    371 
    372 """ % (len(BlockNames), len(Categories)) )
    373 
    374 for block in bkeys:
    375     name = string.replace(block, '-', '')
    376     header.write("XMLPUBFUN int XMLCALL xmlUCSIs%s\t(int code);\n" % name)
    377     output.write("/**\n * xmlUCSIs%s:\n * @code: UCS code point\n" % (name))
    378     output.write(" *\n * Check whether the character is part of %s UCS Block\n"%
    379                  (block))
    380     output.write(" *\n * Returns 1 if true 0 otherwise\n */\n");
    381     output.write("int\nxmlUCSIs%s(int code) {\n    return(" % name)
    382     flag = 0
    383     for (start, end) in BlockNames[block]:
    384         if flag:
    385             output.write(" ||\n           ")
    386         else:
    387             flag = 1
    388         output.write("((code >= %s) && (code <= %s))" % (start, end))
    389     output.write(");\n}\n\n")
    390 
    391 header.write("\nXMLPUBFUN int XMLCALL xmlUCSIsBlock\t(int code, const char *block);\n\n")
    392 output.write(
    393 """/**
    394  * xmlUCSIsBlock:
    395  * @code: UCS code point
    396  * @block: UCS block name
    397  *
    398  * Check whether the character is part of the UCS Block
    399  *
    400  * Returns 1 if true, 0 if false and -1 on unknown block
    401  */
    402 int
    403 xmlUCSIsBlock(int code, const char *block) {
    404     xmlIntFunc *func;
    405 
    406     func = xmlUnicodeLookup(&xmlUnicodeBlockTbl, block);
    407     if (func == NULL)
    408 	return (-1);
    409     return (func(code));
    410 }
    411 
    412 """)
    413 
    414 for name in ckeys:
    415     ranges = Categories[name]
    416     header.write("XMLPUBFUN int XMLCALL xmlUCSIsCat%s\t(int code);\n" % name)
    417     output.write("/**\n * xmlUCSIsCat%s:\n * @code: UCS code point\n" % (name))
    418     output.write(" *\n * Check whether the character is part of %s UCS Category\n"%
    419                  (name))
    420     output.write(" *\n * Returns 1 if true 0 otherwise\n */\n");
    421     output.write("int\nxmlUCSIsCat%s(int code) {\n" % name)
    422     if len(Categories[name]) > minTableSize:
    423         output.write("    return(xmlCharInRange((unsigned int)code, &xml%sG)"
    424             % name)
    425     else:
    426         start = 1
    427         for range in ranges:
    428             (begin, end) = range;
    429             if start:
    430                 output.write("    return(");
    431                 start = 0
    432             else:
    433                 output.write(" ||\n           ");
    434             if (begin == end):
    435                 output.write("(code == %s)" % (hex(begin)))
    436             else:
    437                 output.write("((code >= %s) && (code <= %s))" % (
    438                          hex(begin), hex(end)))
    439     output.write(");\n}\n\n")
    440 
    441 header.write("\nXMLPUBFUN int XMLCALL xmlUCSIsCat\t(int code, const char *cat);\n")
    442 output.write(
    443 """/**
    444  * xmlUCSIsCat:
    445  * @code: UCS code point
    446  * @cat: UCS Category name
    447  *
    448  * Check whether the character is part of the UCS Category
    449  *
    450  * Returns 1 if true, 0 if false and -1 on unknown category
    451  */
    452 int
    453 xmlUCSIsCat(int code, const char *cat) {
    454     xmlIntFunc *func;
    455 
    456     func = xmlUnicodeLookup(&xmlUnicodeCatTbl, cat);
    457     if (func == NULL)
    458 	return (-1);
    459     return (func(code));
    460 }
    461 
    462 #define bottom_xmlunicode
    463 #include "elfgcchack.h"
    464 #endif /* LIBXML_UNICODE_ENABLED */
    465 """)
    466 
    467 header.write("""
    468 #ifdef __cplusplus
    469 }
    470 #endif
    471 
    472 #endif /* LIBXML_UNICODE_ENABLED */
    473 
    474 #endif /* __XML_UNICODE_H__ */
    475 """);
    476 
    477 header.close()
    478 output.close()
    479