Home | History | Annotate | Download | only in unicode
      1 /*
      2  **********************************************************************
      3  *   Copyright (C) 2005-2013, International Business Machines
      4  *   Corporation and others.  All Rights Reserved.
      5  **********************************************************************
      6  *   file name:  ucsdet.h
      7  *   encoding:   US-ASCII
      8  *   indentation:4
      9  *
     10  *   created on: 2005Aug04
     11  *   created by: Andy Heninger
     12  *
     13  *   ICU Character Set Detection, API for C
     14  *
     15  *   Draft version 18 Oct 2005
     16  *
     17  */
     18 
     19 #ifndef __UCSDET_H
     20 #define __UCSDET_H
     21 
     22 #include "unicode/utypes.h"
     23 
     24 #if !UCONFIG_NO_CONVERSION
     25 
     26 #include "unicode/localpointer.h"
     27 #include "unicode/uenum.h"
     28 
     29 /**
     30  * \file
     31  * \brief C API: Charset Detection API
     32  *
     33  * This API provides a facility for detecting the
     34  * charset or encoding of character data in an unknown text format.
     35  * The input data can be from an array of bytes.
     36  * <p>
     37  * Character set detection is at best an imprecise operation.  The detection
     38  * process will attempt to identify the charset that best matches the characteristics
     39  * of the byte data, but the process is partly statistical in nature, and
     40  * the results can not be guaranteed to always be correct.
     41  * <p>
     42  * For best accuracy in charset detection, the input data should be primarily
     43  * in a single language, and a minimum of a few hundred bytes worth of plain text
     44  * in the language are needed.  The detection process will attempt to
     45  * ignore html or xml style markup that could otherwise obscure the content.
     46  */
     47 
     48 
     49 struct UCharsetDetector;
     50 /**
     51   * Structure representing a charset detector
     52   * @stable ICU 3.6
     53   */
     54 typedef struct UCharsetDetector UCharsetDetector;
     55 
     56 struct UCharsetMatch;
     57 /**
     58   *  Opaque structure representing a match that was identified
     59   *  from a charset detection operation.
     60   *  @stable ICU 3.6
     61   */
     62 typedef struct UCharsetMatch UCharsetMatch;
     63 
     64 /**
     65   *  Open a charset detector.
     66   *
     67   *  @param status Any error conditions occurring during the open
     68   *                operation are reported back in this variable.
     69   *  @return the newly opened charset detector.
     70   *  @stable ICU 3.6
     71   */
     72 U_STABLE UCharsetDetector * U_EXPORT2
     73 ucsdet_open(UErrorCode   *status);
     74 
     75 /**
     76   * Close a charset detector.  All storage and any other resources
     77   *   owned by this charset detector will be released.  Failure to
     78   *   close a charset detector when finished with it can result in
     79   *   memory leaks in the application.
     80   *
     81   *  @param ucsd  The charset detector to be closed.
     82   *  @stable ICU 3.6
     83   */
     84 U_STABLE void U_EXPORT2
     85 ucsdet_close(UCharsetDetector *ucsd);
     86 
     87 #if U_SHOW_CPLUSPLUS_API
     88 
     89 U_NAMESPACE_BEGIN
     90 
     91 /**
     92  * \class LocalUCharsetDetectorPointer
     93  * "Smart pointer" class, closes a UCharsetDetector via ucsdet_close().
     94  * For most methods see the LocalPointerBase base class.
     95  *
     96  * @see LocalPointerBase
     97  * @see LocalPointer
     98  * @stable ICU 4.4
     99  */
    100 U_DEFINE_LOCAL_OPEN_POINTER(LocalUCharsetDetectorPointer, UCharsetDetector, ucsdet_close);
    101 
    102 U_NAMESPACE_END
    103 
    104 #endif
    105 
    106 /**
    107   * Set the input byte data whose charset is to detected.
    108   *
    109   * Ownership of the input  text byte array remains with the caller.
    110   * The input string must not be altered or deleted until the charset
    111   * detector is either closed or reset to refer to different input text.
    112   *
    113   * @param ucsd   the charset detector to be used.
    114   * @param textIn the input text of unknown encoding.   .
    115   * @param len    the length of the input text, or -1 if the text
    116   *               is NUL terminated.
    117   * @param status any error conditions are reported back in this variable.
    118   *
    119   * @stable ICU 3.6
    120   */
    121 U_STABLE void U_EXPORT2
    122 ucsdet_setText(UCharsetDetector *ucsd, const char *textIn, int32_t len, UErrorCode *status);
    123 
    124 
    125 /** Set the declared encoding for charset detection.
    126  *  The declared encoding of an input text is an encoding obtained
    127  *  by the user from an http header or xml declaration or similar source that
    128  *  can be provided as an additional hint to the charset detector.
    129  *
    130  *  How and whether the declared encoding will be used during the
    131  *  detection process is TBD.
    132  *
    133  * @param ucsd      the charset detector to be used.
    134  * @param encoding  an encoding for the current data obtained from
    135  *                  a header or declaration or other source outside
    136  *                  of the byte data itself.
    137  * @param length    the length of the encoding name, or -1 if the name string
    138  *                  is NUL terminated.
    139  * @param status    any error conditions are reported back in this variable.
    140  *
    141  * @stable ICU 3.6
    142  */
    143 U_STABLE void U_EXPORT2
    144 ucsdet_setDeclaredEncoding(UCharsetDetector *ucsd, const char *encoding, int32_t length, UErrorCode *status);
    145 
    146 
    147 /**
    148  * Return the charset that best matches the supplied input data.
    149  *
    150  * Note though, that because the detection
    151  * only looks at the start of the input data,
    152  * there is a possibility that the returned charset will fail to handle
    153  * the full set of input data.
    154  * <p>
    155  * The returned UCharsetMatch object is owned by the UCharsetDetector.
    156  * It will remain valid until the detector input is reset, or until
    157  * the detector is closed.
    158  * <p>
    159  * The function will fail if
    160  *  <ul>
    161  *    <li>no charset appears to match the data.</li>
    162  *    <li>no input text has been provided</li>
    163  *  </ul>
    164  *
    165  * @param ucsd      the charset detector to be used.
    166  * @param status    any error conditions are reported back in this variable.
    167  * @return          a UCharsetMatch  representing the best matching charset,
    168  *                  or NULL if no charset matches the byte data.
    169  *
    170  * @stable ICU 3.6
    171  */
    172 U_STABLE const UCharsetMatch * U_EXPORT2
    173 ucsdet_detect(UCharsetDetector *ucsd, UErrorCode *status);
    174 
    175 
    176 /**
    177  *  Find all charset matches that appear to be consistent with the input,
    178  *  returning an array of results.  The results are ordered with the
    179  *  best quality match first.
    180  *
    181  *  Because the detection only looks at a limited amount of the
    182  *  input byte data, some of the returned charsets may fail to handle
    183  *  the all of input data.
    184  *  <p>
    185  *  The returned UCharsetMatch objects are owned by the UCharsetDetector.
    186  *  They will remain valid until the detector is closed or modified
    187  *
    188  * <p>
    189  * Return an error if
    190  *  <ul>
    191  *    <li>no charsets appear to match the input data.</li>
    192  *    <li>no input text has been provided</li>
    193  *  </ul>
    194  *
    195  * @param ucsd          the charset detector to be used.
    196  * @param matchesFound  pointer to a variable that will be set to the
    197  *                      number of charsets identified that are consistent with
    198  *                      the input data.  Output only.
    199  * @param status        any error conditions are reported back in this variable.
    200  * @return              A pointer to an array of pointers to UCharSetMatch objects.
    201  *                      This array, and the UCharSetMatch instances to which it refers,
    202  *                      are owned by the UCharsetDetector, and will remain valid until
    203  *                      the detector is closed or modified.
    204  * @stable ICU 3.6
    205  */
    206 U_STABLE const UCharsetMatch ** U_EXPORT2
    207 ucsdet_detectAll(UCharsetDetector *ucsd, int32_t *matchesFound, UErrorCode *status);
    208 
    209 
    210 
    211 /**
    212  *  Get the name of the charset represented by a UCharsetMatch.
    213  *
    214  *  The storage for the returned name string is owned by the
    215  *  UCharsetMatch, and will remain valid while the UCharsetMatch
    216  *  is valid.
    217  *
    218  *  The name returned is suitable for use with the ICU conversion APIs.
    219  *
    220  *  @param ucsm    The charset match object.
    221  *  @param status  Any error conditions are reported back in this variable.
    222  *  @return        The name of the matching charset.
    223  *
    224  *  @stable ICU 3.6
    225  */
    226 U_STABLE const char * U_EXPORT2
    227 ucsdet_getName(const UCharsetMatch *ucsm, UErrorCode *status);
    228 
    229 /**
    230  *  Get a confidence number for the quality of the match of the byte
    231  *  data with the charset.  Confidence numbers range from zero to 100,
    232  *  with 100 representing complete confidence and zero representing
    233  *  no confidence.
    234  *
    235  *  The confidence values are somewhat arbitrary.  They define an
    236  *  an ordering within the results for any single detection operation
    237  *  but are not generally comparable between the results for different input.
    238  *
    239  *  A confidence value of ten does have a general meaning - it is used
    240  *  for charsets that can represent the input data, but for which there
    241  *  is no other indication that suggests that the charset is the correct one.
    242  *  Pure 7 bit ASCII data, for example, is compatible with a
    243  *  great many charsets, most of which will appear as possible matches
    244  *  with a confidence of 10.
    245  *
    246  *  @param ucsm    The charset match object.
    247  *  @param status  Any error conditions are reported back in this variable.
    248  *  @return        A confidence number for the charset match.
    249  *
    250  *  @stable ICU 3.6
    251  */
    252 U_STABLE int32_t U_EXPORT2
    253 ucsdet_getConfidence(const UCharsetMatch *ucsm, UErrorCode *status);
    254 
    255 /**
    256  *  Get the RFC 3066 code for the language of the input data.
    257  *
    258  *  The Charset Detection service is intended primarily for detecting
    259  *  charsets, not language.  For some, but not all, charsets, a language is
    260  *  identified as a byproduct of the detection process, and that is what
    261  *  is returned by this function.
    262  *
    263  *  CAUTION:
    264  *    1.  Language information is not available for input data encoded in
    265  *        all charsets. In particular, no language is identified
    266  *        for UTF-8 input data.
    267  *
    268  *    2.  Closely related languages may sometimes be confused.
    269  *
    270  *  If more accurate language detection is required, a linguistic
    271  *  analysis package should be used.
    272  *
    273  *  The storage for the returned name string is owned by the
    274  *  UCharsetMatch, and will remain valid while the UCharsetMatch
    275  *  is valid.
    276  *
    277  *  @param ucsm    The charset match object.
    278  *  @param status  Any error conditions are reported back in this variable.
    279  *  @return        The RFC 3066 code for the language of the input data, or
    280  *                 an empty string if the language could not be determined.
    281  *
    282  *  @stable ICU 3.6
    283  */
    284 U_STABLE const char * U_EXPORT2
    285 ucsdet_getLanguage(const UCharsetMatch *ucsm, UErrorCode *status);
    286 
    287 
    288 /**
    289   *  Get the entire input text as a UChar string, placing it into
    290   *  a caller-supplied buffer.  A terminating
    291   *  NUL character will be appended to the buffer if space is available.
    292   *
    293   *  The number of UChars in the output string, not including the terminating
    294   *  NUL, is returned.
    295   *
    296   *  If the supplied buffer is smaller than required to hold the output,
    297   *  the contents of the buffer are undefined.  The full output string length
    298   *  (in UChars) is returned as always, and can be used to allocate a buffer
    299   *  of the correct size.
    300   *
    301   *
    302   * @param ucsm    The charset match object.
    303   * @param buf     A UChar buffer to be filled with the converted text data.
    304   * @param cap     The capacity of the buffer in UChars.
    305   * @param status  Any error conditions are reported back in this variable.
    306   * @return        The number of UChars in the output string.
    307   *
    308   * @stable ICU 3.6
    309   */
    310 U_STABLE  int32_t U_EXPORT2
    311 ucsdet_getUChars(const UCharsetMatch *ucsm,
    312                  UChar *buf, int32_t cap, UErrorCode *status);
    313 
    314 
    315 
    316 /**
    317   *  Get an iterator over the set of all detectable charsets -
    318   *  over the charsets that are known to the charset detection
    319   *  service.
    320   *
    321   *  The returned UEnumeration provides access to the names of
    322   *  the charsets.
    323   *
    324   *  <p>
    325   *  The state of the Charset detector that is passed in does not
    326   *  affect the result of this function, but requiring a valid, open
    327   *  charset detector as a parameter insures that the charset detection
    328   *  service has been safely initialized and that the required detection
    329   *  data is available.
    330   *
    331   *  <p>
    332   *  <b>Note:</b> Multiple different charset encodings in a same family may use
    333   *  a single shared name in this implementation. For example, this method returns
    334   *  an array including "ISO-8859-1" (ISO Latin 1), but not including "windows-1252"
    335   *  (Windows Latin 1). However, actual detection result could be "windows-1252"
    336   *  when the input data matches Latin 1 code points with any points only available
    337   *  in "windows-1252".
    338   *
    339   *  @param ucsd a Charset detector.
    340   *  @param status  Any error conditions are reported back in this variable.
    341   *  @return an iterator providing access to the detectable charset names.
    342   *  @stable ICU 3.6
    343   */
    344 U_STABLE  UEnumeration * U_EXPORT2
    345 ucsdet_getAllDetectableCharsets(const UCharsetDetector *ucsd,  UErrorCode *status);
    346 
    347 /**
    348   *  Test whether input filtering is enabled for this charset detector.
    349   *  Input filtering removes text that appears to be HTML or xml
    350   *  markup from the input before applying the code page detection
    351   *  heuristics.
    352   *
    353   *  @param ucsd  The charset detector to check.
    354   *  @return TRUE if filtering is enabled.
    355   *  @stable ICU 3.6
    356   */
    357 
    358 U_STABLE  UBool U_EXPORT2
    359 ucsdet_isInputFilterEnabled(const UCharsetDetector *ucsd);
    360 
    361 
    362 /**
    363  * Enable filtering of input text. If filtering is enabled,
    364  * text within angle brackets ("<" and ">") will be removed
    365  * before detection, which will remove most HTML or xml markup.
    366  *
    367  * @param ucsd   the charset detector to be modified.
    368  * @param filter <code>true</code> to enable input text filtering.
    369  * @return The previous setting.
    370  *
    371  * @stable ICU 3.6
    372  */
    373 U_STABLE  UBool U_EXPORT2
    374 ucsdet_enableInputFilter(UCharsetDetector *ucsd, UBool filter);
    375 
    376 #ifndef U_HIDE_INTERNAL_API
    377 /**
    378   *  Get an iterator over the set of detectable charsets -
    379   *  over the charsets that are enabled by the specified charset detector.
    380   *
    381   *  The returned UEnumeration provides access to the names of
    382   *  the charsets.
    383   *
    384   *  @param ucsd a Charset detector.
    385   *  @param status  Any error conditions are reported back in this variable.
    386   *  @return an iterator providing access to the detectable charset names by
    387   *  the specified charset detector.
    388   *  @internal
    389   */
    390 U_INTERNAL UEnumeration * U_EXPORT2
    391 ucsdet_getDetectableCharsets(const UCharsetDetector *ucsd,  UErrorCode *status);
    392 
    393 /**
    394   * Enable or disable individual charset encoding.
    395   * A name of charset encoding must be included in the names returned by
    396   * {@link #getAllDetectableCharsets()}.
    397   *
    398   * @param ucsd a Charset detector.
    399   * @param encoding encoding the name of charset encoding.
    400   * @param enabled <code>TRUE</code> to enable, or <code>FALSE</code> to disable the
    401   *   charset encoding.
    402   * @param status receives the return status. When the name of charset encoding
    403   *   is not supported, U_ILLEGAL_ARGUMENT_ERROR is set.
    404   * @internal
    405   */
    406 U_INTERNAL void U_EXPORT2
    407 ucsdet_setDetectableCharset(UCharsetDetector *ucsd, const char *encoding, UBool enabled, UErrorCode *status);
    408 #endif  /* U_HIDE_INTERNAL_API */
    409 
    410 #endif
    411 #endif   /* __UCSDET_H */
    412 
    413 
    414