Home | History | Annotate | Download | only in email
      1 # Copyright (C) 2001-2006 Python Software Foundation
      2 # Author: Ben Gertzfield, Barry Warsaw
      3 # Contact: email-sig (at] python.org
      4 
      5 __all__ = [
      6     'Charset',
      7     'add_alias',
      8     'add_charset',
      9     'add_codec',
     10     ]
     11 
     12 import codecs
     13 import email.base64mime
     14 import email.quoprimime
     15 
     16 from email import errors
     17 from email.encoders import encode_7or8bit
     18 
     19 
     20 
     22 # Flags for types of header encodings
     23 QP          = 1 # Quoted-Printable
     24 BASE64      = 2 # Base64
     25 SHORTEST    = 3 # the shorter of QP and base64, but only for headers
     26 
     27 # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
     28 MISC_LEN = 7
     29 
     30 DEFAULT_CHARSET = 'us-ascii'
     31 
     32 
     33 
     35 # Defaults
     36 CHARSETS = {
     37     # input        header enc  body enc output conv
     38     'iso-8859-1':  (QP,        QP,      None),
     39     'iso-8859-2':  (QP,        QP,      None),
     40     'iso-8859-3':  (QP,        QP,      None),
     41     'iso-8859-4':  (QP,        QP,      None),
     42     # iso-8859-5 is Cyrillic, and not especially used
     43     # iso-8859-6 is Arabic, also not particularly used
     44     # iso-8859-7 is Greek, QP will not make it readable
     45     # iso-8859-8 is Hebrew, QP will not make it readable
     46     'iso-8859-9':  (QP,        QP,      None),
     47     'iso-8859-10': (QP,        QP,      None),
     48     # iso-8859-11 is Thai, QP will not make it readable
     49     'iso-8859-13': (QP,        QP,      None),
     50     'iso-8859-14': (QP,        QP,      None),
     51     'iso-8859-15': (QP,        QP,      None),
     52     'iso-8859-16': (QP,        QP,      None),
     53     'windows-1252':(QP,        QP,      None),
     54     'viscii':      (QP,        QP,      None),
     55     'us-ascii':    (None,      None,    None),
     56     'big5':        (BASE64,    BASE64,  None),
     57     'gb2312':      (BASE64,    BASE64,  None),
     58     'euc-jp':      (BASE64,    None,    'iso-2022-jp'),
     59     'shift_jis':   (BASE64,    None,    'iso-2022-jp'),
     60     'iso-2022-jp': (BASE64,    None,    None),
     61     'koi8-r':      (BASE64,    BASE64,  None),
     62     'utf-8':       (SHORTEST,  BASE64, 'utf-8'),
     63     # We're making this one up to represent raw unencoded 8-bit
     64     '8bit':        (None,      BASE64, 'utf-8'),
     65     }
     66 
     67 # Aliases for other commonly-used names for character sets.  Map
     68 # them to the real ones used in email.
     69 ALIASES = {
     70     'latin_1': 'iso-8859-1',
     71     'latin-1': 'iso-8859-1',
     72     'latin_2': 'iso-8859-2',
     73     'latin-2': 'iso-8859-2',
     74     'latin_3': 'iso-8859-3',
     75     'latin-3': 'iso-8859-3',
     76     'latin_4': 'iso-8859-4',
     77     'latin-4': 'iso-8859-4',
     78     'latin_5': 'iso-8859-9',
     79     'latin-5': 'iso-8859-9',
     80     'latin_6': 'iso-8859-10',
     81     'latin-6': 'iso-8859-10',
     82     'latin_7': 'iso-8859-13',
     83     'latin-7': 'iso-8859-13',
     84     'latin_8': 'iso-8859-14',
     85     'latin-8': 'iso-8859-14',
     86     'latin_9': 'iso-8859-15',
     87     'latin-9': 'iso-8859-15',
     88     'latin_10':'iso-8859-16',
     89     'latin-10':'iso-8859-16',
     90     'cp949':   'ks_c_5601-1987',
     91     'euc_jp':  'euc-jp',
     92     'euc_kr':  'euc-kr',
     93     'ascii':   'us-ascii',
     94     }
     95 
     96 
     97 # Map charsets to their Unicode codec strings.
     98 CODEC_MAP = {
     99     'gb2312':      'eucgb2312_cn',
    100     'big5':        'big5_tw',
    101     # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
    102     # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
    103     # Let that stuff pass through without conversion to/from Unicode.
    104     'us-ascii':    None,
    105     }
    106 
    107 
    108 
    110 # Convenience functions for extending the above mappings
    111 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
    112     """Add character set properties to the global registry.
    113 
    114     charset is the input character set, and must be the canonical name of a
    115     character set.
    116 
    117     Optional header_enc and body_enc is either Charset.QP for
    118     quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
    119     the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
    120     is only valid for header_enc.  It describes how message headers and
    121     message bodies in the input charset are to be encoded.  Default is no
    122     encoding.
    123 
    124     Optional output_charset is the character set that the output should be
    125     in.  Conversions will proceed from input charset, to Unicode, to the
    126     output charset when the method Charset.convert() is called.  The default
    127     is to output in the same character set as the input.
    128 
    129     Both input_charset and output_charset must have Unicode codec entries in
    130     the module's charset-to-codec mapping; use add_codec(charset, codecname)
    131     to add codecs the module does not know about.  See the codecs module's
    132     documentation for more information.
    133     """
    134     if body_enc == SHORTEST:
    135         raise ValueError('SHORTEST not allowed for body_enc')
    136     CHARSETS[charset] = (header_enc, body_enc, output_charset)
    137 
    138 
    139 def add_alias(alias, canonical):
    140     """Add a character set alias.
    141 
    142     alias is the alias name, e.g. latin-1
    143     canonical is the character set's canonical name, e.g. iso-8859-1
    144     """
    145     ALIASES[alias] = canonical
    146 
    147 
    148 def add_codec(charset, codecname):
    149     """Add a codec that map characters in the given charset to/from Unicode.
    150 
    151     charset is the canonical name of a character set.  codecname is the name
    152     of a Python codec, as appropriate for the second argument to the unicode()
    153     built-in, or to the encode() method of a Unicode string.
    154     """
    155     CODEC_MAP[charset] = codecname
    156 
    157 
    158 
    160 class Charset:
    161     """Map character sets to their email properties.
    162 
    163     This class provides information about the requirements imposed on email
    164     for a specific character set.  It also provides convenience routines for
    165     converting between character sets, given the availability of the
    166     applicable codecs.  Given a character set, it will do its best to provide
    167     information on how to use that character set in an email in an
    168     RFC-compliant way.
    169 
    170     Certain character sets must be encoded with quoted-printable or base64
    171     when used in email headers or bodies.  Certain character sets must be
    172     converted outright, and are not allowed in email.  Instances of this
    173     module expose the following information about a character set:
    174 
    175     input_charset: The initial character set specified.  Common aliases
    176                    are converted to their `official' email names (e.g. latin_1
    177                    is converted to iso-8859-1).  Defaults to 7-bit us-ascii.
    178 
    179     header_encoding: If the character set must be encoded before it can be
    180                      used in an email header, this attribute will be set to
    181                      Charset.QP (for quoted-printable), Charset.BASE64 (for
    182                      base64 encoding), or Charset.SHORTEST for the shortest of
    183                      QP or BASE64 encoding.  Otherwise, it will be None.
    184 
    185     body_encoding: Same as header_encoding, but describes the encoding for the
    186                    mail message's body, which indeed may be different than the
    187                    header encoding.  Charset.SHORTEST is not allowed for
    188                    body_encoding.
    189 
    190     output_charset: Some character sets must be converted before the can be
    191                     used in email headers or bodies.  If the input_charset is
    192                     one of them, this attribute will contain the name of the
    193                     charset output will be converted to.  Otherwise, it will
    194                     be None.
    195 
    196     input_codec: The name of the Python codec used to convert the
    197                  input_charset to Unicode.  If no conversion codec is
    198                  necessary, this attribute will be None.
    199 
    200     output_codec: The name of the Python codec used to convert Unicode
    201                   to the output_charset.  If no conversion codec is necessary,
    202                   this attribute will have the same value as the input_codec.
    203     """
    204     def __init__(self, input_charset=DEFAULT_CHARSET):
    205         # RFC 2046, $4.1.2 says charsets are not case sensitive.  We coerce to
    206         # unicode because its .lower() is locale insensitive.  If the argument
    207         # is already a unicode, we leave it at that, but ensure that the
    208         # charset is ASCII, as the standard (RFC XXX) requires.
    209         try:
    210             if isinstance(input_charset, unicode):
    211                 input_charset.encode('ascii')
    212             else:
    213                 input_charset = unicode(input_charset, 'ascii')
    214         except UnicodeError:
    215             raise errors.CharsetError(input_charset)
    216         input_charset = input_charset.lower().encode('ascii')
    217         # Set the input charset after filtering through the aliases and/or codecs
    218         if not (input_charset in ALIASES or input_charset in CHARSETS):
    219             try:
    220                 input_charset = codecs.lookup(input_charset).name
    221             except LookupError:
    222                 pass
    223         self.input_charset = ALIASES.get(input_charset, input_charset)
    224         # We can try to guess which encoding and conversion to use by the
    225         # charset_map dictionary.  Try that first, but let the user override
    226         # it.
    227         henc, benc, conv = CHARSETS.get(self.input_charset,
    228                                         (SHORTEST, BASE64, None))
    229         if not conv:
    230             conv = self.input_charset
    231         # Set the attributes, allowing the arguments to override the default.
    232         self.header_encoding = henc
    233         self.body_encoding = benc
    234         self.output_charset = ALIASES.get(conv, conv)
    235         # Now set the codecs.  If one isn't defined for input_charset,
    236         # guess and try a Unicode codec with the same name as input_codec.
    237         self.input_codec = CODEC_MAP.get(self.input_charset,
    238                                          self.input_charset)
    239         self.output_codec = CODEC_MAP.get(self.output_charset,
    240                                           self.output_charset)
    241 
    242     def __str__(self):
    243         return self.input_charset.lower()
    244 
    245     __repr__ = __str__
    246 
    247     def __eq__(self, other):
    248         return str(self) == str(other).lower()
    249 
    250     def __ne__(self, other):
    251         return not self.__eq__(other)
    252 
    253     def get_body_encoding(self):
    254         """Return the content-transfer-encoding used for body encoding.
    255 
    256         This is either the string `quoted-printable' or `base64' depending on
    257         the encoding used, or it is a function in which case you should call
    258         the function with a single argument, the Message object being
    259         encoded.  The function should then set the Content-Transfer-Encoding
    260         header itself to whatever is appropriate.
    261 
    262         Returns "quoted-printable" if self.body_encoding is QP.
    263         Returns "base64" if self.body_encoding is BASE64.
    264         Returns "7bit" otherwise.
    265         """
    266         assert self.body_encoding != SHORTEST
    267         if self.body_encoding == QP:
    268             return 'quoted-printable'
    269         elif self.body_encoding == BASE64:
    270             return 'base64'
    271         else:
    272             return encode_7or8bit
    273 
    274     def convert(self, s):
    275         """Convert a string from the input_codec to the output_codec."""
    276         if self.input_codec != self.output_codec:
    277             return unicode(s, self.input_codec).encode(self.output_codec)
    278         else:
    279             return s
    280 
    281     def to_splittable(self, s):
    282         """Convert a possibly multibyte string to a safely splittable format.
    283 
    284         Uses the input_codec to try and convert the string to Unicode, so it
    285         can be safely split on character boundaries (even for multibyte
    286         characters).
    287 
    288         Returns the string as-is if it isn't known how to convert it to
    289         Unicode with the input_charset.
    290 
    291         Characters that could not be converted to Unicode will be replaced
    292         with the Unicode replacement character U+FFFD.
    293         """
    294         if isinstance(s, unicode) or self.input_codec is None:
    295             return s
    296         try:
    297             return unicode(s, self.input_codec, 'replace')
    298         except LookupError:
    299             # Input codec not installed on system, so return the original
    300             # string unchanged.
    301             return s
    302 
    303     def from_splittable(self, ustr, to_output=True):
    304         """Convert a splittable string back into an encoded string.
    305 
    306         Uses the proper codec to try and convert the string from Unicode back
    307         into an encoded format.  Return the string as-is if it is not Unicode,
    308         or if it could not be converted from Unicode.
    309 
    310         Characters that could not be converted from Unicode will be replaced
    311         with an appropriate character (usually '?').
    312 
    313         If to_output is True (the default), uses output_codec to convert to an
    314         encoded format.  If to_output is False, uses input_codec.
    315         """
    316         if to_output:
    317             codec = self.output_codec
    318         else:
    319             codec = self.input_codec
    320         if not isinstance(ustr, unicode) or codec is None:
    321             return ustr
    322         try:
    323             return ustr.encode(codec, 'replace')
    324         except LookupError:
    325             # Output codec not installed
    326             return ustr
    327 
    328     def get_output_charset(self):
    329         """Return the output character set.
    330 
    331         This is self.output_charset if that is not None, otherwise it is
    332         self.input_charset.
    333         """
    334         return self.output_charset or self.input_charset
    335 
    336     def encoded_header_len(self, s):
    337         """Return the length of the encoded header string."""
    338         cset = self.get_output_charset()
    339         # The len(s) of a 7bit encoding is len(s)
    340         if self.header_encoding == BASE64:
    341             return email.base64mime.base64_len(s) + len(cset) + MISC_LEN
    342         elif self.header_encoding == QP:
    343             return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN
    344         elif self.header_encoding == SHORTEST:
    345             lenb64 = email.base64mime.base64_len(s)
    346             lenqp = email.quoprimime.header_quopri_len(s)
    347             return min(lenb64, lenqp) + len(cset) + MISC_LEN
    348         else:
    349             return len(s)
    350 
    351     def header_encode(self, s, convert=False):
    352         """Header-encode a string, optionally converting it to output_charset.
    353 
    354         If convert is True, the string will be converted from the input
    355         charset to the output charset automatically.  This is not useful for
    356         multibyte character sets, which have line length issues (multibyte
    357         characters must be split on a character, not a byte boundary); use the
    358         high-level Header class to deal with these issues.  convert defaults
    359         to False.
    360 
    361         The type of encoding (base64 or quoted-printable) will be based on
    362         self.header_encoding.
    363         """
    364         cset = self.get_output_charset()
    365         if convert:
    366             s = self.convert(s)
    367         # 7bit/8bit encodings return the string unchanged (modulo conversions)
    368         if self.header_encoding == BASE64:
    369             return email.base64mime.header_encode(s, cset)
    370         elif self.header_encoding == QP:
    371             return email.quoprimime.header_encode(s, cset, maxlinelen=None)
    372         elif self.header_encoding == SHORTEST:
    373             lenb64 = email.base64mime.base64_len(s)
    374             lenqp = email.quoprimime.header_quopri_len(s)
    375             if lenb64 < lenqp:
    376                 return email.base64mime.header_encode(s, cset)
    377             else:
    378                 return email.quoprimime.header_encode(s, cset, maxlinelen=None)
    379         else:
    380             return s
    381 
    382     def body_encode(self, s, convert=True):
    383         """Body-encode a string and convert it to output_charset.
    384 
    385         If convert is True (the default), the string will be converted from
    386         the input charset to output charset automatically.  Unlike
    387         header_encode(), there are no issues with byte boundaries and
    388         multibyte charsets in email bodies, so this is usually pretty safe.
    389 
    390         The type of encoding (base64 or quoted-printable) will be based on
    391         self.body_encoding.
    392         """
    393         if convert:
    394             s = self.convert(s)
    395         # 7bit/8bit encodings return the string unchanged (module conversions)
    396         if self.body_encoding is BASE64:
    397             return email.base64mime.body_encode(s)
    398         elif self.body_encoding is QP:
    399             return email.quoprimime.body_encode(s)
    400         else:
    401             return s
    402