1 # Test the Unicode versions of normal file functions 2 # open, os.open, os.stat. os.listdir, os.rename, os.remove, os.mkdir, os.chdir, os.rmdir 3 import sys, os, unittest 4 from unicodedata import normalize 5 from test import test_support 6 7 filenames = [ 8 '1_abc', 9 u'2_ascii', 10 u'3_Gr\xfc\xdf-Gott', 11 u'4_\u0393\u03b5\u03b9\u03ac-\u03c3\u03b1\u03c2', 12 u'5_\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435', 13 u'6_\u306b\u307d\u3093', 14 u'7_\u05d4\u05e9\u05e7\u05e6\u05e5\u05e1', 15 u'8_\u66e8\u66e9\u66eb', 16 u'9_\u66e8\u05e9\u3093\u0434\u0393\xdf', 17 # Specific code points: fn, NFC(fn) and NFKC(fn) all differents 18 u'10_\u1fee\u1ffd', 19 ] 20 21 # Mac OS X decomposes Unicode names, using Normal Form D. 22 # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html 23 # "However, most volume formats do not follow the exact specification for 24 # these normal forms. For example, HFS Plus uses a variant of Normal Form D 25 # in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through 26 # U+2FAFF are not decomposed." 27 if sys.platform != 'darwin': 28 filenames.extend([ 29 # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all differents 30 u'11_\u0385\u03d3\u03d4', 31 u'12_\u00a8\u0301\u03d2\u0301\u03d2\u0308', # == NFD(u'\u0385\u03d3\u03d4') 32 u'13_\u0020\u0308\u0301\u038e\u03ab', # == NFKC(u'\u0385\u03d3\u03d4') 33 u'14_\u1e9b\u1fc1\u1fcd\u1fce\u1fcf\u1fdd\u1fde\u1fdf\u1fed', 34 35 # Specific code points: fn, NFC(fn) and NFKC(fn) all differents 36 u'15_\u1fee\u1ffd\ufad1', 37 u'16_\u2000\u2000\u2000A', 38 u'17_\u2001\u2001\u2001A', 39 u'18_\u2003\u2003\u2003A', # == NFC(u'\u2001\u2001\u2001A') 40 u'19_\u0020\u0020\u0020A', # u'\u0020' == u' ' == NFKC(u'\u2000') == 41 # NFKC(u'\u2001') == NFKC(u'\u2003') 42 ]) 43 44 45 # Is it Unicode-friendly? 46 if not os.path.supports_unicode_filenames: 47 fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding() 48 try: 49 for name in filenames: 50 name.encode(fsencoding) 51 except UnicodeEncodeError: 52 raise unittest.SkipTest("only NT+ and systems with " 53 "Unicode-friendly filesystem encoding") 54 55 56 # Destroy directory dirname and all files under it, to one level. 57 def deltree(dirname): 58 # Don't hide legitimate errors: if one of these suckers exists, it's 59 # an error if we can't remove it. 60 if os.path.exists(dirname): 61 # must pass unicode to os.listdir() so we get back unicode results. 62 for fname in os.listdir(unicode(dirname)): 63 os.unlink(os.path.join(dirname, fname)) 64 os.rmdir(dirname) 65 66 67 class UnicodeFileTests(unittest.TestCase): 68 files = set(filenames) 69 normal_form = None 70 71 def setUp(self): 72 try: 73 os.mkdir(test_support.TESTFN) 74 except OSError: 75 pass 76 files = set() 77 for name in self.files: 78 name = os.path.join(test_support.TESTFN, self.norm(name)) 79 with open(name, 'w') as f: 80 f.write((name+'\n').encode("utf-8")) 81 os.stat(name) 82 files.add(name) 83 self.files = files 84 85 def tearDown(self): 86 deltree(test_support.TESTFN) 87 88 def norm(self, s): 89 if self.normal_form and isinstance(s, unicode): 90 return normalize(self.normal_form, s) 91 return s 92 93 def _apply_failure(self, fn, filename, expected_exception, 94 check_fn_in_exception = True): 95 with self.assertRaises(expected_exception) as c: 96 fn(filename) 97 exc_filename = c.exception.filename 98 # the "filename" exception attribute may be encoded 99 if isinstance(exc_filename, str): 100 filename = filename.encode(sys.getfilesystemencoding()) 101 if check_fn_in_exception: 102 self.assertEqual(exc_filename, filename, "Function '%s(%r) failed " 103 "with bad filename in the exception: %r" % 104 (fn.__name__, filename, exc_filename)) 105 106 def test_failures(self): 107 # Pass non-existing Unicode filenames all over the place. 108 for name in self.files: 109 name = "not_" + name 110 self._apply_failure(open, name, IOError) 111 self._apply_failure(os.stat, name, OSError) 112 self._apply_failure(os.chdir, name, OSError) 113 self._apply_failure(os.rmdir, name, OSError) 114 self._apply_failure(os.remove, name, OSError) 115 # listdir may append a wildcard to the filename, so dont check 116 self._apply_failure(os.listdir, name, OSError, False) 117 118 def test_open(self): 119 for name in self.files: 120 f = open(name, 'w') 121 f.write((name+'\n').encode("utf-8")) 122 f.close() 123 os.stat(name) 124 125 # Skip the test on darwin, because darwin does normalize the filename to 126 # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, 127 # NFKD in Python is useless, because darwin will normalize it later and so 128 # open(), os.stat(), etc. don't raise any exception. 129 @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') 130 def test_normalize(self): 131 files = set(f for f in self.files if isinstance(f, unicode)) 132 others = set() 133 for nf in set(['NFC', 'NFD', 'NFKC', 'NFKD']): 134 others |= set(normalize(nf, file) for file in files) 135 others -= files 136 for name in others: 137 self._apply_failure(open, name, IOError) 138 self._apply_failure(os.stat, name, OSError) 139 self._apply_failure(os.chdir, name, OSError) 140 self._apply_failure(os.rmdir, name, OSError) 141 self._apply_failure(os.remove, name, OSError) 142 # listdir may append a wildcard to the filename, so dont check 143 self._apply_failure(os.listdir, name, OSError, False) 144 145 # Skip the test on darwin, because darwin uses a normalization different 146 # than Python NFD normalization: filenames are different even if we use 147 # Python NFD normalization. 148 @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') 149 def test_listdir(self): 150 sf0 = set(self.files) 151 f1 = os.listdir(test_support.TESTFN) 152 f2 = os.listdir(unicode(test_support.TESTFN, 153 sys.getfilesystemencoding())) 154 sf2 = set(os.path.join(unicode(test_support.TESTFN), f) for f in f2) 155 self.assertEqual(sf0, sf2) 156 self.assertEqual(len(f1), len(f2)) 157 158 def test_rename(self): 159 for name in self.files: 160 os.rename(name, "tmp") 161 os.rename("tmp", name) 162 163 def test_directory(self): 164 dirname = os.path.join(test_support.TESTFN, 165 u'Gr\xfc\xdf-\u66e8\u66e9\u66eb') 166 filename = u'\xdf-\u66e8\u66e9\u66eb' 167 oldwd = os.getcwd() 168 os.mkdir(dirname) 169 os.chdir(dirname) 170 try: 171 with open(filename, 'w') as f: 172 f.write((filename + '\n').encode("utf-8")) 173 os.access(filename,os.R_OK) 174 os.remove(filename) 175 finally: 176 os.chdir(oldwd) 177 os.rmdir(dirname) 178 179 180 class UnicodeNFCFileTests(UnicodeFileTests): 181 normal_form = 'NFC' 182 183 184 class UnicodeNFDFileTests(UnicodeFileTests): 185 normal_form = 'NFD' 186 187 188 class UnicodeNFKCFileTests(UnicodeFileTests): 189 normal_form = 'NFKC' 190 191 192 class UnicodeNFKDFileTests(UnicodeFileTests): 193 normal_form = 'NFKD' 194 195 196 def test_main(): 197 try: 198 test_support.run_unittest( 199 UnicodeFileTests, 200 UnicodeNFCFileTests, 201 UnicodeNFDFileTests, 202 UnicodeNFKCFileTests, 203 UnicodeNFKDFileTests, 204 ) 205 finally: 206 deltree(test_support.TESTFN) 207 208 209 if __name__ == "__main__": 210 test_main() 211
