1 """Conversion pipeline templates. 2 3 The problem: 4 ------------ 5 6 Suppose you have some data that you want to convert to another format, 7 such as from GIF image format to PPM image format. Maybe the 8 conversion involves several steps (e.g. piping it through compress or 9 uuencode). Some of the conversion steps may require that their input 10 is a disk file, others may be able to read standard input; similar for 11 their output. The input to the entire conversion may also be read 12 from a disk file or from an open file, and similar for its output. 13 14 The module lets you construct a pipeline template by sticking one or 15 more conversion steps together. It will take care of creating and 16 removing temporary files if they are necessary to hold intermediate 17 data. You can then use the template to do conversions from many 18 different sources to many different destinations. The temporary 19 file names used are different each time the template is used. 20 21 The templates are objects so you can create templates for many 22 different conversion steps and store them in a dictionary, for 23 instance. 24 25 26 Directions: 27 ----------- 28 29 To create a template: 30 t = Template() 31 32 To add a conversion step to a template: 33 t.append(command, kind) 34 where kind is a string of two characters: the first is '-' if the 35 command reads its standard input or 'f' if it requires a file; the 36 second likewise for the output. The command must be valid /bin/sh 37 syntax. If input or output files are required, they are passed as 38 $IN and $OUT; otherwise, it must be possible to use the command in 39 a pipeline. 40 41 To add a conversion step at the beginning: 42 t.prepend(command, kind) 43 44 To convert a file to another file using a template: 45 sts = t.copy(infile, outfile) 46 If infile or outfile are the empty string, standard input is read or 47 standard output is written, respectively. The return value is the 48 exit status of the conversion pipeline. 49 50 To open a file for reading or writing through a conversion pipeline: 51 fp = t.open(file, mode) 52 where mode is 'r' to read the file, or 'w' to write it -- just like 53 for the built-in function open() or for os.popen(). 54 55 To create a new template object initialized to a given one: 56 t2 = t.clone() 57 """ # ' 58 59 60 import re 61 import os 62 import tempfile 63 import string 64 65 __all__ = ["Template"] 66 67 # Conversion step kinds 68 69 FILEIN_FILEOUT = 'ff' # Must read & write real files 70 STDIN_FILEOUT = '-f' # Must write a real file 71 FILEIN_STDOUT = 'f-' # Must read a real file 72 STDIN_STDOUT = '--' # Normal pipeline element 73 SOURCE = '.-' # Must be first, writes stdout 74 SINK = '-.' # Must be last, reads stdin 75 76 stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \ 77 SOURCE, SINK] 78 79 80 class Template: 81 """Class representing a pipeline template.""" 82 83 def __init__(self): 84 """Template() returns a fresh pipeline template.""" 85 self.debugging = 0 86 self.reset() 87 88 def __repr__(self): 89 """t.__repr__() implements repr(t).""" 90 return '<Template instance, steps=%r>' % (self.steps,) 91 92 def reset(self): 93 """t.reset() restores a pipeline template to its initial state.""" 94 self.steps = [] 95 96 def clone(self): 97 """t.clone() returns a new pipeline template with identical 98 initial state as the current one.""" 99 t = Template() 100 t.steps = self.steps[:] 101 t.debugging = self.debugging 102 return t 103 104 def debug(self, flag): 105 """t.debug(flag) turns debugging on or off.""" 106 self.debugging = flag 107 108 def append(self, cmd, kind): 109 """t.append(cmd, kind) adds a new step at the end.""" 110 if type(cmd) is not type(''): 111 raise TypeError, \ 112 'Template.append: cmd must be a string' 113 if kind not in stepkinds: 114 raise ValueError, \ 115 'Template.append: bad kind %r' % (kind,) 116 if kind == SOURCE: 117 raise ValueError, \ 118 'Template.append: SOURCE can only be prepended' 119 if self.steps and self.steps[-1][1] == SINK: 120 raise ValueError, \ 121 'Template.append: already ends with SINK' 122 if kind[0] == 'f' and not re.search(r'\$IN\b', cmd): 123 raise ValueError, \ 124 'Template.append: missing $IN in cmd' 125 if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd): 126 raise ValueError, \ 127 'Template.append: missing $OUT in cmd' 128 self.steps.append((cmd, kind)) 129 130 def prepend(self, cmd, kind): 131 """t.prepend(cmd, kind) adds a new step at the front.""" 132 if type(cmd) is not type(''): 133 raise TypeError, \ 134 'Template.prepend: cmd must be a string' 135 if kind not in stepkinds: 136 raise ValueError, \ 137 'Template.prepend: bad kind %r' % (kind,) 138 if kind == SINK: 139 raise ValueError, \ 140 'Template.prepend: SINK can only be appended' 141 if self.steps and self.steps[0][1] == SOURCE: 142 raise ValueError, \ 143 'Template.prepend: already begins with SOURCE' 144 if kind[0] == 'f' and not re.search(r'\$IN\b', cmd): 145 raise ValueError, \ 146 'Template.prepend: missing $IN in cmd' 147 if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd): 148 raise ValueError, \ 149 'Template.prepend: missing $OUT in cmd' 150 self.steps.insert(0, (cmd, kind)) 151 152 def open(self, file, rw): 153 """t.open(file, rw) returns a pipe or file object open for 154 reading or writing; the file is the other end of the pipeline.""" 155 if rw == 'r': 156 return self.open_r(file) 157 if rw == 'w': 158 return self.open_w(file) 159 raise ValueError, \ 160 'Template.open: rw must be \'r\' or \'w\', not %r' % (rw,) 161 162 def open_r(self, file): 163 """t.open_r(file) and t.open_w(file) implement 164 t.open(file, 'r') and t.open(file, 'w') respectively.""" 165 if not self.steps: 166 return open(file, 'r') 167 if self.steps[-1][1] == SINK: 168 raise ValueError, \ 169 'Template.open_r: pipeline ends width SINK' 170 cmd = self.makepipeline(file, '') 171 return os.popen(cmd, 'r') 172 173 def open_w(self, file): 174 if not self.steps: 175 return open(file, 'w') 176 if self.steps[0][1] == SOURCE: 177 raise ValueError, \ 178 'Template.open_w: pipeline begins with SOURCE' 179 cmd = self.makepipeline('', file) 180 return os.popen(cmd, 'w') 181 182 def copy(self, infile, outfile): 183 return os.system(self.makepipeline(infile, outfile)) 184 185 def makepipeline(self, infile, outfile): 186 cmd = makepipeline(infile, self.steps, outfile) 187 if self.debugging: 188 print cmd 189 cmd = 'set -x; ' + cmd 190 return cmd 191 192 193 def makepipeline(infile, steps, outfile): 194 # Build a list with for each command: 195 # [input filename or '', command string, kind, output filename or ''] 196 197 list = [] 198 for cmd, kind in steps: 199 list.append(['', cmd, kind, '']) 200 # 201 # Make sure there is at least one step 202 # 203 if not list: 204 list.append(['', 'cat', '--', '']) 205 # 206 # Take care of the input and output ends 207 # 208 [cmd, kind] = list[0][1:3] 209 if kind[0] == 'f' and not infile: 210 list.insert(0, ['', 'cat', '--', '']) 211 list[0][0] = infile 212 # 213 [cmd, kind] = list[-1][1:3] 214 if kind[1] == 'f' and not outfile: 215 list.append(['', 'cat', '--', '']) 216 list[-1][-1] = outfile 217 # 218 # Invent temporary files to connect stages that need files 219 # 220 garbage = [] 221 for i in range(1, len(list)): 222 lkind = list[i-1][2] 223 rkind = list[i][2] 224 if lkind[1] == 'f' or rkind[0] == 'f': 225 (fd, temp) = tempfile.mkstemp() 226 os.close(fd) 227 garbage.append(temp) 228 list[i-1][-1] = list[i][0] = temp 229 # 230 for item in list: 231 [inf, cmd, kind, outf] = item 232 if kind[1] == 'f': 233 cmd = 'OUT=' + quote(outf) + '; ' + cmd 234 if kind[0] == 'f': 235 cmd = 'IN=' + quote(inf) + '; ' + cmd 236 if kind[0] == '-' and inf: 237 cmd = cmd + ' <' + quote(inf) 238 if kind[1] == '-' and outf: 239 cmd = cmd + ' >' + quote(outf) 240 item[1] = cmd 241 # 242 cmdlist = list[0][1] 243 for item in list[1:]: 244 [cmd, kind] = item[1:3] 245 if item[0] == '': 246 if 'f' in kind: 247 cmd = '{ ' + cmd + '; }' 248 cmdlist = cmdlist + ' |\n' + cmd 249 else: 250 cmdlist = cmdlist + '\n' + cmd 251 # 252 if garbage: 253 rmcmd = 'rm -f' 254 for file in garbage: 255 rmcmd = rmcmd + ' ' + quote(file) 256 trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15' 257 cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd 258 # 259 return cmdlist 260 261 262 # Reliably quote a string as a single argument for /bin/sh 263 264 # Safe unquoted 265 _safechars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./') 266 267 def quote(file): 268 """Return a shell-escaped version of the file string.""" 269 for c in file: 270 if c not in _safechars: 271 break 272 else: 273 if not file: 274 return "''" 275 return file 276 # use single quotes, and put single quotes into double quotes 277 # the string $'b is then quoted as '$'"'"'b' 278 return "'" + file.replace("'", "'\"'\"'") + "'" 279
