1 #!/usr/bin/env python 2 # 3 # Copyright 2006-2008 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 # This is a utility for converting JavaScript source code into C-style 31 # char arrays. It is used for embedded JavaScript code in the V8 32 # library. 33 34 import os, re, sys, string 35 import jsmin 36 import bz2 37 38 39 def ToCAsciiArray(lines): 40 result = [] 41 for chr in lines: 42 value = ord(chr) 43 assert value < 128 44 result.append(str(value)) 45 return ", ".join(result) 46 47 48 def ToCArray(lines): 49 result = [] 50 for chr in lines: 51 result.append(str(ord(chr))) 52 return ", ".join(result) 53 54 55 def RemoveCommentsAndTrailingWhitespace(lines): 56 lines = re.sub(r'//.*\n', '\n', lines) # end-of-line comments 57 lines = re.sub(re.compile(r'/\*.*?\*/', re.DOTALL), '', lines) # comments. 58 lines = re.sub(r'\s+\n+', '\n', lines) # trailing whitespace 59 return lines 60 61 62 def ReadFile(filename): 63 file = open(filename, "rt") 64 try: 65 lines = file.read() 66 finally: 67 file.close() 68 return lines 69 70 71 def ReadLines(filename): 72 result = [] 73 for line in open(filename, "rt"): 74 if '#' in line: 75 line = line[:line.index('#')] 76 line = line.strip() 77 if len(line) > 0: 78 result.append(line) 79 return result 80 81 82 def LoadConfigFrom(name): 83 import ConfigParser 84 config = ConfigParser.ConfigParser() 85 config.read(name) 86 return config 87 88 89 def ParseValue(string): 90 string = string.strip() 91 if string.startswith('[') and string.endswith(']'): 92 return string.lstrip('[').rstrip(']').split() 93 else: 94 return string 95 96 97 EVAL_PATTERN = re.compile(r'\beval\s*\(') 98 WITH_PATTERN = re.compile(r'\bwith\s*\(') 99 100 101 def Validate(lines, file): 102 lines = RemoveCommentsAndTrailingWhitespace(lines) 103 # Because of simplified context setup, eval and with is not 104 # allowed in the natives files. 105 eval_match = EVAL_PATTERN.search(lines) 106 if eval_match: 107 raise ("Eval disallowed in natives: %s" % file) 108 with_match = WITH_PATTERN.search(lines) 109 if with_match: 110 raise ("With statements disallowed in natives: %s" % file) 111 112 113 def ExpandConstants(lines, constants): 114 for key, value in constants: 115 lines = key.sub(str(value), lines) 116 return lines 117 118 119 def ExpandMacros(lines, macros): 120 # We allow macros to depend on the previously declared macros, but 121 # we don't allow self-dependecies or recursion. 122 for name_pattern, macro in reversed(macros): 123 pattern_match = name_pattern.search(lines, 0) 124 while pattern_match is not None: 125 # Scan over the arguments 126 height = 1 127 start = pattern_match.start() 128 end = pattern_match.end() 129 assert lines[end - 1] == '(' 130 last_match = end 131 arg_index = [0] # Wrap state into array, to work around Python "scoping" 132 mapping = { } 133 def add_arg(str): 134 # Remember to expand recursively in the arguments 135 replacement = ExpandMacros(str.strip(), macros) 136 mapping[macro.args[arg_index[0]]] = replacement 137 arg_index[0] += 1 138 while end < len(lines) and height > 0: 139 # We don't count commas at higher nesting levels. 140 if lines[end] == ',' and height == 1: 141 add_arg(lines[last_match:end]) 142 last_match = end + 1 143 elif lines[end] in ['(', '{', '[']: 144 height = height + 1 145 elif lines[end] in [')', '}', ']']: 146 height = height - 1 147 end = end + 1 148 # Remember to add the last match. 149 add_arg(lines[last_match:end-1]) 150 result = macro.expand(mapping) 151 # Replace the occurrence of the macro with the expansion 152 lines = lines[:start] + result + lines[end:] 153 pattern_match = name_pattern.search(lines, start + len(result)) 154 return lines 155 156 class TextMacro: 157 def __init__(self, args, body): 158 self.args = args 159 self.body = body 160 def expand(self, mapping): 161 result = self.body 162 for key, value in mapping.items(): 163 result = result.replace(key, value) 164 return result 165 166 class PythonMacro: 167 def __init__(self, args, fun): 168 self.args = args 169 self.fun = fun 170 def expand(self, mapping): 171 args = [] 172 for arg in self.args: 173 args.append(mapping[arg]) 174 return str(self.fun(*args)) 175 176 CONST_PATTERN = re.compile(r'^const\s+([a-zA-Z0-9_]+)\s*=\s*([^;]*);$') 177 MACRO_PATTERN = re.compile(r'^macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$') 178 PYTHON_MACRO_PATTERN = re.compile(r'^python\s+macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$') 179 180 181 def ReadMacros(lines): 182 constants = [] 183 macros = [] 184 for line in lines: 185 hash = line.find('#') 186 if hash != -1: line = line[:hash] 187 line = line.strip() 188 if len(line) is 0: continue 189 const_match = CONST_PATTERN.match(line) 190 if const_match: 191 name = const_match.group(1) 192 value = const_match.group(2).strip() 193 constants.append((re.compile("\\b%s\\b" % name), value)) 194 else: 195 macro_match = MACRO_PATTERN.match(line) 196 if macro_match: 197 name = macro_match.group(1) 198 args = map(string.strip, macro_match.group(2).split(',')) 199 body = macro_match.group(3).strip() 200 macros.append((re.compile("\\b%s\\(" % name), TextMacro(args, body))) 201 else: 202 python_match = PYTHON_MACRO_PATTERN.match(line) 203 if python_match: 204 name = python_match.group(1) 205 args = map(string.strip, python_match.group(2).split(',')) 206 body = python_match.group(3).strip() 207 fun = eval("lambda " + ",".join(args) + ': ' + body) 208 macros.append((re.compile("\\b%s\\(" % name), PythonMacro(args, fun))) 209 else: 210 raise ("Illegal line: " + line) 211 return (constants, macros) 212 213 214 HEADER_TEMPLATE = """\ 215 // Copyright 2011 Google Inc. All Rights Reserved. 216 217 // This file was generated from .js source files by SCons. If you 218 // want to make changes to this file you should either change the 219 // javascript source files or the SConstruct script. 220 221 #include "v8.h" 222 #include "natives.h" 223 #include "utils.h" 224 225 namespace v8 { 226 namespace internal { 227 228 static const byte sources[] = { %(sources_data)s }; 229 230 %(raw_sources_declaration)s\ 231 232 template <> 233 int NativesCollection<%(type)s>::GetBuiltinsCount() { 234 return %(builtin_count)i; 235 } 236 237 template <> 238 int NativesCollection<%(type)s>::GetDebuggerCount() { 239 return %(debugger_count)i; 240 } 241 242 template <> 243 int NativesCollection<%(type)s>::GetIndex(const char* name) { 244 %(get_index_cases)s\ 245 return -1; 246 } 247 248 template <> 249 int NativesCollection<%(type)s>::GetRawScriptsSize() { 250 return %(raw_total_length)i; 251 } 252 253 template <> 254 Vector<const char> NativesCollection<%(type)s>::GetRawScriptSource(int index) { 255 %(get_raw_script_source_cases)s\ 256 return Vector<const char>("", 0); 257 } 258 259 template <> 260 Vector<const char> NativesCollection<%(type)s>::GetScriptName(int index) { 261 %(get_script_name_cases)s\ 262 return Vector<const char>("", 0); 263 } 264 265 template <> 266 Vector<const byte> NativesCollection<%(type)s>::GetScriptsSource() { 267 return Vector<const byte>(sources, %(total_length)i); 268 } 269 270 template <> 271 void NativesCollection<%(type)s>::SetRawScriptsSource(Vector<const char> raw_source) { 272 ASSERT(%(raw_total_length)i == raw_source.length()); 273 raw_sources = raw_source.start(); 274 } 275 276 } // internal 277 } // v8 278 """ 279 280 281 RAW_SOURCES_COMPRESSION_DECLARATION = """\ 282 static const char* raw_sources = NULL; 283 """ 284 285 286 RAW_SOURCES_DECLARATION = """\ 287 static const char* raw_sources = reinterpret_cast<const char*>(sources); 288 """ 289 290 291 GET_INDEX_CASE = """\ 292 if (strcmp(name, "%(id)s") == 0) return %(i)i; 293 """ 294 295 296 GET_RAW_SCRIPT_SOURCE_CASE = """\ 297 if (index == %(i)i) return Vector<const char>(raw_sources + %(offset)i, %(raw_length)i); 298 """ 299 300 301 GET_SCRIPT_NAME_CASE = """\ 302 if (index == %(i)i) return Vector<const char>("%(name)s", %(length)i); 303 """ 304 305 def JS2C(source, target, env): 306 ids = [] 307 debugger_ids = [] 308 modules = [] 309 # Locate the macros file name. 310 consts = [] 311 macros = [] 312 for s in source: 313 if 'macros.py' == (os.path.split(str(s))[1]): 314 (consts, macros) = ReadMacros(ReadLines(str(s))) 315 else: 316 modules.append(s) 317 318 minifier = jsmin.JavaScriptMinifier() 319 320 module_offset = 0 321 all_sources = [] 322 for module in modules: 323 filename = str(module) 324 debugger = filename.endswith('-debugger.js') 325 lines = ReadFile(filename) 326 lines = ExpandConstants(lines, consts) 327 lines = ExpandMacros(lines, macros) 328 Validate(lines, filename) 329 lines = minifier.JSMinify(lines) 330 id = (os.path.split(filename)[1])[:-3] 331 if debugger: id = id[:-9] 332 raw_length = len(lines) 333 if debugger: 334 debugger_ids.append((id, raw_length, module_offset)) 335 else: 336 ids.append((id, raw_length, module_offset)) 337 all_sources.append(lines) 338 module_offset += raw_length 339 total_length = raw_total_length = module_offset 340 341 if env['COMPRESSION'] == 'off': 342 raw_sources_declaration = RAW_SOURCES_DECLARATION 343 sources_data = ToCAsciiArray("".join(all_sources)) 344 else: 345 raw_sources_declaration = RAW_SOURCES_COMPRESSION_DECLARATION 346 if env['COMPRESSION'] == 'bz2': 347 all_sources = bz2.compress("".join(all_sources)) 348 total_length = len(all_sources) 349 sources_data = ToCArray(all_sources) 350 351 # Build debugger support functions 352 get_index_cases = [ ] 353 get_raw_script_source_cases = [ ] 354 get_script_name_cases = [ ] 355 356 i = 0 357 for (id, raw_length, module_offset) in debugger_ids + ids: 358 native_name = "native %s.js" % id 359 get_index_cases.append(GET_INDEX_CASE % { 'id': id, 'i': i }) 360 get_raw_script_source_cases.append(GET_RAW_SCRIPT_SOURCE_CASE % { 361 'offset': module_offset, 362 'raw_length': raw_length, 363 'i': i 364 }) 365 get_script_name_cases.append(GET_SCRIPT_NAME_CASE % { 366 'name': native_name, 367 'length': len(native_name), 368 'i': i 369 }) 370 i = i + 1 371 372 # Emit result 373 output = open(str(target[0]), "w") 374 output.write(HEADER_TEMPLATE % { 375 'builtin_count': len(ids) + len(debugger_ids), 376 'debugger_count': len(debugger_ids), 377 'sources_data': sources_data, 378 'raw_sources_declaration': raw_sources_declaration, 379 'raw_total_length': raw_total_length, 380 'total_length': total_length, 381 'get_index_cases': "".join(get_index_cases), 382 'get_raw_script_source_cases': "".join(get_raw_script_source_cases), 383 'get_script_name_cases': "".join(get_script_name_cases), 384 'type': env['TYPE'] 385 }) 386 output.close() 387 388 def main(): 389 natives = sys.argv[1] 390 type = sys.argv[2] 391 compression = sys.argv[3] 392 source_files = sys.argv[4:] 393 JS2C(source_files, [natives], { 'TYPE': type, 'COMPRESSION': compression }) 394 395 if __name__ == "__main__": 396 main() 397
