1 # -*- coding: utf-8 -*- 2 3 import os 4 import re 5 import sys 6 import copy 7 import zlib 8 import time 9 import shlex 10 import shutil 11 import fnmatch 12 import tarfile 13 import argparse 14 import platform 15 import datetime 16 import tempfile 17 import posixpath 18 import subprocess 19 20 from build.common import * 21 from build.config import * 22 from build.build import * 23 24 def die (msg): 25 print msg 26 sys.exit(-1) 27 28 def removeLeadingPath (path, basePath): 29 # Both inputs must be normalized already 30 assert os.path.normpath(path) == path 31 assert os.path.normpath(basePath) == basePath 32 return path[len(basePath) + 1:] 33 34 def findFile (candidates): 35 for file in candidates: 36 if os.path.exists(file): 37 return file 38 return None 39 40 def getFileList (basePath): 41 allFiles = [] 42 basePath = os.path.normpath(basePath) 43 for root, dirs, files in os.walk(basePath): 44 for file in files: 45 relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath) 46 allFiles.append(relPath) 47 return allFiles 48 49 def toDatetime (dateTuple): 50 Y, M, D = dateTuple 51 return datetime.datetime(Y, M, D) 52 53 class PackageBuildInfo: 54 def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath): 55 self.releaseConfig = releaseConfig 56 self.srcBasePath = srcBasePath 57 self.dstBasePath = dstBasePath 58 self.tmpBasePath = tmpBasePath 59 60 def getReleaseConfig (self): 61 return self.releaseConfig 62 63 def getReleaseVersion (self): 64 return self.releaseConfig.getVersion() 65 66 def getReleaseId (self): 67 # Release id is crc32(releaseConfig + release) 68 return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff 69 70 def getSrcBasePath (self): 71 return self.srcBasePath 72 73 def getTmpBasePath (self): 74 return self.tmpBasePath 75 76 class DstFile (object): 77 def __init__ (self, dstFile): 78 self.dstFile = dstFile 79 80 def makeDir (self): 81 dirName = os.path.dirname(self.dstFile) 82 if not os.path.exists(dirName): 83 os.makedirs(dirName) 84 85 def make (self, packageBuildInfo): 86 assert False # Should not be called 87 88 class CopyFile (DstFile): 89 def __init__ (self, srcFile, dstFile): 90 super(CopyFile, self).__init__(dstFile) 91 self.srcFile = srcFile 92 93 def make (self, packageBuildInfo): 94 self.makeDir() 95 if os.path.exists(self.dstFile): 96 die("%s already exists" % self.dstFile) 97 shutil.copyfile(self.srcFile, self.dstFile) 98 99 class GenInfoFile (DstFile): 100 def __init__ (self, srcFile, dstFile): 101 super(GenInfoFile, self).__init__(dstFile) 102 self.srcFile = srcFile 103 104 def make (self, packageBuildInfo): 105 self.makeDir() 106 print " GenInfoFile: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath) 107 src = readFile(self.srcFile) 108 109 for define, value in [("DEQP_RELEASE_NAME", "\"%s\"" % packageBuildInfo.getReleaseVersion()), 110 ("DEQP_RELEASE_ID", "0x%08x" % packageBuildInfo.getReleaseId())]: 111 src = re.sub('(#\s*define\s+%s\s+)[^\n\r]+' % re.escape(define), r'\1 %s' % value, src) 112 113 writeFile(self.dstFile, src) 114 115 class GenCMake (DstFile): 116 def __init__ (self, srcFile, dstFile, replaceVars): 117 super(GenCMake, self).__init__(dstFile) 118 self.srcFile = srcFile 119 self.replaceVars = replaceVars 120 121 def make (self, packageBuildInfo): 122 self.makeDir() 123 print " GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath) 124 src = readFile(self.srcFile) 125 for var, value in self.replaceVars: 126 src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var), 127 'set(%s "%s"' % (var, value), src) 128 writeFile(self.dstFile, src) 129 130 def createFileTargets (srcBasePath, dstBasePath, files, filters): 131 usedFiles = set() # Files that are already included by other filters 132 targets = [] 133 134 for isMatch, createFileObj in filters: 135 # Build list of files that match filter 136 matchingFiles = [] 137 for file in files: 138 if not file in usedFiles and isMatch(file): 139 matchingFiles.append(file) 140 141 # Build file objects, add to used set 142 for file in matchingFiles: 143 usedFiles.add(file) 144 targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file))) 145 146 return targets 147 148 # Generates multiple file targets based on filters 149 class FileTargetGroup: 150 def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath): 151 self.srcBasePath = srcBasePath 152 self.dstBasePath = dstBasePath 153 self.filters = filters 154 self.getSrcBasePath = srcBasePathFunc 155 156 def make (self, packageBuildInfo): 157 fullSrcPath = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath)) 158 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath)) 159 160 allFiles = getFileList(fullSrcPath) 161 targets = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters) 162 163 # Make all file targets 164 for file in targets: 165 file.make(packageBuildInfo) 166 167 # Single file target 168 class SingleFileTarget: 169 def __init__ (self, srcFile, dstFile, makeTarget): 170 self.srcFile = srcFile 171 self.dstFile = dstFile 172 self.makeTarget = makeTarget 173 174 def make (self, packageBuildInfo): 175 fullSrcPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile)) 176 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile)) 177 178 target = self.makeTarget(fullSrcPath, fullDstPath) 179 target.make(packageBuildInfo) 180 181 class BuildTarget: 182 def __init__ (self, baseConfig, generator, targets = None): 183 self.baseConfig = baseConfig 184 self.generator = generator 185 self.targets = targets 186 187 def make (self, packageBuildInfo): 188 print " Building %s" % self.baseConfig.getBuildDir() 189 190 # Create config with full build dir path 191 config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()), 192 self.baseConfig.getBuildType(), 193 self.baseConfig.getArgs(), 194 srcPath = os.path.join(packageBuildInfo.dstBasePath, "src")) 195 196 assert not os.path.exists(config.getBuildDir()) 197 build(config, self.generator, self.targets) 198 199 class BuildAndroidTarget: 200 def __init__ (self, dstFile): 201 self.dstFile = dstFile 202 203 def make (self, packageBuildInfo): 204 print " Building Android binary" 205 206 buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build") 207 208 assert not os.path.exists(buildRoot) 209 os.makedirs(buildRoot) 210 211 # Execute build script 212 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py")) 213 execute([ 214 "python", 215 "-B", # no .py[co] 216 scriptPath, 217 "--build-root=%s" % buildRoot, 218 ]) 219 220 srcFile = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk")) 221 dstFile = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile)) 222 223 CopyFile(srcFile, dstFile).make(packageBuildInfo) 224 225 class FetchExternalSourcesTarget: 226 def __init__ (self): 227 pass 228 229 def make (self, packageBuildInfo): 230 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py")) 231 execute([ 232 "python", 233 "-B", # no .py[co] 234 scriptPath, 235 ]) 236 237 class RemoveSourcesTarget: 238 def __init__ (self): 239 pass 240 241 def make (self, packageBuildInfo): 242 shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False) 243 244 class Module: 245 def __init__ (self, name, targets): 246 self.name = name 247 self.targets = targets 248 249 def make (self, packageBuildInfo): 250 for target in self.targets: 251 target.make(packageBuildInfo) 252 253 class ReleaseConfig: 254 def __init__ (self, name, version, modules, sources = True): 255 self.name = name 256 self.version = version 257 self.modules = modules 258 self.sources = sources 259 260 def getName (self): 261 return self.name 262 263 def getVersion (self): 264 return self.version 265 266 def getModules (self): 267 return self.modules 268 269 def packageWithSources (self): 270 return self.sources 271 272 def matchIncludeExclude (includePatterns, excludePatterns, filename): 273 components = os.path.normpath(filename).split(os.sep) 274 for pattern in excludePatterns: 275 for component in components: 276 if fnmatch.fnmatch(component, pattern): 277 return False 278 279 for pattern in includePatterns: 280 for component in components: 281 if fnmatch.fnmatch(component, pattern): 282 return True 283 284 return False 285 286 def copyFileFilter (includePatterns, excludePatterns=[]): 287 return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f), 288 lambda s, d: CopyFile(s, d)) 289 290 def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]): 291 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)]) 292 293 def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]): 294 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath) 295 296 def makeFileCopy (srcFile, dstFile): 297 return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d)) 298 299 def getReleaseFileName (configName, releaseName): 300 today = datetime.date.today() 301 return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName) 302 303 def getTempDir (): 304 dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases") 305 if not os.path.exists(dirName): 306 os.makedirs(dirName) 307 return dirName 308 309 def makeRelease (releaseConfig): 310 releaseName = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion()) 311 tmpPath = getTempDir() 312 srcBasePath = DEQP_DIR 313 dstBasePath = os.path.join(tmpPath, releaseName) 314 tmpBasePath = os.path.join(tmpPath, releaseName + "-tmp") 315 packageBuildInfo = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath) 316 dstArchiveName = releaseName + ".tar.bz2" 317 318 print "Creating release %s to %s" % (releaseName, tmpPath) 319 320 # Remove old temporary dirs 321 for path in [dstBasePath, tmpBasePath]: 322 if os.path.exists(path): 323 shutil.rmtree(path, ignore_errors=False) 324 325 # Make all modules 326 for module in releaseConfig.getModules(): 327 print " Processing module %s" % module.name 328 module.make(packageBuildInfo) 329 330 # Remove sources? 331 if not releaseConfig.packageWithSources(): 332 shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False) 333 334 # Create archive 335 print "Creating %s" % dstArchiveName 336 archive = tarfile.open(dstArchiveName, 'w:bz2') 337 archive.add(dstBasePath, arcname=releaseName) 338 archive.close() 339 340 # Remove tmp dirs 341 for path in [dstBasePath, tmpBasePath]: 342 if os.path.exists(path): 343 shutil.rmtree(path, ignore_errors=False) 344 345 print "Done!" 346 347 # Module declarations 348 349 SRC_FILE_PATTERNS = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"] 350 TARGET_PATTERNS = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"] 351 352 BASE = Module("Base", [ 353 makeFileCopy ("LICENSE", "src/LICENSE"), 354 makeFileCopy ("CMakeLists.txt", "src/CMakeLists.txt"), 355 makeFileCopyGroup ("targets", "src/targets", TARGET_PATTERNS), 356 makeFileCopyGroup ("execserver", "src/execserver", SRC_FILE_PATTERNS), 357 makeFileCopyGroup ("executor", "src/executor", SRC_FILE_PATTERNS), 358 makeFileCopy ("modules/CMakeLists.txt", "src/modules/CMakeLists.txt"), 359 makeFileCopyGroup ("external", "src/external", ["CMakeLists.txt", "*.py"]), 360 361 # Stylesheet for displaying test logs on browser 362 makeFileCopyGroup ("doc/testlog-stylesheet", "doc/testlog-stylesheet", ["*"]), 363 364 # Non-optional parts of framework 365 # \note qpInfo.c must be processed! 366 FileTargetGroup ("framework", "src/framework", [ 367 # If file is qpInfo.c use GenInfoFile 368 (lambda f: f[-8:] == "qpInfo.c", lambda s, d: GenInfoFile(s, d)), 369 # Otherwise just CopyFile targets 370 copyFileFilter(SRC_FILE_PATTERNS, ["imagedifftester", "randomshaders", "simplereference", "referencerenderer"]) 371 ]), 372 373 # Main modules CMakeLists.txt 374 375 # android sources 376 makeFileCopyGroup ("android/package/src", "src/android/package/src", SRC_FILE_PATTERNS), 377 makeFileCopy ("android/package/AndroidManifest.xml", "src/android/package/AndroidManifest.xml"), 378 makeFileCopyGroup ("android/package/res", "src/android/package/res", ["*.png", "*.xml"]), 379 makeFileCopyGroup ("android/scripts", "src/android/scripts", [ 380 "common.py", 381 "build.py", 382 "resources.py", 383 "install.py", 384 "launch.py", 385 "debug.py" 386 ]), 387 ]) 388 389 DOCUMENTATION = Module("Documentation", [ 390 makeFileCopyGroup ("doc/pdf", "doc", ["*.pdf"]), 391 makeFileCopyGroup ("doc", "doc", ["porting_layer_changes_*.txt"]), 392 ]) 393 394 GLSHARED = Module("Shared GL Tests", [ 395 # Optional framework components 396 makeFileCopyGroup ("framework/randomshaders", "src/framework/randomshaders", SRC_FILE_PATTERNS), 397 makeFileCopyGroup ("framework/opengl/simplereference", "src/framework/opengl/simplereference", SRC_FILE_PATTERNS), 398 makeFileCopyGroup ("framework/referencerenderer", "src/framework/referencerenderer", SRC_FILE_PATTERNS), 399 400 makeFileCopyGroup ("modules/glshared", "src/modules/glshared", SRC_FILE_PATTERNS), 401 ]) 402 403 GLES2 = Module("GLES2", [ 404 makeFileCopyGroup ("modules/gles2", "src/modules/gles2", SRC_FILE_PATTERNS), 405 makeFileCopyGroup ("data/gles2", "src/data/gles2", ["*.*"]), 406 makeFileCopyGroup ("doc/testspecs/GLES2", "doc/testspecs/GLES2", ["*.txt"]) 407 ]) 408 409 GLES3 = Module("GLES3", [ 410 makeFileCopyGroup ("modules/gles3", "src/modules/gles3", SRC_FILE_PATTERNS), 411 makeFileCopyGroup ("data/gles3", "src/data/gles3", ["*.*"]), 412 makeFileCopyGroup ("doc/testspecs/GLES3", "doc/testspecs/GLES3", ["*.txt"]) 413 ]) 414 415 GLES31 = Module("GLES31", [ 416 makeFileCopyGroup ("modules/gles31", "src/modules/gles31", SRC_FILE_PATTERNS), 417 makeFileCopyGroup ("data/gles31", "src/data/gles31", ["*.*"]), 418 makeFileCopyGroup ("doc/testspecs/GLES31", "doc/testspecs/GLES31", ["*.txt"]) 419 ]) 420 421 EGL = Module("EGL", [ 422 makeFileCopyGroup ("modules/egl", "src/modules/egl", SRC_FILE_PATTERNS) 423 ]) 424 425 INTERNAL = Module("Internal", [ 426 makeFileCopyGroup ("modules/internal", "src/modules/internal", SRC_FILE_PATTERNS), 427 makeFileCopyGroup ("data/internal", "src/data/internal", ["*.*"]), 428 ]) 429 430 EXTERNAL_SRCS = Module("External sources", [ 431 FetchExternalSourcesTarget() 432 ]) 433 434 ANDROID_BINARIES = Module("Android Binaries", [ 435 BuildAndroidTarget ("bin/android/dEQP.apk"), 436 makeFileCopyGroup ("targets/android", "bin/android", ["*.bat", "*.sh"]), 437 ]) 438 439 COMMON_BUILD_ARGS = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))] 440 NULL_X32_CONFIG = BuildConfig('null-x32', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS) 441 NULL_X64_CONFIG = BuildConfig('null-x64', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS) 442 GLX_X32_CONFIG = BuildConfig('glx-x32', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS) 443 GLX_X64_CONFIG = BuildConfig('glx-x64', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS) 444 445 EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"] 446 447 LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [ 448 BuildTarget (NULL_X32_CONFIG, ANY_UNIX_GENERATOR), 449 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES), 450 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES), 451 ]) 452 453 LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [ 454 BuildTarget (NULL_X64_CONFIG, ANY_UNIX_GENERATOR), 455 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES), 456 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES), 457 ]) 458 459 # Special module to remove src dir, for example after binary build 460 REMOVE_SOURCES = Module("Remove sources from package", [ 461 RemoveSourcesTarget() 462 ]) 463 464 # Release configuration 465 466 ALL_MODULES = [ 467 BASE, 468 DOCUMENTATION, 469 GLSHARED, 470 GLES2, 471 GLES3, 472 GLES31, 473 EGL, 474 INTERNAL, 475 EXTERNAL_SRCS, 476 ] 477 478 ALL_BINARIES = [ 479 LINUX_X64_COMMON_BINARIES, 480 ANDROID_BINARIES, 481 ] 482 483 RELEASE_CONFIGS = { 484 "src": ALL_MODULES, 485 "src-bin": ALL_MODULES + ALL_BINARIES, 486 "bin": ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES], 487 } 488 489 def parseArgs (): 490 parser = argparse.ArgumentParser(description = "Build release package") 491 parser.add_argument("-c", 492 "--config", 493 dest="config", 494 choices=RELEASE_CONFIGS.keys(), 495 required=True, 496 help="Release configuration") 497 parser.add_argument("-n", 498 "--name", 499 dest="name", 500 required=True, 501 help="Package-specific name") 502 parser.add_argument("-v", 503 "--version", 504 dest="version", 505 required=True, 506 help="Version code") 507 return parser.parse_args() 508 509 if __name__ == "__main__": 510 args = parseArgs() 511 config = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config]) 512 makeRelease(config) 513
