1 # -*- python -*- 2 # ex: set syntax=python: 3 4 c = BuildmasterConfig = {} 5 6 from buildbot.buildslave import BuildSlave 7 from buildbot.changes.pb import PBChangeSource 8 from buildbot.scheduler import AnyBranchScheduler, Triggerable 9 from buildbot.status import html 10 from buildbot.process import buildstep, factory, properties 11 from buildbot.steps import master, shell, source, transfer, trigger 12 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED 13 14 from twisted.internet import defer 15 16 import re 17 import simplejson 18 19 WithProperties = properties.WithProperties 20 21 class ConfigureBuild(buildstep.BuildStep): 22 name = "configure build" 23 description = ["configuring build"] 24 descriptionDone = ["configured build"] 25 def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs): 26 buildstep.BuildStep.__init__(self, *args, **kwargs) 27 self.platform = platform.split('-', 1)[0] 28 self.fullPlatform = platform 29 self.configuration = configuration 30 self.architecture = architecture 31 self.buildOnly = buildOnly 32 self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly) 33 34 def start(self): 35 self.setProperty("platform", self.platform) 36 self.setProperty("fullPlatform", self.fullPlatform) 37 self.setProperty("configuration", self.configuration) 38 self.setProperty("architecture", self.architecture) 39 self.setProperty("buildOnly", self.buildOnly) 40 self.finished(SUCCESS) 41 return defer.succeed(None) 42 43 44 class CheckOutSource(source.SVN): 45 baseURL = "http://svn.webkit.org/repository/webkit/" 46 mode = "update" 47 def __init__(self, *args, **kwargs): 48 source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs) 49 50 51 class InstallWin32Dependencies(shell.Compile): 52 description = ["installing dependencies"] 53 descriptionDone = ["installed dependencies"] 54 command = ["perl", "./WebKitTools/Scripts/update-webkit-auxiliary-libs"] 55 56 57 class InstallChromiumDependencies(shell.ShellCommand): 58 name = "gclient" 59 description = ["updating chromium dependencies"] 60 descriptionDone = ["updated chromium dependencies"] 61 command = ["perl", "./WebKitTools/Scripts/update-webkit-chromium"] 62 haltOnFailure = True 63 64 65 def appendCustomBuildFlags(step, platform): 66 if platform in ('gtk', 'wx', 'qt', 'chromium'): 67 step.setCommand(step.command + ['--' + platform]) 68 69 70 class CompileWebKit(shell.Compile): 71 command = ["perl", "./WebKitTools/Scripts/build-webkit", WithProperties("--%(configuration)s")] 72 env = {'MFLAGS':''} 73 name = "compile-webkit" 74 description = ["compiling"] 75 descriptionDone = ["compiled"] 76 warningPattern = ".*arning: .*" 77 78 def start(self): 79 platform = self.getProperty('platform') 80 buildOnly = self.getProperty('buildOnly') 81 if platform == 'mac' and buildOnly: 82 self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym']) 83 84 appendCustomBuildFlags(self, platform) 85 return shell.Compile.start(self) 86 87 88 class ArchiveBuiltProduct(shell.ShellCommand): 89 command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive", 90 WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] 91 name = "archive-built-product" 92 description = ["archiving built product"] 93 descriptionDone = ["archived built product"] 94 haltOnFailure = True 95 96 97 class ExtractBuiltProduct(shell.ShellCommand): 98 command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive", 99 WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"] 100 name = "extract-built-product" 101 description = ["extracting built product"] 102 descriptionDone = ["extracted built product"] 103 haltOnFailure = True 104 105 106 class UploadBuiltProduct(transfer.FileUpload): 107 slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip") 108 masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") 109 haltOnFailure = True 110 111 def __init__(self): 112 transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) 113 114 115 class DownloadBuiltProduct(transfer.FileDownload): 116 slavedest = WithProperties("WebKitBuild/%(configuration)s.zip") 117 mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip") 118 haltOnFailure = True 119 120 def __init__(self): 121 transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest) 122 123 124 class RunJavaScriptCoreTests(shell.Test): 125 name = "jscore-test" 126 description = ["jscore-tests running"] 127 descriptionDone = ["jscore-tests"] 128 command = ["perl", "./WebKitTools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")] 129 logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'} 130 131 def __init__(self, skipBuild=False, *args, **kwargs): 132 self.skipBuild = skipBuild 133 shell.Test.__init__(self, *args, **kwargs) 134 self.addFactoryArguments(skipBuild=skipBuild) 135 136 def start(self): 137 appendCustomBuildFlags(self, self.getProperty('platform')) 138 if self.skipBuild: 139 self.setCommand(self.command + ['--skip-build']) 140 return shell.Test.start(self) 141 142 def commandComplete(self, cmd): 143 shell.Test.commandComplete(self, cmd) 144 145 logText = cmd.logs['stdio'].getText() 146 statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0] 147 if statusLines and statusLines[0].split()[0] != '0': 148 self.regressionLine = statusLines[0] 149 else: 150 self.regressionLine = None 151 152 def evaluateCommand(self, cmd): 153 if self.regressionLine: 154 return FAILURE 155 156 if cmd.rc != 0: 157 return FAILURE 158 159 return SUCCESS 160 161 def getText(self, cmd, results): 162 return self.getText2(cmd, results) 163 164 def getText2(self, cmd, results): 165 if results != SUCCESS and self.regressionLine: 166 return [self.name, self.regressionLine] 167 168 return [self.name] 169 170 171 class RunWebKitTests(shell.Test): 172 name = "layout-test" 173 description = ["layout-tests running"] 174 descriptionDone = ["layout-tests"] 175 command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results", 176 "--no-sample-on-timeout", "--results-directory", "layout-test-results", "--use-remote-links-to-tests", 177 WithProperties("--%(configuration)s"), "--exit-after-n-failures", "20"] 178 179 def __init__(self, skipBuild=False, *args, **kwargs): 180 self.skipBuild = skipBuild 181 shell.Test.__init__(self, *args, **kwargs) 182 self.addFactoryArguments(skipBuild=skipBuild) 183 184 def start(self): 185 appendCustomBuildFlags(self, self.getProperty('platform')) 186 if self.skipBuild: 187 self.setCommand(self.command + ['--root=WebKitBuild/bin']) 188 return shell.Test.start(self) 189 190 def commandComplete(self, cmd): 191 shell.Test.commandComplete(self, cmd) 192 193 logText = cmd.logs['stdio'].getText() 194 incorrectLayoutLines = [] 195 for line in logText.splitlines(): 196 if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0: 197 incorrectLayoutLines.append(line) 198 elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0): 199 incorrectLayoutLines.append(line) 200 elif line.startswith("WARNING:") and line.find(' leak') >= 0: 201 incorrectLayoutLines.append(line.replace('WARNING: ', '')) 202 elif line.find('Exiting early') >= 0: 203 incorrectLayoutLines.append(line) 204 205 # FIXME: Detect and summarize leaks of RefCounted objects 206 207 self.incorrectLayoutLines = incorrectLayoutLines 208 209 def evaluateCommand(self, cmd): 210 if self.incorrectLayoutLines: 211 if len(self.incorrectLayoutLines) == 1: 212 line = self.incorrectLayoutLines[0] 213 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0: 214 return WARNINGS 215 216 return FAILURE 217 218 if cmd.rc != 0: 219 return FAILURE 220 221 return SUCCESS 222 223 def getText(self, cmd, results): 224 return self.getText2(cmd, results) 225 226 def getText2(self, cmd, results): 227 if results != SUCCESS and self.incorrectLayoutLines: 228 return self.incorrectLayoutLines 229 230 return [self.name] 231 232 233 class RunWebKitLeakTests(RunWebKitTests): 234 def start(self): 235 self.setCommand(self.command + ["--leaks"]) 236 return RunWebKitTests.start(self) 237 238 239 class ArchiveTestResults(shell.ShellCommand): 240 command = ["python", "./WebKitTools/BuildSlaveSupport/test-result-archive", 241 WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"] 242 name = "archive-test-results" 243 description = ["archiving test results"] 244 descriptionDone = ["archived test results"] 245 haltOnFailure = True 246 247 248 class UploadTestResults(transfer.FileUpload): 249 slavesrc = "layout-test-results.zip" 250 masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") 251 252 def __init__(self): 253 transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest) 254 255 256 class ExtractTestResults(master.MasterShellCommand): 257 zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip") 258 resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)") 259 260 def __init__(self): 261 master.MasterShellCommand.__init__(self, "") 262 263 def start(self): 264 self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)] 265 return master.MasterShellCommand.start(self) 266 267 def finished(self, result): 268 url = self.build.getProperties().render(self.resultDirectory).replace("public_html/", "") 269 self.addURL("view results", url) 270 result = master.MasterShellCommand.finished(self, result) 271 self.step_status.setText(["uploaded results"]) 272 return result 273 274 275 class Factory(factory.BuildFactory): 276 def __init__(self, platform, configuration, architectures, buildOnly): 277 factory.BuildFactory.__init__(self) 278 self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly) 279 self.addStep(CheckOutSource) 280 if platform == "win": 281 self.addStep(InstallWin32Dependencies) 282 if platform == "chromium": 283 self.addStep(InstallChromiumDependencies) 284 285 class BuildFactory(Factory): 286 def __init__(self, platform, configuration, architectures, triggers=None): 287 Factory.__init__(self, platform, configuration, architectures, True) 288 self.addStep(CompileWebKit) 289 if triggers: 290 self.addStep(ArchiveBuiltProduct) 291 self.addStep(UploadBuiltProduct) 292 self.addStep(trigger.Trigger, schedulerNames=triggers) 293 294 class TestFactory(Factory): 295 def __init__(self, platform, configuration, architectures): 296 Factory.__init__(self, platform, configuration, architectures, False) 297 self.addStep(DownloadBuiltProduct) 298 self.addStep(ExtractBuiltProduct) 299 self.addStep(RunJavaScriptCoreTests, skipBuild=True) 300 self.addStep(RunWebKitTests, skipBuild=(platform == 'win')) 301 self.addStep(ArchiveTestResults) 302 self.addStep(UploadTestResults) 303 self.addStep(ExtractTestResults) 304 305 class BuildAndTestFactory(Factory): 306 TestClass = RunWebKitTests 307 def __init__(self, platform, configuration, architectures): 308 Factory.__init__(self, platform, configuration, architectures, False) 309 self.addStep(CompileWebKit) 310 self.addStep(RunJavaScriptCoreTests) 311 self.addStep(self.TestClass) 312 self.addStep(ArchiveTestResults) 313 self.addStep(UploadTestResults) 314 self.addStep(ExtractTestResults) 315 316 class BuildAndTestLeaksFactory(BuildAndTestFactory): 317 TestClass = RunWebKitLeakTests 318 319 320 def loadBuilderConfig(c): 321 passwords = simplejson.load(open('passwords.json')) 322 323 config = simplejson.load(open('config.json')) 324 325 c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']] 326 327 c['schedulers'] = [] 328 for scheduler in config['schedulers']: 329 kls = globals()[scheduler.pop('type')] 330 c['schedulers'].append(kls(**scheduler)) 331 332 c['builders'] = [] 333 for builder in config['builders']: 334 for slaveName in builder['slavenames']: 335 for slave in config['slaves']: 336 if slave['name'] != slaveName or slave['platform'] == '*': 337 continue 338 339 if slave['platform'] != builder['platform']: 340 raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform']) 341 342 break 343 344 factory = globals()["%sFactory" % builder.pop('type')] 345 factoryArgs = [] 346 for key in "platform", "configuration", "architectures", "triggers": 347 value = builder.pop(key, None) 348 if value: 349 factoryArgs.append(value) 350 351 builder["factory"] = factory(*factoryArgs) 352 353 c['builders'].append(builder) 354 355 loadBuilderConfig(c) 356 357 c['change_source'] = PBChangeSource() 358 359 c['status'] = [] 360 c['status'].append(html.WebStatus(http_port=8710, allowForce=True)) 361 362 c['slavePortnum'] = 17000 363 c['projectName'] = "WebKit" 364 c['projectURL'] = "http://webkit.org" 365 c['buildbotURL'] = "http://build.webkit.org/" 366
