1 // Copyright 2016 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 package main 6 7 /* 8 Generate the tasks.json file. 9 */ 10 11 import ( 12 "encoding/json" 13 "flag" 14 "fmt" 15 "io/ioutil" 16 "os" 17 "path" 18 "path/filepath" 19 "regexp" 20 "runtime" 21 "sort" 22 "strconv" 23 "strings" 24 "time" 25 26 "github.com/skia-dev/glog" 27 "go.skia.org/infra/go/sklog" 28 "go.skia.org/infra/go/util" 29 "go.skia.org/infra/task_scheduler/go/specs" 30 ) 31 32 const ( 33 BUNDLE_RECIPES_NAME = "Housekeeper-PerCommit-BundleRecipes" 34 ISOLATE_SKIMAGE_NAME = "Housekeeper-PerCommit-IsolateSkImage" 35 ISOLATE_SKP_NAME = "Housekeeper-PerCommit-IsolateSKP" 36 ISOLATE_SVG_NAME = "Housekeeper-PerCommit-IsolateSVG" 37 ISOLATE_NDK_LINUX_NAME = "Housekeeper-PerCommit-IsolateAndroidNDKLinux" 38 ISOLATE_SDK_LINUX_NAME = "Housekeeper-PerCommit-IsolateAndroidSDKLinux" 39 ISOLATE_WIN_TOOLCHAIN_NAME = "Housekeeper-PerCommit-IsolateWinToolchain" 40 ISOLATE_WIN_VULKAN_SDK_NAME = "Housekeeper-PerCommit-IsolateWinVulkanSDK" 41 42 DEFAULT_OS_DEBIAN = "Debian-9.1" 43 DEFAULT_OS_LINUX_GCE = "Debian-9.2" 44 DEFAULT_OS_MAC = "Mac-10.13.3" 45 DEFAULT_OS_UBUNTU = "Ubuntu-14.04" 46 DEFAULT_OS_WIN = "Windows-2016Server-14393" 47 48 // Name prefix for upload jobs. 49 PREFIX_UPLOAD = "Upload" 50 ) 51 52 var ( 53 // "Constants" 54 55 // Top-level list of all jobs to run at each commit; loaded from 56 // jobs.json. 57 JOBS []string 58 59 // General configuration information. 60 CONFIG struct { 61 GsBucketCoverage string `json:"gs_bucket_coverage"` 62 GsBucketGm string `json:"gs_bucket_gm"` 63 GsBucketNano string `json:"gs_bucket_nano"` 64 GsBucketCalm string `json:"gs_bucket_calm"` 65 NoUpload []string `json:"no_upload"` 66 Pool string `json:"pool"` 67 } 68 69 // alternateSwarmDimensions can be set in an init function to override the default swarming bot 70 // dimensions for the given task. 71 alternateSwarmDimensions func(parts map[string]string) []string 72 73 // internalHardwareLabelFn can be set in an init function to provide an 74 // internal_hardware_label variable to the recipe. 75 internalHardwareLabelFn func(parts map[string]string) *int 76 77 // Defines the structure of job names. 78 jobNameSchema *JobNameSchema 79 80 // Git 2.13. 81 cipdGit1 = &specs.CipdPackage{ 82 Name: fmt.Sprintf("infra/git/${platform}"), 83 Path: "git", 84 Version: fmt.Sprintf("version:2.13.0.chromium9"), 85 } 86 cipdGit2 = &specs.CipdPackage{ 87 Name: fmt.Sprintf("infra/tools/git/${platform}"), 88 Path: "git", 89 Version: fmt.Sprintf("git_revision:a78b5f3658c0578a017db48df97d20ac09822bcd"), 90 } 91 92 // Flags. 93 builderNameSchemaFile = flag.String("builder_name_schema", "", "Path to the builder_name_schema.json file. If not specified, uses infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json from this repo.") 94 assetsDir = flag.String("assets_dir", "", "Directory containing assets.") 95 cfgFile = flag.String("cfg_file", "", "JSON file containing general configuration information.") 96 jobsFile = flag.String("jobs", "", "JSON file containing jobs to run.") 97 ) 98 99 // internalHardwareLabel returns the internal ID for the bot, if any. 100 func internalHardwareLabel(parts map[string]string) *int { 101 if internalHardwareLabelFn != nil { 102 return internalHardwareLabelFn(parts) 103 } 104 return nil 105 } 106 107 // linuxGceDimensions are the Swarming dimensions for Linux GCE 108 // instances. 109 func linuxGceDimensions() []string { 110 return []string{ 111 // Specify CPU to avoid running builds on bots with a more unique CPU. 112 "cpu:x86-64-Haswell_GCE", 113 "gpu:none", 114 fmt.Sprintf("os:%s", DEFAULT_OS_LINUX_GCE), 115 fmt.Sprintf("pool:%s", CONFIG.Pool), 116 } 117 } 118 119 // deriveCompileTaskName returns the name of a compile task based on the given 120 // job name. 121 func deriveCompileTaskName(jobName string, parts map[string]string) string { 122 if strings.Contains(jobName, "Bookmaker") { 123 return "Build-Debian9-GCC-x86_64-Release" 124 } else if parts["role"] == "Housekeeper" { 125 return "Build-Debian9-GCC-x86_64-Release-Shared" 126 } else if parts["role"] == "Test" || parts["role"] == "Perf" || parts["role"] == "Calmbench" { 127 task_os := parts["os"] 128 ec := []string{} 129 if val := parts["extra_config"]; val != "" { 130 ec = strings.Split(val, "_") 131 ignore := []string{"Skpbench", "AbandonGpuContext", "PreAbandonGpuContext", "Valgrind", "ReleaseAndAbandonGpuContext", "CCPR", "FSAA", "FAAA", "FDAA", "NativeFonts", "GDI", "NoGPUThreads"} 132 keep := make([]string, 0, len(ec)) 133 for _, part := range ec { 134 if !util.In(part, ignore) { 135 keep = append(keep, part) 136 } 137 } 138 ec = keep 139 } 140 if task_os == "Android" { 141 if !util.In("Android", ec) { 142 ec = append([]string{"Android"}, ec...) 143 } 144 task_os = "Debian9" 145 } else if task_os == "Chromecast" { 146 task_os = "Debian9" 147 ec = append([]string{"Chromecast"}, ec...) 148 } else if strings.Contains(task_os, "ChromeOS") { 149 ec = append([]string{"Chromebook", "GLES"}, ec...) 150 task_os = "Debian9" 151 } else if task_os == "iOS" { 152 ec = append([]string{task_os}, ec...) 153 task_os = "Mac" 154 } else if strings.Contains(task_os, "Win") { 155 task_os = "Win" 156 } else if strings.Contains(task_os, "Ubuntu") || strings.Contains(task_os, "Debian") { 157 task_os = "Debian9" 158 } 159 jobNameMap := map[string]string{ 160 "role": "Build", 161 "os": task_os, 162 "compiler": parts["compiler"], 163 "target_arch": parts["arch"], 164 "configuration": parts["configuration"], 165 } 166 if len(ec) > 0 { 167 jobNameMap["extra_config"] = strings.Join(ec, "_") 168 } 169 name, err := jobNameSchema.MakeJobName(jobNameMap) 170 if err != nil { 171 glog.Fatal(err) 172 } 173 return name 174 } else { 175 return jobName 176 } 177 } 178 179 // swarmDimensions generates swarming bot dimensions for the given task. 180 func swarmDimensions(parts map[string]string) []string { 181 if alternateSwarmDimensions != nil { 182 return alternateSwarmDimensions(parts) 183 } 184 return defaultSwarmDimensions(parts) 185 } 186 187 // defaultSwarmDimensions generates default swarming bot dimensions for the given task. 188 func defaultSwarmDimensions(parts map[string]string) []string { 189 d := map[string]string{ 190 "pool": CONFIG.Pool, 191 } 192 if os, ok := parts["os"]; ok { 193 d["os"], ok = map[string]string{ 194 "Android": "Android", 195 "Chromecast": "Android", 196 "ChromeOS": "ChromeOS", 197 "Debian9": DEFAULT_OS_DEBIAN, 198 "Mac": DEFAULT_OS_MAC, 199 "Ubuntu14": DEFAULT_OS_UBUNTU, 200 "Ubuntu16": "Ubuntu-16.10", 201 "Ubuntu17": "Ubuntu-17.04", 202 "Win": DEFAULT_OS_WIN, 203 "Win10": "Windows-10-16299.248", 204 "Win2k8": "Windows-2008ServerR2-SP1", 205 "Win2016": DEFAULT_OS_WIN, 206 "Win7": "Windows-7-SP1", 207 "Win8": "Windows-8.1-SP0", 208 "iOS": "iOS-10.3.1", 209 }[os] 210 if !ok { 211 glog.Fatalf("Entry %q not found in OS mapping.", os) 212 } 213 if os == "Win10" && parts["model"] == "Golo" { 214 // Golo/MTV lab bots have Windows 10 version 1703, whereas Skolo bots have Windows 10 version 215 // 1709. 216 d["os"] = "Windows-10-15063" 217 } 218 } else { 219 d["os"] = DEFAULT_OS_DEBIAN 220 } 221 if parts["role"] == "Test" || parts["role"] == "Perf" || parts["role"] == "Calmbench" { 222 if strings.Contains(parts["os"], "Android") || strings.Contains(parts["os"], "Chromecast") { 223 // For Android, the device type is a better dimension 224 // than CPU or GPU. 225 deviceInfo, ok := map[string][]string{ 226 "AndroidOne": {"sprout", "MOB30Q"}, 227 "Chorizo": {"chorizo", "1.30_109591"}, 228 "GalaxyS6": {"zerofltetmo", "NRD90M_G920TUVU5FQK1"}, 229 "GalaxyS7_G930A": {"heroqlteatt", "NRD90M_G930AUCS4BQC2"}, 230 "GalaxyS7_G930FD": {"herolte", "NRD90M_G930FXXU1DQAS"}, 231 "MotoG4": {"athene", "NPJ25.93-14"}, 232 "NVIDIA_Shield": {"foster", "NRD90M_1915764_848"}, 233 "Nexus5": {"hammerhead", "M4B30Z_3437181"}, 234 "Nexus5x": {"bullhead", "OPR6.170623.023"}, 235 "Nexus7": {"grouper", "LMY47V_1836172"}, // 2012 Nexus 7 236 "NexusPlayer": {"fugu", "OPR6.170623.021"}, 237 "Pixel": {"sailfish", "OPM1.171019.016"}, 238 "Pixel2XL": {"taimen", "OPD1.170816.023"}, 239 "PixelC": {"dragon", "OPR1.170623.034"}, 240 }[parts["model"]] 241 if !ok { 242 glog.Fatalf("Entry %q not found in Android mapping.", parts["model"]) 243 } 244 d["device_type"] = deviceInfo[0] 245 d["device_os"] = deviceInfo[1] 246 // TODO(kjlubick): Remove the python dimension after we have removed the 247 // Nexus5x devices from the local lab (on Monday, Dec 11, 2017 should be fine). 248 d["python"] = "2.7.9" // This indicates a RPI, e.g. in Skolo. Golo is 2.7.12 249 if parts["model"] == "Nexus5x" { 250 d["python"] = "2.7.12" 251 } 252 } else if strings.Contains(parts["os"], "iOS") { 253 device, ok := map[string]string{ 254 "iPadMini4": "iPad5,1", 255 "iPhone6": "iPhone7,2", 256 "iPhone7": "iPhone9,1", 257 "iPadPro": "iPad6,3", 258 }[parts["model"]] 259 if !ok { 260 glog.Fatalf("Entry %q not found in iOS mapping.", parts["model"]) 261 } 262 d["device"] = device 263 } else if parts["cpu_or_gpu"] == "CPU" { 264 modelMapping, ok := map[string]map[string]string{ 265 "AVX": { 266 "MacMini7.1": "x86-64-E5-2697_v2", 267 "Golo": "x86-64-E5-2670", 268 }, 269 "AVX2": { 270 "GCE": "x86-64-Haswell_GCE", 271 "NUC5i7RYH": "x86-64-i7-5557U", 272 }, 273 "AVX512": { 274 "GCE": "x86-64-Skylake_GCE", 275 }, 276 }[parts["cpu_or_gpu_value"]] 277 if !ok { 278 glog.Fatalf("Entry %q not found in CPU mapping.", parts["cpu_or_gpu_value"]) 279 } 280 cpu, ok := modelMapping[parts["model"]] 281 if !ok { 282 glog.Fatalf("Entry %q not found in %q model mapping.", parts["model"], parts["cpu_or_gpu_value"]) 283 } 284 d["cpu"] = cpu 285 if parts["model"] == "GCE" && d["os"] == DEFAULT_OS_DEBIAN { 286 d["os"] = DEFAULT_OS_LINUX_GCE 287 } 288 if parts["model"] == "GCE" && d["os"] == DEFAULT_OS_WIN { 289 // Use normal-size machines for Test and Perf tasks on Win GCE. 290 d["machine_type"] = "n1-standard-16" 291 } 292 } else { 293 if strings.Contains(parts["os"], "Win") { 294 gpu, ok := map[string]string{ 295 "GT610": "10de:104a-22.21.13.8205", 296 "GTX1070": "10de:1ba1-23.21.13.9101", 297 "GTX660": "10de:11c0-23.21.13.9101", 298 "GTX960": "10de:1401-23.21.13.9101", 299 "IntelHD4400": "8086:0a16-20.19.15.4835", 300 "IntelIris540": "8086:1926-21.20.16.4590", 301 "IntelIris6100": "8086:162b-20.19.15.4835", 302 "RadeonHD7770": "1002:683d-23.20.15017.4003", 303 "RadeonR9M470X": "1002:6646-23.20.15017.4003", 304 "QuadroP400": "10de:1cb3-22.21.13.8205", 305 }[parts["cpu_or_gpu_value"]] 306 if !ok { 307 glog.Fatalf("Entry %q not found in Win GPU mapping.", parts["cpu_or_gpu_value"]) 308 } 309 d["gpu"] = gpu 310 311 // Specify cpu dimension for NUCs and ShuttleCs. We temporarily have two 312 // types of machines with a GTX960. 313 cpu, ok := map[string]string{ 314 "NUC6i7KYK": "x86-64-i7-6770HQ", 315 "ShuttleC": "x86-64-i7-6700K", 316 }[parts["model"]] 317 if ok { 318 d["cpu"] = cpu 319 } 320 } else if strings.Contains(parts["os"], "Ubuntu") || strings.Contains(parts["os"], "Debian") { 321 gpu, ok := map[string]string{ 322 // Intel drivers come from CIPD, so no need to specify the version here. 323 "IntelBayTrail": "8086:0f31", 324 "IntelHD2000": "8086:0102", 325 "IntelHD405": "8086:22b1", 326 "IntelIris640": "8086:5926", 327 "QuadroP400": "10de:1cb3-384.59", 328 }[parts["cpu_or_gpu_value"]] 329 if !ok { 330 glog.Fatalf("Entry %q not found in Ubuntu GPU mapping.", parts["cpu_or_gpu_value"]) 331 } 332 d["gpu"] = gpu 333 } else if strings.Contains(parts["os"], "Mac") { 334 gpu, ok := map[string]string{ 335 "IntelHD6000": "8086:1626", 336 "IntelHD615": "8086:591e", 337 "IntelIris5100": "8086:0a2e", 338 }[parts["cpu_or_gpu_value"]] 339 if !ok { 340 glog.Fatalf("Entry %q not found in Mac GPU mapping.", parts["cpu_or_gpu_value"]) 341 } 342 d["gpu"] = gpu 343 // Yuck. We have two different types of MacMini7,1 with the same GPU but different CPUs. 344 if parts["cpu_or_gpu_value"] == "IntelIris5100" { 345 // Run all tasks on Golo machines for now. 346 d["cpu"] = "x86-64-i7-4578U" 347 } 348 } else if strings.Contains(parts["os"], "ChromeOS") { 349 version, ok := map[string]string{ 350 "MaliT604": "9901.12.0", 351 "MaliT764": "10172.0.0", 352 "MaliT860": "10172.0.0", 353 "PowerVRGX6250": "10176.5.0", 354 "TegraK1": "10172.0.0", 355 "IntelHDGraphics615": "10032.17.0", 356 }[parts["cpu_or_gpu_value"]] 357 if !ok { 358 glog.Fatalf("Entry %q not found in ChromeOS GPU mapping.", parts["cpu_or_gpu_value"]) 359 } 360 d["gpu"] = parts["cpu_or_gpu_value"] 361 d["release_version"] = version 362 } else { 363 glog.Fatalf("Unknown GPU mapping for OS %q.", parts["os"]) 364 } 365 } 366 } else { 367 d["gpu"] = "none" 368 if d["os"] == DEFAULT_OS_DEBIAN { 369 return linuxGceDimensions() 370 } else if d["os"] == DEFAULT_OS_WIN { 371 // Windows CPU bots. 372 d["cpu"] = "x86-64-Haswell_GCE" 373 // Use many-core machines for Build tasks on Win GCE, except for Goma. 374 if strings.Contains(parts["extra_config"], "Goma") { 375 d["machine_type"] = "n1-standard-16" 376 } else { 377 d["machine_type"] = "n1-highcpu-64" 378 } 379 } else if d["os"] == DEFAULT_OS_MAC { 380 // Mac CPU bots. 381 d["cpu"] = "x86-64-E5-2697_v2" 382 } 383 } 384 385 rv := make([]string, 0, len(d)) 386 for k, v := range d { 387 rv = append(rv, fmt.Sprintf("%s:%s", k, v)) 388 } 389 sort.Strings(rv) 390 return rv 391 } 392 393 // relpath returns the relative path to the given file from the config file. 394 func relpath(f string) string { 395 _, filename, _, _ := runtime.Caller(0) 396 dir := path.Dir(filename) 397 rel := dir 398 if *cfgFile != "" { 399 rel = path.Dir(*cfgFile) 400 } 401 rv, err := filepath.Rel(rel, path.Join(dir, f)) 402 if err != nil { 403 sklog.Fatal(err) 404 } 405 return rv 406 } 407 408 // bundleRecipes generates the task to bundle and isolate the recipes. 409 func bundleRecipes(b *specs.TasksCfgBuilder) string { 410 b.MustAddTask(BUNDLE_RECIPES_NAME, &specs.TaskSpec{ 411 CipdPackages: []*specs.CipdPackage{cipdGit1, cipdGit2}, 412 Dimensions: linuxGceDimensions(), 413 ExtraArgs: []string{ 414 "--workdir", "../../..", "bundle_recipes", 415 fmt.Sprintf("buildername=%s", BUNDLE_RECIPES_NAME), 416 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 417 }, 418 Isolate: relpath("bundle_recipes.isolate"), 419 Priority: 0.7, 420 }) 421 return BUNDLE_RECIPES_NAME 422 } 423 424 // useBundledRecipes returns true iff the given bot should use bundled recipes 425 // instead of syncing recipe DEPS itself. 426 func useBundledRecipes(parts map[string]string) bool { 427 // Use bundled recipes for all test/perf tasks. 428 return true 429 } 430 431 type isolateAssetCfg struct { 432 isolateFile string 433 cipdPkg string 434 } 435 436 var ISOLATE_ASSET_MAPPING = map[string]isolateAssetCfg{ 437 ISOLATE_SKIMAGE_NAME: { 438 isolateFile: "isolate_skimage.isolate", 439 cipdPkg: "skimage", 440 }, 441 ISOLATE_SKP_NAME: { 442 isolateFile: "isolate_skp.isolate", 443 cipdPkg: "skp", 444 }, 445 ISOLATE_SVG_NAME: { 446 isolateFile: "isolate_svg.isolate", 447 cipdPkg: "svg", 448 }, 449 ISOLATE_NDK_LINUX_NAME: { 450 isolateFile: "isolate_ndk_linux.isolate", 451 cipdPkg: "android_ndk_linux", 452 }, 453 ISOLATE_SDK_LINUX_NAME: { 454 isolateFile: "isolate_android_sdk_linux.isolate", 455 cipdPkg: "android_sdk_linux", 456 }, 457 ISOLATE_WIN_TOOLCHAIN_NAME: { 458 isolateFile: "isolate_win_toolchain.isolate", 459 cipdPkg: "win_toolchain", 460 }, 461 ISOLATE_WIN_VULKAN_SDK_NAME: { 462 isolateFile: "isolate_win_vulkan_sdk.isolate", 463 cipdPkg: "win_vulkan_sdk", 464 }, 465 } 466 467 // bundleRecipes generates the task to bundle and isolate the recipes. 468 func isolateCIPDAsset(b *specs.TasksCfgBuilder, name string) string { 469 b.MustAddTask(name, &specs.TaskSpec{ 470 CipdPackages: []*specs.CipdPackage{ 471 b.MustGetCipdPackageFromAsset(ISOLATE_ASSET_MAPPING[name].cipdPkg), 472 }, 473 Dimensions: linuxGceDimensions(), 474 Isolate: relpath(ISOLATE_ASSET_MAPPING[name].isolateFile), 475 Priority: 0.7, 476 }) 477 return name 478 } 479 480 // getIsolatedCIPDDeps returns the slice of Isolate_* tasks a given task needs. 481 // This allows us to save time on I/O bound bots, like the RPIs. 482 func getIsolatedCIPDDeps(parts map[string]string) []string { 483 deps := []string{} 484 // Only do this on the RPIs for now. Other, faster machines shouldn't see much 485 // benefit and we don't need the extra complexity, for now 486 rpiOS := []string{"Android", "ChromeOS", "iOS"} 487 488 if o := parts["os"]; strings.Contains(o, "Chromecast") { 489 // Chromecasts don't have enough disk space to fit all of the content, 490 // so we do a subset of the skps. 491 deps = append(deps, ISOLATE_SKP_NAME) 492 } else if e := parts["extra_config"]; strings.Contains(e, "Skpbench") { 493 // Skpbench only needs skps 494 deps = append(deps, ISOLATE_SKP_NAME) 495 } else if util.In(o, rpiOS) { 496 deps = append(deps, ISOLATE_SKP_NAME) 497 deps = append(deps, ISOLATE_SVG_NAME) 498 deps = append(deps, ISOLATE_SKIMAGE_NAME) 499 } 500 501 return deps 502 } 503 504 // compile generates a compile task. Returns the name of the last task in the 505 // generated chain of tasks, which the Job should add as a dependency. 506 func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) string { 507 // Collect the necessary CIPD packages. 508 pkgs := []*specs.CipdPackage{} 509 deps := []string{} 510 511 // Android bots require a toolchain. 512 if strings.Contains(name, "Android") { 513 if parts["extra_config"] == "Android_Framework" { 514 // Do not need a toolchain when building the 515 // Android Framework. 516 } else if strings.Contains(name, "Mac") { 517 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("android_ndk_darwin")) 518 } else if strings.Contains(name, "Win") { 519 pkg := b.MustGetCipdPackageFromAsset("android_ndk_windows") 520 pkg.Path = "n" 521 pkgs = append(pkgs, pkg) 522 } else { 523 deps = append(deps, isolateCIPDAsset(b, ISOLATE_NDK_LINUX_NAME)) 524 } 525 } else if strings.Contains(name, "Chromecast") { 526 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("cast_toolchain")) 527 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("chromebook_arm_gles")) 528 } else if strings.Contains(name, "Chromebook") { 529 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux")) 530 if parts["target_arch"] == "x86_64" { 531 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("chromebook_x86_64_gles")) 532 } else if parts["target_arch"] == "arm" { 533 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("armhf_sysroot")) 534 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("chromebook_arm_gles")) 535 } 536 } else if strings.Contains(name, "Debian") { 537 if strings.Contains(name, "Clang") { 538 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux")) 539 } 540 if strings.Contains(name, "Vulkan") { 541 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("linux_vulkan_sdk")) 542 } 543 if strings.Contains(name, "EMCC") { 544 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("emscripten_sdk")) 545 } 546 } else if strings.Contains(name, "Win") { 547 deps = append(deps, isolateCIPDAsset(b, ISOLATE_WIN_TOOLCHAIN_NAME)) 548 if strings.Contains(name, "Clang") { 549 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_win")) 550 } 551 if strings.Contains(name, "Vulkan") { 552 deps = append(deps, isolateCIPDAsset(b, ISOLATE_WIN_VULKAN_SDK_NAME)) 553 } 554 } 555 556 dimensions := swarmDimensions(parts) 557 558 // Add the task. 559 b.MustAddTask(name, &specs.TaskSpec{ 560 CipdPackages: pkgs, 561 Dimensions: dimensions, 562 Dependencies: deps, 563 ExtraArgs: []string{ 564 "--workdir", "../../..", "compile", 565 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 566 fmt.Sprintf("buildername=%s", name), 567 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 568 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 569 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 570 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 571 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 572 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 573 }, 574 Isolate: relpath("compile_skia.isolate"), 575 Priority: 0.8, 576 }) 577 // All compile tasks are runnable as their own Job. Assert that the Job 578 // is listed in JOBS. 579 if !util.In(name, JOBS) { 580 glog.Fatalf("Job %q is missing from the JOBS list!", name) 581 } 582 583 // Upload the skiaserve binary only for Linux Android compile bots. 584 // See skbug.com/7399 for context. 585 if parts["configuration"] == "Release" && 586 parts["extra_config"] == "Android" && 587 !strings.Contains(parts["os"], "Win") && 588 !strings.Contains(parts["os"], "Mac") { 589 uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name) 590 b.MustAddTask(uploadName, &specs.TaskSpec{ 591 Dependencies: []string{name}, 592 Dimensions: linuxGceDimensions(), 593 ExtraArgs: []string{ 594 "--workdir", "../../..", "upload_skiaserve", 595 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 596 fmt.Sprintf("buildername=%s", name), 597 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 598 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 599 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 600 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 601 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 602 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 603 }, 604 // We're using the same isolate as upload_dm_results 605 Isolate: relpath("upload_dm_results.isolate"), 606 Priority: 0.8, 607 }) 608 return uploadName 609 } 610 611 return name 612 } 613 614 // recreateSKPs generates a RecreateSKPs task. Returns the name of the last 615 // task in the generated chain of tasks, which the Job should add as a 616 // dependency. 617 func recreateSKPs(b *specs.TasksCfgBuilder, name string) string { 618 b.MustAddTask(name, &specs.TaskSpec{ 619 CipdPackages: []*specs.CipdPackage{b.MustGetCipdPackageFromAsset("go")}, 620 Dimensions: linuxGceDimensions(), 621 ExecutionTimeout: 4 * time.Hour, 622 ExtraArgs: []string{ 623 "--workdir", "../../..", "recreate_skps", 624 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 625 fmt.Sprintf("buildername=%s", name), 626 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 627 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 628 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 629 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 630 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 631 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 632 }, 633 IoTimeout: 40 * time.Minute, 634 Isolate: relpath("compile_skia.isolate"), 635 Priority: 0.8, 636 }) 637 return name 638 } 639 640 // updateMetaConfig generates a UpdateMetaConfig task. Returns the name of the 641 // last task in the generated chain of tasks, which the Job should add as a 642 // dependency. 643 func updateMetaConfig(b *specs.TasksCfgBuilder, name string) string { 644 b.MustAddTask(name, &specs.TaskSpec{ 645 CipdPackages: []*specs.CipdPackage{}, 646 Dimensions: linuxGceDimensions(), 647 ExtraArgs: []string{ 648 "--workdir", "../../..", "update_meta_config", 649 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 650 fmt.Sprintf("buildername=%s", name), 651 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 652 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 653 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 654 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 655 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 656 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 657 }, 658 Isolate: relpath("meta_config.isolate"), 659 Priority: 0.8, 660 }) 661 return name 662 } 663 664 // ctSKPs generates a CT SKPs task. Returns the name of the last task in the 665 // generated chain of tasks, which the Job should add as a dependency. 666 func ctSKPs(b *specs.TasksCfgBuilder, name string) string { 667 b.MustAddTask(name, &specs.TaskSpec{ 668 CipdPackages: []*specs.CipdPackage{}, 669 Dimensions: []string{ 670 "pool:SkiaCT", 671 fmt.Sprintf("os:%s", DEFAULT_OS_LINUX_GCE), 672 }, 673 ExecutionTimeout: 24 * time.Hour, 674 ExtraArgs: []string{ 675 "--workdir", "../../..", "ct_skps", 676 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 677 fmt.Sprintf("buildername=%s", name), 678 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 679 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 680 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 681 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 682 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 683 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 684 }, 685 IoTimeout: time.Hour, 686 Isolate: relpath("ct_skps_skia.isolate"), 687 Priority: 0.8, 688 }) 689 return name 690 } 691 692 // checkGeneratedFiles verifies that no generated SKSL files have been edited 693 // by hand. 694 func checkGeneratedFiles(b *specs.TasksCfgBuilder, name string) string { 695 b.MustAddTask(name, &specs.TaskSpec{ 696 CipdPackages: []*specs.CipdPackage{}, 697 Dimensions: linuxGceDimensions(), 698 ExtraArgs: []string{ 699 "--workdir", "../../..", "check_generated_files", 700 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 701 fmt.Sprintf("buildername=%s", name), 702 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 703 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 704 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 705 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 706 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 707 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 708 }, 709 Isolate: relpath("compile_skia.isolate"), 710 Priority: 0.8, 711 }) 712 return name 713 } 714 715 // housekeeper generates a Housekeeper task. Returns the name of the last task 716 // in the generated chain of tasks, which the Job should add as a dependency. 717 func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string { 718 b.MustAddTask(name, &specs.TaskSpec{ 719 CipdPackages: []*specs.CipdPackage{b.MustGetCipdPackageFromAsset("go")}, 720 Dependencies: []string{compileTaskName}, 721 Dimensions: linuxGceDimensions(), 722 ExtraArgs: []string{ 723 "--workdir", "../../..", "housekeeper", 724 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 725 fmt.Sprintf("buildername=%s", name), 726 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 727 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 728 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 729 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 730 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 731 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 732 }, 733 Isolate: relpath("housekeeper_skia.isolate"), 734 Priority: 0.8, 735 }) 736 return name 737 } 738 739 // bookmaker generates a Bookmaker task. Returns the name of the last task 740 // in the generated chain of tasks, which the Job should add as a dependency. 741 func bookmaker(b *specs.TasksCfgBuilder, name, compileTaskName string) string { 742 b.MustAddTask(name, &specs.TaskSpec{ 743 CipdPackages: []*specs.CipdPackage{b.MustGetCipdPackageFromAsset("go")}, 744 Dependencies: []string{compileTaskName}, 745 Dimensions: linuxGceDimensions(), 746 ExtraArgs: []string{ 747 "--workdir", "../../..", "bookmaker", 748 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 749 fmt.Sprintf("buildername=%s", name), 750 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 751 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 752 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 753 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 754 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 755 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 756 }, 757 Isolate: relpath("compile_skia.isolate"), 758 Priority: 0.8, 759 ExecutionTimeout: 2 * time.Hour, 760 IoTimeout: 2 * time.Hour, 761 }) 762 return name 763 } 764 765 // androidFrameworkCompile generates an Android Framework Compile task. Returns 766 // the name of the last task in the generated chain of tasks, which the Job 767 // should add as a dependency. 768 func androidFrameworkCompile(b *specs.TasksCfgBuilder, name string) string { 769 b.MustAddTask(name, &specs.TaskSpec{ 770 Dimensions: linuxGceDimensions(), 771 ExtraArgs: []string{ 772 "--workdir", "../../..", "android_compile", 773 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 774 fmt.Sprintf("buildername=%s", name), 775 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 776 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 777 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 778 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 779 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 780 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 781 }, 782 Isolate: relpath("compile_skia.isolate"), 783 Priority: 0.8, 784 }) 785 return name 786 } 787 788 // infra generates an infra_tests task. Returns the name of the last task in the 789 // generated chain of tasks, which the Job should add as a dependency. 790 func infra(b *specs.TasksCfgBuilder, name string) string { 791 b.MustAddTask(name, &specs.TaskSpec{ 792 CipdPackages: []*specs.CipdPackage{b.MustGetCipdPackageFromAsset("go")}, 793 Dimensions: linuxGceDimensions(), 794 ExtraArgs: []string{ 795 "--workdir", "../../..", "infra", 796 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 797 fmt.Sprintf("buildername=%s", name), 798 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 799 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 800 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 801 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 802 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 803 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 804 }, 805 Isolate: relpath("infra_skia.isolate"), 806 Priority: 0.8, 807 }) 808 return name 809 } 810 811 func getParentRevisionName(compileTaskName string, parts map[string]string) string { 812 if parts["extra_config"] == "" { 813 return compileTaskName + "-ParentRevision" 814 } else { 815 return compileTaskName + "_ParentRevision" 816 } 817 } 818 819 // calmbench generates a calmbench task. Returns the name of the last task in the 820 // generated chain of tasks, which the Job should add as a dependency. 821 func calmbench(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, compileParentName string) string { 822 s := &specs.TaskSpec{ 823 Dependencies: []string{compileTaskName, compileParentName}, 824 CipdPackages: []*specs.CipdPackage{b.MustGetCipdPackageFromAsset("clang_linux")}, 825 Dimensions: swarmDimensions(parts), 826 ExtraArgs: []string{ 827 "--workdir", "../../..", "calmbench", 828 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 829 fmt.Sprintf("buildername=%s", name), 830 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 831 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 832 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 833 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 834 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 835 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 836 }, 837 Isolate: relpath("calmbench.isolate"), 838 Priority: 0.8, 839 } 840 841 s.Dependencies = append(s.Dependencies, ISOLATE_SKP_NAME, ISOLATE_SVG_NAME) 842 843 b.MustAddTask(name, s) 844 845 // Upload results if necessary. 846 if strings.Contains(name, "Release") && doUpload(name) { 847 uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name) 848 b.MustAddTask(uploadName, &specs.TaskSpec{ 849 Dependencies: []string{name}, 850 Dimensions: linuxGceDimensions(), 851 ExtraArgs: []string{ 852 "--workdir", "../../..", "upload_calmbench_results", 853 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 854 fmt.Sprintf("buildername=%s", name), 855 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 856 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 857 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 858 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 859 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 860 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 861 fmt.Sprintf("gs_bucket=%s", CONFIG.GsBucketCalm), 862 }, 863 // We're using the same isolate as upload_nano_results 864 Isolate: relpath("upload_nano_results.isolate"), 865 Priority: 0.8, 866 }) 867 return uploadName 868 } 869 870 return name 871 } 872 873 // doUpload indicates whether the given Job should upload its results. 874 func doUpload(name string) bool { 875 for _, s := range CONFIG.NoUpload { 876 m, err := regexp.MatchString(s, name) 877 if err != nil { 878 glog.Fatal(err) 879 } 880 if m { 881 return false 882 } 883 } 884 return true 885 } 886 887 // test generates a Test task. Returns the name of the last task in the 888 // generated chain of tasks, which the Job should add as a dependency. 889 func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string { 890 deps := []string{compileTaskName} 891 if strings.Contains(name, "Android_ASAN") { 892 deps = append(deps, isolateCIPDAsset(b, ISOLATE_NDK_LINUX_NAME)) 893 } 894 895 s := &specs.TaskSpec{ 896 CipdPackages: pkgs, 897 Dependencies: deps, 898 Dimensions: swarmDimensions(parts), 899 ExecutionTimeout: 4 * time.Hour, 900 Expiration: 20 * time.Hour, 901 ExtraArgs: []string{ 902 "--workdir", "../../..", "test", 903 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 904 fmt.Sprintf("buildbucket_build_id=%s", specs.PLACEHOLDER_BUILDBUCKET_BUILD_ID), 905 fmt.Sprintf("buildername=%s", name), 906 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 907 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 908 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 909 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 910 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 911 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 912 }, 913 IoTimeout: 40 * time.Minute, 914 Isolate: relpath("test_skia.isolate"), 915 MaxAttempts: 1, 916 Priority: 0.8, 917 } 918 if useBundledRecipes(parts) { 919 s.Dependencies = append(s.Dependencies, BUNDLE_RECIPES_NAME) 920 if strings.Contains(parts["os"], "Win") { 921 s.Isolate = relpath("test_skia_bundled_win.isolate") 922 } else { 923 s.Isolate = relpath("test_skia_bundled_unix.isolate") 924 } 925 } 926 if deps := getIsolatedCIPDDeps(parts); len(deps) > 0 { 927 s.Dependencies = append(s.Dependencies, deps...) 928 } 929 if strings.Contains(parts["extra_config"], "Valgrind") { 930 s.ExecutionTimeout = 9 * time.Hour 931 s.Expiration = 48 * time.Hour 932 s.IoTimeout = time.Hour 933 s.CipdPackages = append(s.CipdPackages, b.MustGetCipdPackageFromAsset("valgrind")) 934 s.Dimensions = append(s.Dimensions, "valgrind:1") 935 } else if strings.Contains(parts["extra_config"], "MSAN") { 936 s.ExecutionTimeout = 9 * time.Hour 937 } else if parts["arch"] == "x86" && parts["configuration"] == "Debug" { 938 // skia:6737 939 s.ExecutionTimeout = 6 * time.Hour 940 } 941 iid := internalHardwareLabel(parts) 942 if iid != nil { 943 s.ExtraArgs = append(s.ExtraArgs, fmt.Sprintf("internal_hardware_label=%d", *iid)) 944 } 945 b.MustAddTask(name, s) 946 947 // Upload results if necessary. TODO(kjlubick): If we do coverage analysis at the same 948 // time as normal tests (which would be nice), cfg.json needs to have Coverage removed. 949 if doUpload(name) { 950 uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name) 951 b.MustAddTask(uploadName, &specs.TaskSpec{ 952 Dependencies: []string{name}, 953 Dimensions: linuxGceDimensions(), 954 ExtraArgs: []string{ 955 "--workdir", "../../..", "upload_dm_results", 956 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 957 fmt.Sprintf("buildername=%s", name), 958 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 959 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 960 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 961 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 962 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 963 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 964 fmt.Sprintf("gs_bucket=%s", CONFIG.GsBucketGm), 965 }, 966 Isolate: relpath("upload_dm_results.isolate"), 967 Priority: 0.8, 968 }) 969 return uploadName 970 } 971 972 return name 973 } 974 975 func coverage(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string { 976 shards := 1 977 deps := []string{} 978 979 tf := parts["test_filter"] 980 if strings.Contains(tf, "Shard") { 981 // Expected Shard_NN 982 shardstr := strings.Split(tf, "_")[1] 983 var err error 984 shards, err = strconv.Atoi(shardstr) 985 if err != nil { 986 glog.Fatalf("Expected int for number of shards %q in %s: %s", shardstr, name, err) 987 } 988 } 989 for i := 0; i < shards; i++ { 990 n := strings.Replace(name, tf, fmt.Sprintf("shard_%02d_%02d", i, shards), 1) 991 s := &specs.TaskSpec{ 992 CipdPackages: pkgs, 993 Dependencies: []string{compileTaskName}, 994 Dimensions: swarmDimensions(parts), 995 ExecutionTimeout: 4 * time.Hour, 996 Expiration: 20 * time.Hour, 997 ExtraArgs: []string{ 998 "--workdir", "../../..", "test", 999 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 1000 fmt.Sprintf("buildername=%s", n), 1001 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 1002 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 1003 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 1004 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 1005 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 1006 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 1007 }, 1008 IoTimeout: 40 * time.Minute, 1009 Isolate: relpath("test_skia.isolate"), 1010 MaxAttempts: 1, 1011 Priority: 0.8, 1012 } 1013 if useBundledRecipes(parts) { 1014 s.Dependencies = append(s.Dependencies, BUNDLE_RECIPES_NAME) 1015 if strings.Contains(parts["os"], "Win") { 1016 s.Isolate = relpath("test_skia_bundled_win.isolate") 1017 } else { 1018 s.Isolate = relpath("test_skia_bundled_unix.isolate") 1019 } 1020 } 1021 if deps := getIsolatedCIPDDeps(parts); len(deps) > 0 { 1022 s.Dependencies = append(s.Dependencies, deps...) 1023 } 1024 b.MustAddTask(n, s) 1025 deps = append(deps, n) 1026 } 1027 1028 uploadName := fmt.Sprintf("%s%s%s", "Upload", jobNameSchema.Sep, name) 1029 // We need clang_linux to get access to the llvm-profdata and llvm-cov binaries 1030 // which are used to deal with the raw coverage data output by the Test step. 1031 pkgs = append([]*specs.CipdPackage{}, b.MustGetCipdPackageFromAsset("clang_linux")) 1032 deps = append(deps, compileTaskName) 1033 1034 b.MustAddTask(uploadName, &specs.TaskSpec{ 1035 // A dependency on compileTaskName makes the TaskScheduler link the 1036 // isolated output of the compile step to the input of the upload step, 1037 // which gives us access to the instrumented binary. The binary is 1038 // needed to figure out symbol names and line numbers. 1039 Dependencies: deps, 1040 Dimensions: linuxGceDimensions(), 1041 CipdPackages: pkgs, 1042 ExtraArgs: []string{ 1043 "--workdir", "../../..", "upload_coverage_results", 1044 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 1045 fmt.Sprintf("buildername=%s", name), 1046 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 1047 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 1048 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 1049 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 1050 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 1051 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 1052 fmt.Sprintf("gs_bucket=%s", CONFIG.GsBucketCoverage), 1053 }, 1054 Isolate: relpath("upload_coverage_results.isolate"), 1055 Priority: 0.8, 1056 }) 1057 return uploadName 1058 } 1059 1060 // perf generates a Perf task. Returns the name of the last task in the 1061 // generated chain of tasks, which the Job should add as a dependency. 1062 func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string { 1063 recipe := "perf" 1064 isolate := relpath("perf_skia.isolate") 1065 if strings.Contains(parts["extra_config"], "Skpbench") { 1066 recipe = "skpbench" 1067 isolate = relpath("skpbench_skia.isolate") 1068 if useBundledRecipes(parts) { 1069 if strings.Contains(parts["os"], "Win") { 1070 isolate = relpath("skpbench_skia_bundled_win.isolate") 1071 } else { 1072 isolate = relpath("skpbench_skia_bundled_unix.isolate") 1073 } 1074 } 1075 } else if useBundledRecipes(parts) { 1076 if strings.Contains(parts["os"], "Win") { 1077 isolate = relpath("perf_skia_bundled_win.isolate") 1078 } else { 1079 isolate = relpath("perf_skia_bundled_unix.isolate") 1080 } 1081 } 1082 s := &specs.TaskSpec{ 1083 CipdPackages: pkgs, 1084 Dependencies: []string{compileTaskName}, 1085 Dimensions: swarmDimensions(parts), 1086 ExecutionTimeout: 4 * time.Hour, 1087 Expiration: 20 * time.Hour, 1088 ExtraArgs: []string{ 1089 "--workdir", "../../..", recipe, 1090 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 1091 fmt.Sprintf("buildername=%s", name), 1092 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 1093 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 1094 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 1095 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 1096 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 1097 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 1098 }, 1099 IoTimeout: 40 * time.Minute, 1100 Isolate: isolate, 1101 MaxAttempts: 1, 1102 Priority: 0.8, 1103 } 1104 if useBundledRecipes(parts) { 1105 s.Dependencies = append(s.Dependencies, BUNDLE_RECIPES_NAME) 1106 } 1107 if deps := getIsolatedCIPDDeps(parts); len(deps) > 0 { 1108 s.Dependencies = append(s.Dependencies, deps...) 1109 } 1110 1111 if strings.Contains(parts["extra_config"], "Valgrind") { 1112 s.ExecutionTimeout = 9 * time.Hour 1113 s.Expiration = 48 * time.Hour 1114 s.IoTimeout = time.Hour 1115 s.CipdPackages = append(s.CipdPackages, b.MustGetCipdPackageFromAsset("valgrind")) 1116 s.Dimensions = append(s.Dimensions, "valgrind:1") 1117 } else if strings.Contains(parts["extra_config"], "MSAN") { 1118 s.ExecutionTimeout = 9 * time.Hour 1119 } else if parts["arch"] == "x86" && parts["configuration"] == "Debug" { 1120 // skia:6737 1121 s.ExecutionTimeout = 6 * time.Hour 1122 } 1123 iid := internalHardwareLabel(parts) 1124 if iid != nil { 1125 s.ExtraArgs = append(s.ExtraArgs, fmt.Sprintf("internal_hardware_label=%d", *iid)) 1126 } 1127 b.MustAddTask(name, s) 1128 1129 // Upload results if necessary. 1130 if strings.Contains(name, "Release") && doUpload(name) { 1131 uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name) 1132 b.MustAddTask(uploadName, &specs.TaskSpec{ 1133 Dependencies: []string{name}, 1134 Dimensions: linuxGceDimensions(), 1135 ExtraArgs: []string{ 1136 "--workdir", "../../..", "upload_nano_results", 1137 fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), 1138 fmt.Sprintf("buildername=%s", name), 1139 fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), 1140 fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), 1141 fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), 1142 fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), 1143 fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), 1144 fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), 1145 fmt.Sprintf("gs_bucket=%s", CONFIG.GsBucketNano), 1146 }, 1147 Isolate: relpath("upload_nano_results.isolate"), 1148 Priority: 0.8, 1149 }) 1150 return uploadName 1151 } 1152 return name 1153 } 1154 1155 // process generates tasks and jobs for the given job name. 1156 func process(b *specs.TasksCfgBuilder, name string) { 1157 deps := []string{} 1158 1159 // Bundle Recipes. 1160 if name == BUNDLE_RECIPES_NAME { 1161 deps = append(deps, bundleRecipes(b)) 1162 } 1163 1164 // Isolate CIPD assets. 1165 if _, ok := ISOLATE_ASSET_MAPPING[name]; ok { 1166 deps = append(deps, isolateCIPDAsset(b, name)) 1167 } 1168 1169 parts, err := jobNameSchema.ParseJobName(name) 1170 if err != nil { 1171 glog.Fatal(err) 1172 } 1173 1174 // RecreateSKPs. 1175 if strings.Contains(name, "RecreateSKPs") { 1176 deps = append(deps, recreateSKPs(b, name)) 1177 } 1178 1179 // UpdateMetaConfig bot. 1180 if strings.Contains(name, "UpdateMetaConfig") { 1181 deps = append(deps, updateMetaConfig(b, name)) 1182 } 1183 1184 // CT bots. 1185 if strings.Contains(name, "-CT_") { 1186 deps = append(deps, ctSKPs(b, name)) 1187 } 1188 1189 // Infra tests. 1190 if name == "Housekeeper-PerCommit-InfraTests" { 1191 deps = append(deps, infra(b, name)) 1192 } 1193 1194 // Compile bots. 1195 if parts["role"] == "Build" { 1196 if parts["extra_config"] == "Android_Framework" { 1197 // Android Framework compile tasks use a different recipe. 1198 deps = append(deps, androidFrameworkCompile(b, name)) 1199 } else { 1200 deps = append(deps, compile(b, name, parts)) 1201 } 1202 } 1203 1204 // Most remaining bots need a compile task. 1205 compileTaskName := deriveCompileTaskName(name, parts) 1206 compileTaskParts, err := jobNameSchema.ParseJobName(compileTaskName) 1207 if err != nil { 1208 glog.Fatal(err) 1209 } 1210 compileParentName := getParentRevisionName(compileTaskName, compileTaskParts) 1211 compileParentParts, err := jobNameSchema.ParseJobName(compileParentName) 1212 if err != nil { 1213 glog.Fatal(err) 1214 } 1215 1216 // These bots do not need a compile task. 1217 if parts["role"] != "Build" && 1218 name != "Housekeeper-PerCommit-BundleRecipes" && 1219 name != "Housekeeper-PerCommit-InfraTests" && 1220 name != "Housekeeper-PerCommit-CheckGeneratedFiles" && 1221 !strings.Contains(name, "Android_Framework") && 1222 !strings.Contains(name, "RecreateSKPs") && 1223 !strings.Contains(name, "UpdateMetaConfig") && 1224 !strings.Contains(name, "-CT_") && 1225 !strings.Contains(name, "Housekeeper-PerCommit-Isolate") { 1226 compile(b, compileTaskName, compileTaskParts) 1227 if parts["role"] == "Calmbench" { 1228 compile(b, compileParentName, compileParentParts) 1229 } 1230 } 1231 1232 // Housekeepers. 1233 if name == "Housekeeper-PerCommit" { 1234 deps = append(deps, housekeeper(b, name, compileTaskName)) 1235 } 1236 if name == "Housekeeper-PerCommit-CheckGeneratedFiles" { 1237 deps = append(deps, checkGeneratedFiles(b, name)) 1238 } 1239 if strings.Contains(name, "Bookmaker") { 1240 deps = append(deps, bookmaker(b, name, compileTaskName)) 1241 } 1242 1243 // Common assets needed by the remaining bots. 1244 1245 pkgs := []*specs.CipdPackage{} 1246 1247 if deps := getIsolatedCIPDDeps(parts); len(deps) == 0 { 1248 pkgs = []*specs.CipdPackage{ 1249 b.MustGetCipdPackageFromAsset("skimage"), 1250 b.MustGetCipdPackageFromAsset("skp"), 1251 b.MustGetCipdPackageFromAsset("svg"), 1252 } 1253 } 1254 1255 if strings.Contains(name, "Ubuntu") || strings.Contains(name, "Debian") { 1256 if strings.Contains(name, "SAN") { 1257 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux")) 1258 } 1259 if strings.Contains(name, "Vulkan") { 1260 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("linux_vulkan_sdk")) 1261 } 1262 if strings.Contains(name, "Intel") && strings.Contains(name, "GPU") { 1263 if strings.Contains(name, "Release") { 1264 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("linux_vulkan_intel_driver_release")) 1265 } else { 1266 pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("linux_vulkan_intel_driver_debug")) 1267 } 1268 } 1269 } 1270 1271 // Test bots. 1272 1273 if parts["role"] == "Test" { 1274 if strings.Contains(parts["extra_config"], "Coverage") { 1275 deps = append(deps, coverage(b, name, parts, compileTaskName, pkgs)) 1276 } else if !strings.Contains(name, "-CT_") { 1277 deps = append(deps, test(b, name, parts, compileTaskName, pkgs)) 1278 } 1279 1280 } 1281 1282 // Perf bots. 1283 if parts["role"] == "Perf" && !strings.Contains(name, "-CT_") { 1284 deps = append(deps, perf(b, name, parts, compileTaskName, pkgs)) 1285 } 1286 1287 // Calmbench bots. 1288 if parts["role"] == "Calmbench" { 1289 deps = append(deps, calmbench(b, name, parts, compileTaskName, compileParentName)) 1290 } 1291 1292 // Add the Job spec. 1293 j := &specs.JobSpec{ 1294 Priority: 0.8, 1295 TaskSpecs: deps, 1296 Trigger: specs.TRIGGER_ANY_BRANCH, 1297 } 1298 if strings.Contains(name, "-Nightly-") { 1299 j.Trigger = specs.TRIGGER_NIGHTLY 1300 } else if strings.Contains(name, "-Weekly-") || strings.Contains(name, "CT_DM_1m_SKPs") { 1301 j.Trigger = specs.TRIGGER_WEEKLY 1302 } else if strings.Contains(name, "Flutter") || strings.Contains(name, "PDFium") || strings.Contains(name, "CommandBuffer") { 1303 j.Trigger = specs.TRIGGER_MASTER_ONLY 1304 } 1305 b.MustAddJob(name, j) 1306 } 1307 1308 func loadJson(flag *string, defaultFlag string, val interface{}) { 1309 if *flag == "" { 1310 *flag = defaultFlag 1311 } 1312 b, err := ioutil.ReadFile(*flag) 1313 if err != nil { 1314 glog.Fatal(err) 1315 } 1316 if err := json.Unmarshal(b, val); err != nil { 1317 glog.Fatal(err) 1318 } 1319 } 1320 1321 // Regenerate the tasks.json file. 1322 func main() { 1323 b := specs.MustNewTasksCfgBuilder() 1324 b.SetAssetsDir(*assetsDir) 1325 infraBots := path.Join(b.CheckoutRoot(), "infra", "bots") 1326 1327 // Load the jobs from a JSON file. 1328 loadJson(jobsFile, path.Join(infraBots, "jobs.json"), &JOBS) 1329 1330 // Load general config information from a JSON file. 1331 loadJson(cfgFile, path.Join(infraBots, "cfg.json"), &CONFIG) 1332 1333 // Create the JobNameSchema. 1334 if *builderNameSchemaFile == "" { 1335 *builderNameSchemaFile = path.Join(b.CheckoutRoot(), "infra", "bots", "recipe_modules", "builder_name_schema", "builder_name_schema.json") 1336 } 1337 schema, err := NewJobNameSchema(*builderNameSchemaFile) 1338 if err != nil { 1339 glog.Fatal(err) 1340 } 1341 jobNameSchema = schema 1342 1343 // Create Tasks and Jobs. 1344 for _, name := range JOBS { 1345 process(b, name) 1346 } 1347 1348 b.MustFinish() 1349 } 1350 1351 // TODO(borenet): The below really belongs in its own file, probably next to the 1352 // builder_name_schema.json file. 1353 1354 // JobNameSchema is a struct used for (de)constructing Job names in a 1355 // predictable format. 1356 type JobNameSchema struct { 1357 Schema map[string][]string `json:"builder_name_schema"` 1358 Sep string `json:"builder_name_sep"` 1359 } 1360 1361 // NewJobNameSchema returns a JobNameSchema instance based on the given JSON 1362 // file. 1363 func NewJobNameSchema(jsonFile string) (*JobNameSchema, error) { 1364 var rv JobNameSchema 1365 f, err := os.Open(jsonFile) 1366 if err != nil { 1367 return nil, err 1368 } 1369 defer util.Close(f) 1370 if err := json.NewDecoder(f).Decode(&rv); err != nil { 1371 return nil, err 1372 } 1373 return &rv, nil 1374 } 1375 1376 // ParseJobName splits the given Job name into its component parts, according 1377 // to the schema. 1378 func (s *JobNameSchema) ParseJobName(n string) (map[string]string, error) { 1379 split := strings.Split(n, s.Sep) 1380 if len(split) < 2 { 1381 return nil, fmt.Errorf("Invalid job name: %q", n) 1382 } 1383 role := split[0] 1384 split = split[1:] 1385 keys, ok := s.Schema[role] 1386 if !ok { 1387 return nil, fmt.Errorf("Invalid job name; %q is not a valid role.", role) 1388 } 1389 extraConfig := "" 1390 if len(split) == len(keys)+1 { 1391 extraConfig = split[len(split)-1] 1392 split = split[:len(split)-1] 1393 } 1394 if len(split) != len(keys) { 1395 return nil, fmt.Errorf("Invalid job name; %q has incorrect number of parts.", n) 1396 } 1397 rv := make(map[string]string, len(keys)+2) 1398 rv["role"] = role 1399 if extraConfig != "" { 1400 rv["extra_config"] = extraConfig 1401 } 1402 for i, k := range keys { 1403 rv[k] = split[i] 1404 } 1405 return rv, nil 1406 } 1407 1408 // MakeJobName assembles the given parts of a Job name, according to the schema. 1409 func (s *JobNameSchema) MakeJobName(parts map[string]string) (string, error) { 1410 role, ok := parts["role"] 1411 if !ok { 1412 return "", fmt.Errorf("Invalid job parts; jobs must have a role.") 1413 } 1414 keys, ok := s.Schema[role] 1415 if !ok { 1416 return "", fmt.Errorf("Invalid job parts; unknown role %q", role) 1417 } 1418 rvParts := make([]string, 0, len(parts)) 1419 rvParts = append(rvParts, role) 1420 for _, k := range keys { 1421 v, ok := parts[k] 1422 if !ok { 1423 return "", fmt.Errorf("Invalid job parts; missing %q", k) 1424 } 1425 rvParts = append(rvParts, v) 1426 } 1427 if _, ok := parts["extra_config"]; ok { 1428 rvParts = append(rvParts, parts["extra_config"]) 1429 } 1430 return strings.Join(rvParts, s.Sep), nil 1431 } 1432
