1 #!/usr/bin/env python2.7 2 # Copyright 2017 gRPC authors. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 """Build and upload docker images to Google Container Registry per matrix.""" 16 17 from __future__ import print_function 18 19 import argparse 20 import atexit 21 import multiprocessing 22 import os 23 import shutil 24 import subprocess 25 import sys 26 import tempfile 27 28 # Langauage Runtime Matrix 29 import client_matrix 30 31 python_util_dir = os.path.abspath( 32 os.path.join(os.path.dirname(__file__), '../run_tests/python_utils')) 33 sys.path.append(python_util_dir) 34 import dockerjob 35 import jobset 36 37 _IMAGE_BUILDER = 'tools/run_tests/dockerize/build_interop_image.sh' 38 _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys() 39 # All gRPC release tags, flattened, deduped and sorted. 40 _RELEASES = sorted( 41 list( 42 set( 43 client_matrix.get_release_tag_name(info) 44 for lang in client_matrix.LANG_RELEASE_MATRIX.values() 45 for info in lang))) 46 47 # Destination directory inside docker image to keep extra info from build time. 48 _BUILD_INFO = '/var/local/build_info' 49 50 argp = argparse.ArgumentParser(description='Run interop tests.') 51 argp.add_argument( 52 '--gcr_path', 53 default='gcr.io/grpc-testing', 54 help='Path of docker images in Google Container Registry') 55 56 argp.add_argument( 57 '--release', 58 default='master', 59 choices=['all', 'master'] + _RELEASES, 60 help='github commit tag to checkout. When building all ' 61 'releases defined in client_matrix.py, use "all". Valid only ' 62 'with --git_checkout.') 63 64 argp.add_argument( 65 '-l', 66 '--language', 67 choices=['all'] + sorted(_LANGUAGES), 68 nargs='+', 69 default=['all'], 70 help='Test languages to build docker images for.') 71 72 argp.add_argument( 73 '--git_checkout', 74 action='store_true', 75 help='Use a separate git clone tree for building grpc stack. ' 76 'Required when using --release flag. By default, current' 77 'tree and the sibling will be used for building grpc stack.') 78 79 argp.add_argument( 80 '--git_checkout_root', 81 default='/export/hda3/tmp/grpc_matrix', 82 help='Directory under which grpc-go/java/main repo will be ' 83 'cloned. Valid only with --git_checkout.') 84 85 argp.add_argument( 86 '--keep', 87 action='store_true', 88 help='keep the created local images after uploading to GCR') 89 90 argp.add_argument( 91 '--reuse_git_root', 92 default=False, 93 action='store_const', 94 const=True, 95 help='reuse the repo dir. If False, the existing git root ' 96 'directory will removed before a clean checkout, because ' 97 'reusing the repo can cause git checkout error if you switch ' 98 'between releases.') 99 100 argp.add_argument( 101 '--upload_images', 102 action='store_true', 103 help='If set, images will be uploaded to container registry after building.' 104 ) 105 106 args = argp.parse_args() 107 108 109 def add_files_to_image(image, with_files, label=None): 110 """Add files to a docker image. 111 112 image: docker image name, i.e. grpc_interop_java:26328ad8 113 with_files: additional files to include in the docker image. 114 label: label string to attach to the image. 115 """ 116 tag_idx = image.find(':') 117 if tag_idx == -1: 118 jobset.message( 119 'FAILED', 'invalid docker image %s' % image, do_newline=True) 120 sys.exit(1) 121 orig_tag = '%s_' % image 122 subprocess.check_output(['docker', 'tag', image, orig_tag]) 123 124 lines = ['FROM ' + orig_tag] 125 if label: 126 lines.append('LABEL %s' % label) 127 128 temp_dir = tempfile.mkdtemp() 129 atexit.register(lambda: subprocess.call(['rm', '-rf', temp_dir])) 130 131 # Copy with_files inside the tmp directory, which will be the docker build 132 # context. 133 for f in with_files: 134 shutil.copy(f, temp_dir) 135 lines.append('COPY %s %s/' % (os.path.basename(f), _BUILD_INFO)) 136 137 # Create a Dockerfile. 138 with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as f: 139 f.write('\n'.join(lines)) 140 141 jobset.message('START', 'Repackaging %s' % image, do_newline=True) 142 build_cmd = ['docker', 'build', '--rm', '--tag', image, temp_dir] 143 subprocess.check_output(build_cmd) 144 dockerjob.remove_image(orig_tag, skip_nonexistent=True) 145 146 147 def build_image_jobspec(runtime, env, gcr_tag, stack_base): 148 """Build interop docker image for a language with runtime. 149 150 runtime: a <lang><version> string, for example go1.8. 151 env: dictionary of env to passed to the build script. 152 gcr_tag: the tag for the docker image (i.e. v1.3.0). 153 stack_base: the local gRPC repo path. 154 """ 155 basename = 'grpc_interop_%s' % runtime 156 tag = '%s/%s:%s' % (args.gcr_path, basename, gcr_tag) 157 build_env = {'INTEROP_IMAGE': tag, 'BASE_NAME': basename, 'TTY_FLAG': '-t'} 158 build_env.update(env) 159 image_builder_path = _IMAGE_BUILDER 160 if client_matrix.should_build_docker_interop_image_from_release_tag(lang): 161 image_builder_path = os.path.join(stack_base, _IMAGE_BUILDER) 162 build_job = jobset.JobSpec( 163 cmdline=[image_builder_path], 164 environ=build_env, 165 shortname='build_docker_%s' % runtime, 166 timeout_seconds=30 * 60) 167 build_job.tag = tag 168 return build_job 169 170 171 def build_all_images_for_lang(lang): 172 """Build all docker images for a language across releases and runtimes.""" 173 if not args.git_checkout: 174 if args.release != 'master': 175 print( 176 'Cannot use --release without also enabling --git_checkout.\n') 177 sys.exit(1) 178 releases = [args.release] 179 else: 180 if args.release == 'all': 181 releases = client_matrix.get_release_tags(lang) 182 else: 183 # Build a particular release. 184 if args.release not in ['master' 185 ] + client_matrix.get_release_tags(lang): 186 jobset.message( 187 'SKIPPED', 188 '%s for %s is not defined' % (args.release, lang), 189 do_newline=True) 190 return [] 191 releases = [args.release] 192 193 images = [] 194 for release in releases: 195 images += build_all_images_for_release(lang, release) 196 jobset.message( 197 'SUCCESS', 198 'All docker images built for %s at %s.' % (lang, releases), 199 do_newline=True) 200 return images 201 202 203 def build_all_images_for_release(lang, release): 204 """Build all docker images for a release across all runtimes.""" 205 docker_images = [] 206 build_jobs = [] 207 208 env = {} 209 # If we not using current tree or the sibling for grpc stack, do checkout. 210 stack_base = '' 211 if args.git_checkout: 212 stack_base = checkout_grpc_stack(lang, release) 213 var = { 214 'go': 'GRPC_GO_ROOT', 215 'java': 'GRPC_JAVA_ROOT', 216 'node': 'GRPC_NODE_ROOT' 217 }.get(lang, 'GRPC_ROOT') 218 env[var] = stack_base 219 220 for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]: 221 job = build_image_jobspec(runtime, env, release, stack_base) 222 docker_images.append(job.tag) 223 build_jobs.append(job) 224 225 jobset.message('START', 'Building interop docker images.', do_newline=True) 226 print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) 227 228 num_failures, _ = jobset.run( 229 build_jobs, 230 newline_on_success=True, 231 maxjobs=multiprocessing.cpu_count()) 232 if num_failures: 233 jobset.message( 234 'FAILED', 'Failed to build interop docker images.', do_newline=True) 235 docker_images_cleanup.extend(docker_images) 236 sys.exit(1) 237 238 jobset.message( 239 'SUCCESS', 240 'All docker images built for %s at %s.' % (lang, release), 241 do_newline=True) 242 243 if release != 'master': 244 commit_log = os.path.join(stack_base, 'commit_log') 245 if os.path.exists(commit_log): 246 for image in docker_images: 247 add_files_to_image(image, [commit_log], 'release=%s' % release) 248 return docker_images 249 250 251 def cleanup(): 252 if not args.keep: 253 for image in docker_images_cleanup: 254 dockerjob.remove_image(image, skip_nonexistent=True) 255 256 257 docker_images_cleanup = [] 258 atexit.register(cleanup) 259 260 261 def maybe_apply_patches_on_git_tag(stack_base, lang, release): 262 files_to_patch = [] 263 for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]: 264 if client_matrix.get_release_tag_name(release_info) == release: 265 if release_info[release] is not None: 266 files_to_patch = release_info[release].get('patch') 267 break 268 if not files_to_patch: 269 return 270 patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release) 271 patch_file = os.path.abspath( 272 os.path.join(os.path.dirname(__file__), patch_file_relative_path)) 273 if not os.path.exists(patch_file): 274 jobset.message('FAILED', 275 'expected patch file |%s| to exist' % patch_file) 276 sys.exit(1) 277 subprocess.check_output( 278 ['git', 'apply', patch_file], cwd=stack_base, stderr=subprocess.STDOUT) 279 280 # TODO(jtattermusch): this really would need simplification and refactoring 281 # - "git add" and "git commit" can easily be done in a single command 282 # - it looks like the only reason for the existence of the "files_to_patch" 283 # entry is to perform "git add" - which is clumsy and fragile. 284 # - we only allow a single patch with name "git_repo.patch". A better design 285 # would be to allow multiple patches that can have more descriptive names. 286 for repo_relative_path in files_to_patch: 287 subprocess.check_output( 288 ['git', 'add', repo_relative_path], 289 cwd=stack_base, 290 stderr=subprocess.STDOUT) 291 subprocess.check_output( 292 [ 293 'git', 'commit', '-m', 294 ('Hack performed on top of %s git ' 295 'tag in order to build and run the %s ' 296 'interop tests on that tag.' % (lang, release)) 297 ], 298 cwd=stack_base, 299 stderr=subprocess.STDOUT) 300 301 302 def checkout_grpc_stack(lang, release): 303 """Invokes 'git check' for the lang/release and returns directory created.""" 304 assert args.git_checkout and args.git_checkout_root 305 306 if not os.path.exists(args.git_checkout_root): 307 os.makedirs(args.git_checkout_root) 308 309 repo = client_matrix.get_github_repo(lang) 310 # Get the subdir name part of repo 311 # For example, 'git (at] github.com:grpc/grpc-go.git' should use 'grpc-go'. 312 repo_dir = os.path.splitext(os.path.basename(repo))[0] 313 stack_base = os.path.join(args.git_checkout_root, repo_dir) 314 315 # Clean up leftover repo dir if necessary. 316 if not args.reuse_git_root and os.path.exists(stack_base): 317 jobset.message('START', 'Removing git checkout root.', do_newline=True) 318 shutil.rmtree(stack_base) 319 320 if not os.path.exists(stack_base): 321 subprocess.check_call( 322 ['git', 'clone', '--recursive', repo], 323 cwd=os.path.dirname(stack_base)) 324 325 # git checkout. 326 jobset.message( 327 'START', 328 'git checkout %s from %s' % (release, stack_base), 329 do_newline=True) 330 # We should NEVER do checkout on current tree !!! 331 assert not os.path.dirname(__file__).startswith(stack_base) 332 output = subprocess.check_output( 333 ['git', 'checkout', release], cwd=stack_base, stderr=subprocess.STDOUT) 334 maybe_apply_patches_on_git_tag(stack_base, lang, release) 335 commit_log = subprocess.check_output(['git', 'log', '-1'], cwd=stack_base) 336 jobset.message( 337 'SUCCESS', 338 'git checkout', 339 '%s: %s' % (str(output), commit_log), 340 do_newline=True) 341 342 # Write git log to commit_log so it can be packaged with the docker image. 343 with open(os.path.join(stack_base, 'commit_log'), 'w') as f: 344 f.write(commit_log) 345 return stack_base 346 347 348 languages = args.language if args.language != ['all'] else _LANGUAGES 349 for lang in languages: 350 docker_images = build_all_images_for_lang(lang) 351 for image in docker_images: 352 if args.upload_images: 353 jobset.message('START', 'Uploading %s' % image, do_newline=True) 354 # docker image name must be in the format <gcr_path>/<image>:<gcr_tag> 355 assert image.startswith(args.gcr_path) and image.find(':') != -1 356 subprocess.call(['gcloud', 'docker', '--', 'push', image]) 357 else: 358 # Uploading (and overwriting images) by default can easily break things. 359 print('Not uploading image %s, run with --upload_images to upload.' 360 % image) 361
