1 #!/usr/bin/env python 2 # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # ============================================================================== 16 """Help include git hash in tensorflow bazel build. 17 18 This creates symlinks from the internal git repository directory so 19 that the build system can see changes in the version state. We also 20 remember what branch git was on so when the branch changes we can 21 detect that the ref file is no longer correct (so we can suggest users 22 run ./configure again). 23 24 NOTE: this script is only used in opensource. 25 26 """ 27 from __future__ import absolute_import 28 from __future__ import division 29 from __future__ import print_function 30 import argparse 31 import json 32 import os 33 import subprocess 34 import shutil 35 36 37 def parse_branch_ref(filename): 38 """Given a filename of a .git/HEAD file return ref path. 39 40 In particular, if git is in detached head state, this will 41 return None. If git is in attached head, it will return 42 the branch reference. E.g. if on 'master', the HEAD will 43 contain 'ref: refs/heads/master' so 'refs/heads/master' 44 will be returned. 45 46 Example: parse_branch_ref(".git/HEAD") 47 Args: 48 filename: file to treat as a git HEAD file 49 Returns: 50 None if detached head, otherwise ref subpath 51 Raises: 52 RuntimeError: if the HEAD file is unparseable. 53 """ 54 55 data = open(filename).read().strip() 56 items = data.split(" ") 57 if len(items) == 1: 58 return None 59 elif len(items) == 2 and items[0] == "ref:": 60 return items[1].strip() 61 else: 62 raise RuntimeError("Git directory has unparseable HEAD") 63 64 65 def configure(src_base_path, gen_path, debug=False): 66 """Configure `src_base_path` to embed git hashes if available.""" 67 68 # TODO(aselle): No files generated or symlinked here are deleted by 69 # the build system. I don't know of a way to do it in bazel. It 70 # should only be a problem if somebody moves a sandbox directory 71 # without running ./configure again. 72 73 git_path = os.path.join(src_base_path, ".git") 74 75 # Remove and recreate the path 76 if os.path.exists(gen_path): 77 if os.path.isdir(gen_path): 78 try: 79 shutil.rmtree(gen_path) 80 except OSError: 81 raise RuntimeError("Cannot delete directory %s due to permission " 82 "error, inspect and remove manually" % gen_path) 83 else: 84 raise RuntimeError("Cannot delete non-directory %s, inspect ", 85 "and remove manually" % gen_path) 86 os.makedirs(gen_path) 87 88 if not os.path.isdir(gen_path): 89 raise RuntimeError("gen_git_source.py: Failed to create dir") 90 91 # file that specifies what the state of the git repo is 92 spec = {} 93 94 # value file names will be mapped to the keys 95 link_map = {"head": None, "branch_ref": None} 96 97 if not os.path.isdir(git_path): 98 # No git directory 99 spec["git"] = False 100 open(os.path.join(gen_path, "head"), "w").write("") 101 open(os.path.join(gen_path, "branch_ref"), "w").write("") 102 else: 103 # Git directory, possibly detached or attached 104 spec["git"] = True 105 spec["path"] = src_base_path 106 git_head_path = os.path.join(git_path, "HEAD") 107 spec["branch"] = parse_branch_ref(git_head_path) 108 link_map["head"] = git_head_path 109 if spec["branch"] is not None: 110 # attached method 111 link_map["branch_ref"] = os.path.join(git_path, * 112 os.path.split(spec["branch"])) 113 # Create symlinks or dummy files 114 for target, src in link_map.items(): 115 if src is None: 116 open(os.path.join(gen_path, target), "w").write("") 117 else: 118 try: 119 # In python 3.5, symlink function exists even on Windows. But requires 120 # Windows Admin privileges, otherwise an OSError will be thrown. 121 if hasattr(os, 'symlink'): 122 os.symlink(src, os.path.join(gen_path, target)) 123 else: 124 shutil.copy2(src, os.path.join(gen_path, target)) 125 except OSError: 126 shutil.copy2(src, os.path.join(gen_path, target)) 127 128 json.dump(spec, open(os.path.join(gen_path, "spec.json"), "w"), indent=2) 129 if debug: 130 print("gen_git_source.py: list %s" % gen_path) 131 print("gen_git_source.py: %s" + repr(os.listdir(gen_path))) 132 print("gen_git_source.py: spec is %r" % spec) 133 134 135 def get_git_version(git_base_path): 136 """Get the git version from the repository. 137 138 This function runs `git describe ...` in the path given as `git_base_path`. 139 This will return a string of the form: 140 <base-tag>-<number of commits since tag>-<shortened sha hash> 141 142 For example, 'v0.10.0-1585-gbb717a6' means v0.10.0 was the last tag when 143 compiled. 1585 commits are after that commit tag, and we can get back to this 144 version by running `git checkout gbb717a6`. 145 146 Args: 147 git_base_path: where the .git directory is located 148 Returns: 149 A bytestring representing the git version 150 """ 151 unknown_label = b"unknown" 152 try: 153 val = bytes(subprocess.check_output([ 154 "git", str("--git-dir=%s/.git" % git_base_path), 155 str("--work-tree=" + git_base_path), "describe", "--long", "--tags" 156 ]).strip()) 157 return val if val else unknown_label 158 except subprocess.CalledProcessError: 159 return unknown_label 160 161 162 def write_version_info(filename, git_version): 163 """Write a c file that defines the version functions. 164 165 Args: 166 filename: filename to write to. 167 git_version: the result of a git describe. 168 """ 169 if b"\"" in git_version or b"\\" in git_version: 170 git_version = "git_version_is_invalid" # do not cause build to fail! 171 contents = """/* Generated by gen_git_source.py */ 172 #include <string> 173 const char* tf_git_version() {return "%s";} 174 const char* tf_compiler_version() {return __VERSION__;} 175 const int tf_cxx11_abi_flag() { 176 #ifdef _GLIBCXX_USE_CXX11_ABI 177 return _GLIBCXX_USE_CXX11_ABI; 178 #else 179 return 0; 180 #endif 181 } 182 const int tf_monolithic_build() { 183 #ifdef TENSORFLOW_MONOLITHIC_BUILD 184 return 1; 185 #else 186 return 0; 187 #endif 188 } 189 """ % git_version 190 open(filename, "w").write(contents) 191 192 193 def generate(arglist): 194 """Generate version_info.cc as given `destination_file`. 195 196 Args: 197 arglist: should be a sequence that contains 198 spec, head_symlink, ref_symlink, destination_file. 199 200 `destination_file` is the filename where version_info.cc will be written 201 202 `spec` is a filename where the file contains a JSON dictionary 203 'git' bool that is true if the source is in a git repo 204 'path' base path of the source code 205 'branch' the name of the ref specification of the current branch/tag 206 207 `head_symlink` is a filename to HEAD that is cross-referenced against 208 what is contained in the json branch designation. 209 210 `ref_symlink` is unused in this script but passed, because the build 211 system uses that file to detect when commits happen. 212 213 Raises: 214 RuntimeError: If ./configure needs to be run, RuntimeError will be raised. 215 """ 216 217 # unused ref_symlink arg 218 spec, head_symlink, _, dest_file = arglist 219 data = json.load(open(spec)) 220 git_version = None 221 if not data["git"]: 222 git_version = b"unknown" 223 else: 224 old_branch = data["branch"] 225 new_branch = parse_branch_ref(head_symlink) 226 if new_branch != old_branch: 227 raise RuntimeError( 228 "Run ./configure again, branch was '%s' but is now '%s'" % 229 (old_branch, new_branch)) 230 git_version = get_git_version(data["path"]) 231 write_version_info(dest_file, git_version) 232 233 234 def raw_generate(output_file): 235 """Simple generator used for cmake/make build systems. 236 237 This does not create any symlinks. It requires the build system 238 to build unconditionally. 239 240 Args: 241 output_file: Output filename for the version info cc 242 """ 243 244 git_version = get_git_version(".") 245 write_version_info(output_file, git_version) 246 247 248 parser = argparse.ArgumentParser(description="""Git hash injection into bazel. 249 If used with --configure <path> will search for git directory and put symlinks 250 into source so that a bazel genrule can call --generate""") 251 252 parser.add_argument( 253 "--debug", 254 type=bool, 255 help="print debugging information about paths", 256 default=False) 257 258 parser.add_argument( 259 "--configure", type=str, 260 help="Path to configure as a git repo dependency tracking sentinel") 261 262 parser.add_argument( 263 "--gen_root_path", type=str, 264 help="Root path to place generated git files (created by --configure).") 265 266 parser.add_argument( 267 "--generate", 268 type=str, 269 help="Generate given spec-file, HEAD-symlink-file, ref-symlink-file", 270 nargs="+") 271 272 parser.add_argument( 273 "--raw_generate", 274 type=str, 275 help="Generate version_info.cc (simpler version used for cmake/make)") 276 277 args = parser.parse_args() 278 279 if args.configure is not None: 280 if args.gen_root_path is None: 281 raise RuntimeError("Must pass --gen_root_path arg when running --configure") 282 configure(args.configure, args.gen_root_path, debug=args.debug) 283 elif args.generate is not None: 284 generate(args.generate) 285 elif args.raw_generate is not None: 286 raw_generate(args.raw_generate) 287 else: 288 raise RuntimeError("--configure or --generate or --raw_generate " 289 "must be used") 290
