1 # Adapted with modifications from tensorflow/third_party/py/ 2 """Repository rule for Python autoconfiguration. 3 4 `python_configure` depends on the following environment variables: 5 6 * `PYTHON_BIN_PATH`: location of python binary. 7 * `PYTHON_LIB_PATH`: Location of python libraries. 8 """ 9 10 _BAZEL_SH = "BAZEL_SH" 11 _PYTHON_BIN_PATH = "PYTHON_BIN_PATH" 12 _PYTHON_LIB_PATH = "PYTHON_LIB_PATH" 13 _PYTHON_CONFIG_REPO = "PYTHON_CONFIG_REPO" 14 15 16 def _tpl(repository_ctx, tpl, substitutions={}, out=None): 17 if not out: 18 out = tpl 19 repository_ctx.template(out, Label("//third_party/py:%s.tpl" % tpl), 20 substitutions) 21 22 23 def _fail(msg): 24 """Output failure message when auto configuration fails.""" 25 red = "\033[0;31m" 26 no_color = "\033[0m" 27 fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) 28 29 30 def _is_windows(repository_ctx): 31 """Returns true if the host operating system is windows.""" 32 os_name = repository_ctx.os.name.lower() 33 return os_name.find("windows") != -1 34 35 36 def _execute(repository_ctx, 37 cmdline, 38 error_msg=None, 39 error_details=None, 40 empty_stdout_fine=False): 41 """Executes an arbitrary shell command. 42 43 Args: 44 repository_ctx: the repository_ctx object 45 cmdline: list of strings, the command to execute 46 error_msg: string, a summary of the error if the command fails 47 error_details: string, details about the error or steps to fix it 48 empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise 49 it's an error 50 Return: 51 the result of repository_ctx.execute(cmdline) 52 """ 53 result = repository_ctx.execute(cmdline) 54 if result.stderr or not (empty_stdout_fine or result.stdout): 55 _fail("\n".join([ 56 error_msg.strip() if error_msg else "Repository command failed", 57 result.stderr.strip(), error_details if error_details else "" 58 ])) 59 else: 60 return result 61 62 63 def _read_dir(repository_ctx, src_dir): 64 """Returns a string with all files in a directory. 65 66 Finds all files inside a directory, traversing subfolders and following 67 symlinks. The returned string contains the full path of all files 68 separated by line breaks. 69 """ 70 if _is_windows(repository_ctx): 71 src_dir = src_dir.replace("/", "\\") 72 find_result = _execute( 73 repository_ctx, 74 ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], 75 empty_stdout_fine=True) 76 # src_files will be used in genrule.outs where the paths must 77 # use forward slashes. 78 return find_result.stdout.replace("\\", "/") 79 else: 80 find_result = _execute( 81 repository_ctx, ["find", src_dir, "-follow", "-type", "f"], 82 empty_stdout_fine=True) 83 return find_result.stdout 84 85 86 def _genrule(src_dir, genrule_name, command, outs): 87 """Returns a string with a genrule. 88 89 Genrule executes the given command and produces the given outputs. 90 """ 91 return ('genrule(\n' + ' name = "' + genrule_name + '",\n' + 92 ' outs = [\n' + outs + '\n ],\n' + ' cmd = """\n' + 93 command + '\n """,\n' + ')\n') 94 95 96 def _normalize_path(path): 97 """Returns a path with '/' and remove the trailing slash.""" 98 path = path.replace("\\", "/") 99 if path[-1] == "/": 100 path = path[:-1] 101 return path 102 103 104 def _symlink_genrule_for_dir(repository_ctx, 105 src_dir, 106 dest_dir, 107 genrule_name, 108 src_files=[], 109 dest_files=[]): 110 """Returns a genrule to symlink(or copy if on Windows) a set of files. 111 112 If src_dir is passed, files will be read from the given directory; otherwise 113 we assume files are in src_files and dest_files 114 """ 115 if src_dir != None: 116 src_dir = _normalize_path(src_dir) 117 dest_dir = _normalize_path(dest_dir) 118 files = '\n'.join( 119 sorted(_read_dir(repository_ctx, src_dir).splitlines())) 120 # Create a list with the src_dir stripped to use for outputs. 121 dest_files = files.replace(src_dir, '').splitlines() 122 src_files = files.splitlines() 123 command = [] 124 outs = [] 125 for i in range(len(dest_files)): 126 if dest_files[i] != "": 127 # If we have only one file to link we do not want to use the dest_dir, as 128 # $(@D) will include the full path to the file. 129 dest = '$(@D)/' + dest_dir + dest_files[i] if len( 130 dest_files) != 1 else '$(@D)/' + dest_files[i] 131 # On Windows, symlink is not supported, so we just copy all the files. 132 cmd = 'cp -f' if _is_windows(repository_ctx) else 'ln -s' 133 command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) 134 outs.append(' "' + dest_dir + dest_files[i] + '",') 135 return _genrule(src_dir, genrule_name, " && ".join(command), 136 "\n".join(outs)) 137 138 139 def _get_python_bin(repository_ctx): 140 """Gets the python bin path.""" 141 python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) 142 if python_bin != None: 143 return python_bin 144 python_bin_path = repository_ctx.which("python") 145 if python_bin_path != None: 146 return str(python_bin_path) 147 _fail("Cannot find python in PATH, please make sure " + 148 "python is installed and add its directory in PATH, or --define " + 149 "%s='/something/else'.\nPATH=%s" % 150 (_PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) 151 152 153 def _get_bash_bin(repository_ctx): 154 """Gets the bash bin path.""" 155 bash_bin = repository_ctx.os.environ.get(_BAZEL_SH) 156 if bash_bin != None: 157 return bash_bin 158 else: 159 bash_bin_path = repository_ctx.which("bash") 160 if bash_bin_path != None: 161 return str(bash_bin_path) 162 else: 163 _fail( 164 "Cannot find bash in PATH, please make sure " + 165 "bash is installed and add its directory in PATH, or --define " 166 + "%s='/path/to/bash'.\nPATH=%s" % 167 (_BAZEL_SH, repository_ctx.os.environ.get("PATH", ""))) 168 169 170 def _get_python_lib(repository_ctx, python_bin): 171 """Gets the python lib path.""" 172 python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) 173 if python_lib != None: 174 return python_lib 175 print_lib = ( 176 "<<END\n" + "from __future__ import print_function\n" + 177 "import site\n" + "import os\n" + "\n" + "try:\n" + 178 " input = raw_input\n" + "except NameError:\n" + " pass\n" + "\n" + 179 "python_paths = []\n" + "if os.getenv('PYTHONPATH') is not None:\n" + 180 " python_paths = os.getenv('PYTHONPATH').split(':')\n" + "try:\n" + 181 " library_paths = site.getsitepackages()\n" + 182 "except AttributeError:\n" + 183 " from distutils.sysconfig import get_python_lib\n" + 184 " library_paths = [get_python_lib()]\n" + 185 "all_paths = set(python_paths + library_paths)\n" + "paths = []\n" + 186 "for path in all_paths:\n" + " if os.path.isdir(path):\n" + 187 " paths.append(path)\n" + "if len(paths) >=1:\n" + 188 " print(paths[0])\n" + "END") 189 cmd = '%s - %s' % (python_bin, print_lib) 190 result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) 191 return result.stdout.strip('\n') 192 193 194 def _check_python_lib(repository_ctx, python_lib): 195 """Checks the python lib path.""" 196 cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) 197 result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) 198 if result.return_code == 1: 199 _fail("Invalid python library path: %s" % python_lib) 200 201 202 def _check_python_bin(repository_ctx, python_bin): 203 """Checks the python bin path.""" 204 cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) 205 result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) 206 if result.return_code == 1: 207 _fail("--define %s='%s' is not executable. Is it the python binary?" % 208 (_PYTHON_BIN_PATH, python_bin)) 209 210 211 def _get_python_include(repository_ctx, python_bin): 212 """Gets the python include path.""" 213 result = _execute( 214 repository_ctx, [ 215 python_bin, "-c", 'from __future__ import print_function;' + 216 'from distutils import sysconfig;' + 217 'print(sysconfig.get_python_inc())' 218 ], 219 error_msg="Problem getting python include path.", 220 error_details=( 221 "Is the Python binary path set up right? " + "(See ./configure or " 222 + _PYTHON_BIN_PATH + ".) " + "Is distutils installed?")) 223 return result.stdout.splitlines()[0] 224 225 226 def _get_python_import_lib_name(repository_ctx, python_bin): 227 """Get Python import library name (pythonXY.lib) on Windows.""" 228 result = _execute( 229 repository_ctx, [ 230 python_bin, "-c", 231 'import sys;' + 'print("python" + str(sys.version_info[0]) + ' + 232 ' str(sys.version_info[1]) + ".lib")' 233 ], 234 error_msg="Problem getting python import library.", 235 error_details=("Is the Python binary path set up right? " + 236 "(See ./configure or " + _PYTHON_BIN_PATH + ".) ")) 237 return result.stdout.splitlines()[0] 238 239 240 def _create_local_python_repository(repository_ctx): 241 """Creates the repository containing files set up to build with Python.""" 242 python_bin = _get_python_bin(repository_ctx) 243 _check_python_bin(repository_ctx, python_bin) 244 python_lib = _get_python_lib(repository_ctx, python_bin) 245 _check_python_lib(repository_ctx, python_lib) 246 python_include = _get_python_include(repository_ctx, python_bin) 247 python_include_rule = _symlink_genrule_for_dir( 248 repository_ctx, python_include, 'python_include', 'python_include') 249 python_import_lib_genrule = "" 250 # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib 251 # See https://docs.python.org/3/extending/windows.html 252 if _is_windows(repository_ctx): 253 python_include = _normalize_path(python_include) 254 python_import_lib_name = _get_python_import_lib_name( 255 repository_ctx, python_bin) 256 python_import_lib_src = python_include.rsplit( 257 '/', 1)[0] + "/libs/" + python_import_lib_name 258 python_import_lib_genrule = _symlink_genrule_for_dir( 259 repository_ctx, None, '', 'python_import_lib', 260 [python_import_lib_src], [python_import_lib_name]) 261 _tpl( 262 repository_ctx, "BUILD", { 263 "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, 264 "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, 265 }) 266 267 268 def _create_remote_python_repository(repository_ctx, remote_config_repo): 269 """Creates pointers to a remotely configured repo set up to build with Python. 270 """ 271 _tpl(repository_ctx, "remote.BUILD", { 272 "%{REMOTE_PYTHON_REPO}": remote_config_repo, 273 }, "BUILD") 274 275 276 def _python_autoconf_impl(repository_ctx): 277 """Implementation of the python_autoconf repository rule.""" 278 if _PYTHON_CONFIG_REPO in repository_ctx.os.environ: 279 _create_remote_python_repository( 280 repository_ctx, repository_ctx.os.environ[_PYTHON_CONFIG_REPO]) 281 else: 282 _create_local_python_repository(repository_ctx) 283 284 285 python_configure = repository_rule( 286 implementation=_python_autoconf_impl, 287 environ=[ 288 _BAZEL_SH, 289 _PYTHON_BIN_PATH, 290 _PYTHON_LIB_PATH, 291 _PYTHON_CONFIG_REPO, 292 ], 293 ) 294 """Detects and configures the local Python. 295 296 Add the following to your WORKSPACE FILE: 297 298 ```python 299 python_configure(name = "local_config_python") 300 ``` 301 302 Args: 303 name: A unique name for this workspace rule. 304 """ 305 306
