1 // Copyright 2017 Google Inc. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package python 16 17 // This file contains the "Base" module type for building Python program. 18 19 import ( 20 "fmt" 21 "path/filepath" 22 "regexp" 23 "sort" 24 "strings" 25 26 "github.com/google/blueprint" 27 28 "android/soong/android" 29 ) 30 31 func init() { 32 android.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { 33 ctx.BottomUp("version_split", versionSplitMutator()).Parallel() 34 }) 35 } 36 37 // the version properties that apply to python libraries and binaries. 38 type PythonVersionProperties struct { 39 // true, if the module is required to be built with this version. 40 Enabled *bool 41 42 // if specified, common src files are converted to specific version with converter tool. 43 // Converter bool 44 45 // non-empty list of .py files under this strict Python version. 46 // srcs may reference the outputs of other modules that produce source files like genrule 47 // or filegroup using the syntax ":module". 48 Srcs []string 49 50 // list of the Python libraries under this Python version. 51 Libs []string 52 } 53 54 // properties that apply to python libraries and binaries. 55 type PythonBaseModuleProperties struct { 56 // the package path prefix within the output artifact at which to place the source/data 57 // files of the current module. 58 // eg. Pkg_path = "a/b/c"; Other packages can reference this module by using 59 // (from a.b.c import ...) statement. 60 // if left unspecified, all the source/data files of current module are copied to 61 // "runfiles/" tree directory directly. 62 Pkg_path string 63 64 // list of source (.py) files compatible both with Python2 and Python3 used to compile the 65 // Python module. 66 // srcs may reference the outputs of other modules that produce source files like genrule 67 // or filegroup using the syntax ":module". 68 // Srcs has to be non-empty. 69 Srcs []string 70 71 // list of files or filegroup modules that provide data that should be installed alongside 72 // the test. the file extension can be arbitrary except for (.py). 73 Data []string 74 75 // list of the Python libraries compatible both with Python2 and Python3. 76 Libs []string 77 78 Version struct { 79 // all the "srcs" or Python dependencies that are to be used only for Python2. 80 Py2 PythonVersionProperties 81 82 // all the "srcs" or Python dependencies that are to be used only for Python3. 83 Py3 PythonVersionProperties 84 } 85 86 // the actual version each module uses after variations created. 87 // this property name is hidden from users' perspectives, and soong will populate it during 88 // runtime. 89 ActualVersion string `blueprint:"mutated"` 90 } 91 92 type pathMapping struct { 93 dest string 94 src android.Path 95 } 96 97 type pythonBaseModule struct { 98 android.ModuleBase 99 subModule PythonSubModule 100 101 properties PythonBaseModuleProperties 102 103 // the Python files of current module after expanding source dependencies. 104 // pathMapping: <dest: runfile_path, src: source_path> 105 srcsPathMappings []pathMapping 106 107 // the data files of current module after expanding source dependencies. 108 // pathMapping: <dest: runfile_path, src: source_path> 109 dataPathMappings []pathMapping 110 111 // the soong_zip arguments for zipping current module source/data files. 112 parSpec parSpec 113 114 // the installer might be nil. 115 installer installer 116 117 subAndroidMkOnce map[subAndroidMkProvider]bool 118 } 119 120 type PythonSubModule interface { 121 GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath 122 } 123 124 type PythonDependency interface { 125 GetSrcsPathMappings() []pathMapping 126 GetDataPathMappings() []pathMapping 127 GetParSpec() parSpec 128 } 129 130 type pythonDecorator struct { 131 baseInstaller *pythonInstaller 132 } 133 134 type installer interface { 135 install(ctx android.ModuleContext, path android.Path) 136 } 137 138 func (p *pythonBaseModule) GetSrcsPathMappings() []pathMapping { 139 return p.srcsPathMappings 140 } 141 142 func (p *pythonBaseModule) GetDataPathMappings() []pathMapping { 143 return p.dataPathMappings 144 } 145 146 func (p *pythonBaseModule) GetParSpec() parSpec { 147 return p.parSpec 148 } 149 150 var _ PythonDependency = (*pythonBaseModule)(nil) 151 152 var _ android.AndroidMkDataProvider = (*pythonBaseModule)(nil) 153 154 func InitPythonBaseModule(baseModule *pythonBaseModule, subModule PythonSubModule, 155 hod android.HostOrDeviceSupported) android.Module { 156 157 baseModule.subModule = subModule 158 159 baseModule.AddProperties(&baseModule.properties) 160 161 android.InitAndroidArchModule(baseModule, hod, android.MultilibCommon) 162 163 return baseModule 164 } 165 166 // the tag used to mark dependencies within "py_libs" attribute. 167 type pythonDependencyTag struct { 168 blueprint.BaseDependencyTag 169 } 170 171 var pyDependencyTag pythonDependencyTag 172 173 var ( 174 pyIdentifierRegexp = regexp.MustCompile(`^([a-z]|[A-Z]|_)([a-z]|[A-Z]|[0-9]|_)*$`) 175 pyExt = ".py" 176 pyVersion2 = "PY2" 177 pyVersion3 = "PY3" 178 initFileName = "__init__.py" 179 mainFileName = "__main__.py" 180 parFileExt = ".zip" 181 runFiles = "runfiles" 182 ) 183 184 // create version variants for modules. 185 func versionSplitMutator() func(android.BottomUpMutatorContext) { 186 return func(mctx android.BottomUpMutatorContext) { 187 if base, ok := mctx.Module().(*pythonBaseModule); ok { 188 versionNames := []string{} 189 if base.properties.Version.Py2.Enabled != nil && 190 *(base.properties.Version.Py2.Enabled) == true { 191 versionNames = append(versionNames, pyVersion2) 192 } 193 if !(base.properties.Version.Py3.Enabled != nil && 194 *(base.properties.Version.Py3.Enabled) == false) { 195 versionNames = append(versionNames, pyVersion3) 196 } 197 modules := mctx.CreateVariations(versionNames...) 198 for i, v := range versionNames { 199 // set the actual version for Python module. 200 modules[i].(*pythonBaseModule).properties.ActualVersion = v 201 } 202 } 203 } 204 } 205 206 func (p *pythonBaseModule) DepsMutator(ctx android.BottomUpMutatorContext) { 207 // deps from "data". 208 android.ExtractSourcesDeps(ctx, p.properties.Data) 209 // deps from "srcs". 210 android.ExtractSourcesDeps(ctx, p.properties.Srcs) 211 212 switch p.properties.ActualVersion { 213 case pyVersion2: 214 // deps from "version.py2.srcs" property. 215 android.ExtractSourcesDeps(ctx, p.properties.Version.Py2.Srcs) 216 217 ctx.AddVariationDependencies(nil, pyDependencyTag, 218 uniqueLibs(ctx, p.properties.Libs, "version.py2.libs", 219 p.properties.Version.Py2.Libs)...) 220 case pyVersion3: 221 // deps from "version.py3.srcs" property. 222 android.ExtractSourcesDeps(ctx, p.properties.Version.Py3.Srcs) 223 224 ctx.AddVariationDependencies(nil, pyDependencyTag, 225 uniqueLibs(ctx, p.properties.Libs, "version.py3.libs", 226 p.properties.Version.Py3.Libs)...) 227 default: 228 panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.", 229 p.properties.ActualVersion, ctx.ModuleName())) 230 } 231 } 232 233 // check "libs" duplicates from current module dependencies. 234 func uniqueLibs(ctx android.BottomUpMutatorContext, 235 commonLibs []string, versionProp string, versionLibs []string) []string { 236 set := make(map[string]string) 237 ret := []string{} 238 239 // deps from "libs" property. 240 for _, l := range commonLibs { 241 if _, found := set[l]; found { 242 ctx.PropertyErrorf("libs", "%q has duplicates within libs.", l) 243 } else { 244 set[l] = "libs" 245 ret = append(ret, l) 246 } 247 } 248 // deps from "version.pyX.libs" property. 249 for _, l := range versionLibs { 250 if _, found := set[l]; found { 251 ctx.PropertyErrorf(versionProp, "%q has duplicates within %q.", set[l]) 252 } else { 253 set[l] = versionProp 254 ret = append(ret, l) 255 } 256 } 257 258 return ret 259 } 260 261 func (p *pythonBaseModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { 262 installSource := p.subModule.GeneratePythonBuildActions(ctx) 263 264 if p.installer != nil && installSource.Valid() { 265 p.installer.install(ctx, installSource.Path()) 266 } 267 } 268 269 func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath { 270 // expand python files from "srcs" property. 271 srcs := p.properties.Srcs 272 switch p.properties.ActualVersion { 273 case pyVersion2: 274 srcs = append(srcs, p.properties.Version.Py2.Srcs...) 275 case pyVersion3: 276 srcs = append(srcs, p.properties.Version.Py3.Srcs...) 277 default: 278 panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.", 279 p.properties.ActualVersion, ctx.ModuleName())) 280 } 281 expandedSrcs := ctx.ExpandSources(srcs, nil) 282 if len(expandedSrcs) == 0 { 283 ctx.ModuleErrorf("doesn't have any source files!") 284 } 285 286 // expand data files from "data" property. 287 expandedData := ctx.ExpandSources(p.properties.Data, nil) 288 289 // sanitize pkg_path. 290 pkg_path := p.properties.Pkg_path 291 if pkg_path != "" { 292 pkg_path = filepath.Clean(p.properties.Pkg_path) 293 if pkg_path == ".." || strings.HasPrefix(pkg_path, "../") || 294 strings.HasPrefix(pkg_path, "/") { 295 ctx.PropertyErrorf("pkg_path", "%q is not a valid format.", 296 p.properties.Pkg_path) 297 return android.OptionalPath{} 298 } 299 // pkg_path starts from "runfiles/" implicitly. 300 pkg_path = filepath.Join(runFiles, pkg_path) 301 } else { 302 // pkg_path starts from "runfiles/" implicitly. 303 pkg_path = runFiles 304 } 305 306 p.genModulePathMappings(ctx, pkg_path, expandedSrcs, expandedData) 307 308 p.parSpec = p.dumpFileList(ctx, pkg_path) 309 310 p.uniqWholeRunfilesTree(ctx) 311 312 return android.OptionalPath{} 313 } 314 315 // generate current module unique pathMappings: <dest: runfiles_path, src: source_path> 316 // for python/data files. 317 func (p *pythonBaseModule) genModulePathMappings(ctx android.ModuleContext, pkg_path string, 318 expandedSrcs, expandedData android.Paths) { 319 // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to 320 // check duplicates. 321 destToPySrcs := make(map[string]string) 322 destToPyData := make(map[string]string) 323 324 for _, s := range expandedSrcs { 325 if s.Ext() != pyExt { 326 ctx.PropertyErrorf("srcs", "found non (.py) file: %q!", s.String()) 327 continue 328 } 329 runfilesPath := filepath.Join(pkg_path, s.Rel()) 330 identifiers := strings.Split(strings.TrimSuffix(runfilesPath, pyExt), "/") 331 for _, token := range identifiers { 332 if !pyIdentifierRegexp.MatchString(token) { 333 ctx.PropertyErrorf("srcs", "the path %q contains invalid token %q.", 334 runfilesPath, token) 335 } 336 } 337 if fillInMap(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) { 338 p.srcsPathMappings = append(p.srcsPathMappings, 339 pathMapping{dest: runfilesPath, src: s}) 340 } 341 } 342 343 for _, d := range expandedData { 344 if d.Ext() == pyExt { 345 ctx.PropertyErrorf("data", "found (.py) file: %q!", d.String()) 346 continue 347 } 348 runfilesPath := filepath.Join(pkg_path, d.Rel()) 349 if fillInMap(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) { 350 p.dataPathMappings = append(p.dataPathMappings, 351 pathMapping{dest: runfilesPath, src: d}) 352 } 353 } 354 355 } 356 357 // register build actions to dump filelist to disk. 358 func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec { 359 relativeRootMap := make(map[string]android.Paths) 360 // the soong_zip params in order to pack current module's Python/data files. 361 ret := parSpec{rootPrefix: pkg_path} 362 363 pathMappings := append(p.srcsPathMappings, p.dataPathMappings...) 364 365 // "srcs" or "data" properties may have filegroup so it might happen that 366 // the relative root for each source path is different. 367 for _, path := range pathMappings { 368 relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel()) 369 if v, found := relativeRootMap[relativeRoot]; found { 370 relativeRootMap[relativeRoot] = append(v, path.src) 371 } else { 372 relativeRootMap[relativeRoot] = android.Paths{path.src} 373 } 374 } 375 376 var keys []string 377 378 // in order to keep stable order of soong_zip params, we sort the keys here. 379 for k := range relativeRootMap { 380 keys = append(keys, k) 381 } 382 sort.Strings(keys) 383 384 for _, k := range keys { 385 // use relative root as filelist name. 386 fileListPath := registerBuildActionForModuleFileList( 387 ctx, strings.Replace(k, "/", "_", -1), relativeRootMap[k]) 388 ret.fileListSpecs = append(ret.fileListSpecs, 389 fileListSpec{fileList: fileListPath, relativeRoot: k}) 390 } 391 392 return ret 393 } 394 395 // check Python/data files duplicates from current module and its whole dependencies. 396 func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) { 397 // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to 398 // check duplicates. 399 destToPySrcs := make(map[string]string) 400 destToPyData := make(map[string]string) 401 402 for _, path := range p.srcsPathMappings { 403 destToPySrcs[path.dest] = path.src.String() 404 } 405 for _, path := range p.dataPathMappings { 406 destToPyData[path.dest] = path.src.String() 407 } 408 409 // visit all its dependencies in depth first. 410 ctx.VisitDepsDepthFirst(func(module blueprint.Module) { 411 // module can only depend on Python library. 412 if base, ok := module.(*pythonBaseModule); ok { 413 if _, ok := base.subModule.(*PythonLibrary); !ok { 414 panic(fmt.Errorf( 415 "the dependency %q of module %q is not Python library!", 416 ctx.ModuleName(), ctx.OtherModuleName(module))) 417 } 418 } else { 419 return 420 } 421 if dep, ok := module.(PythonDependency); ok { 422 srcs := dep.GetSrcsPathMappings() 423 for _, path := range srcs { 424 if !fillInMap(ctx, destToPySrcs, 425 path.dest, path.src.String(), ctx.ModuleName(), 426 ctx.OtherModuleName(module)) { 427 continue 428 } 429 // binary needs the Python runfiles paths from all its 430 // dependencies to fill __init__.py in each runfiles dir. 431 if sub, ok := p.subModule.(*pythonBinaryBase); ok { 432 sub.depsPyRunfiles = append(sub.depsPyRunfiles, path.dest) 433 } 434 } 435 data := dep.GetDataPathMappings() 436 for _, path := range data { 437 fillInMap(ctx, destToPyData, 438 path.dest, path.src.String(), ctx.ModuleName(), 439 ctx.OtherModuleName(module)) 440 } 441 // binary needs the soong_zip arguments from all its 442 // dependencies to generate executable par file. 443 if sub, ok := p.subModule.(*pythonBinaryBase); ok { 444 sub.depsParSpecs = append(sub.depsParSpecs, dep.GetParSpec()) 445 } 446 } 447 }) 448 } 449 450 func fillInMap(ctx android.ModuleContext, m map[string]string, 451 key, value, curModule, otherModule string) bool { 452 if oldValue, found := m[key]; found { 453 ctx.ModuleErrorf("found two files to be placed at the same runfiles location %q."+ 454 " First file: in module %s at path %q."+ 455 " Second file: in module %s at path %q.", 456 key, curModule, oldValue, otherModule, value) 457 return false 458 } else { 459 m[key] = value 460 } 461 462 return true 463 } 464