1 package bpdoc 2 3 import ( 4 "bytes" 5 "fmt" 6 "go/ast" 7 "go/doc" 8 "go/parser" 9 "go/token" 10 "io/ioutil" 11 "reflect" 12 "sort" 13 "strconv" 14 "strings" 15 "sync" 16 "text/template" 17 18 "github.com/google/blueprint" 19 "github.com/google/blueprint/proptools" 20 ) 21 22 type DocCollector struct { 23 pkgFiles map[string][]string // Map of package name to source files, provided by constructor 24 25 mutex sync.Mutex 26 pkgDocs map[string]*doc.Package // Map of package name to parsed Go AST, protected by mutex 27 docs map[string]*PropertyStructDocs // Map of type name to docs, protected by mutex 28 } 29 30 func NewDocCollector(pkgFiles map[string][]string) *DocCollector { 31 return &DocCollector{ 32 pkgFiles: pkgFiles, 33 pkgDocs: make(map[string]*doc.Package), 34 docs: make(map[string]*PropertyStructDocs), 35 } 36 } 37 38 // Return the PropertyStructDocs associated with a property struct type. The type should be in the 39 // format <package path>.<type name> 40 func (dc *DocCollector) Docs(pkg, name string, defaults reflect.Value) (*PropertyStructDocs, error) { 41 docs := dc.getDocs(pkg, name) 42 43 if docs == nil { 44 pkgDocs, err := dc.packageDocs(pkg) 45 if err != nil { 46 return nil, err 47 } 48 49 for _, t := range pkgDocs.Types { 50 if t.Name == name { 51 docs, err = newDocs(t) 52 if err != nil { 53 return nil, err 54 } 55 docs = dc.putDocs(pkg, name, docs) 56 } 57 } 58 } 59 60 if docs == nil { 61 return nil, fmt.Errorf("package %q type %q not found", pkg, name) 62 } 63 64 docs = docs.Clone() 65 docs.SetDefaults(defaults) 66 67 return docs, nil 68 } 69 70 func (dc *DocCollector) getDocs(pkg, name string) *PropertyStructDocs { 71 dc.mutex.Lock() 72 defer dc.mutex.Unlock() 73 74 name = pkg + "." + name 75 76 return dc.docs[name] 77 } 78 79 func (dc *DocCollector) putDocs(pkg, name string, docs *PropertyStructDocs) *PropertyStructDocs { 80 dc.mutex.Lock() 81 defer dc.mutex.Unlock() 82 83 name = pkg + "." + name 84 85 if dc.docs[name] != nil { 86 return dc.docs[name] 87 } else { 88 dc.docs[name] = docs 89 return docs 90 } 91 } 92 93 type PropertyStructDocs struct { 94 Name string 95 Text string 96 Properties []PropertyDocs 97 } 98 99 type PropertyDocs struct { 100 Name string 101 OtherNames []string 102 Type string 103 Tag reflect.StructTag 104 Text string 105 OtherTexts []string 106 Properties []PropertyDocs 107 Default string 108 } 109 110 func (docs *PropertyStructDocs) Clone() *PropertyStructDocs { 111 ret := *docs 112 ret.Properties = append([]PropertyDocs(nil), ret.Properties...) 113 for i, prop := range ret.Properties { 114 ret.Properties[i] = prop.Clone() 115 } 116 117 return &ret 118 } 119 120 func (docs *PropertyDocs) Clone() PropertyDocs { 121 ret := *docs 122 ret.Properties = append([]PropertyDocs(nil), ret.Properties...) 123 for i, prop := range ret.Properties { 124 ret.Properties[i] = prop.Clone() 125 } 126 127 return ret 128 } 129 130 func (docs *PropertyDocs) Equal(other PropertyDocs) bool { 131 return docs.Name == other.Name && docs.Type == other.Type && docs.Tag == other.Tag && 132 docs.Text == other.Text && docs.Default == other.Default && 133 stringArrayEqual(docs.OtherNames, other.OtherNames) && 134 stringArrayEqual(docs.OtherTexts, other.OtherTexts) && 135 docs.SameSubProperties(other) 136 } 137 138 func (docs *PropertyStructDocs) SetDefaults(defaults reflect.Value) { 139 setDefaults(docs.Properties, defaults) 140 } 141 142 func setDefaults(properties []PropertyDocs, defaults reflect.Value) { 143 for i := range properties { 144 prop := &properties[i] 145 fieldName := proptools.FieldNameForProperty(prop.Name) 146 f := defaults.FieldByName(fieldName) 147 if (f == reflect.Value{}) { 148 panic(fmt.Errorf("property %q does not exist in %q", fieldName, defaults.Type())) 149 } 150 151 if reflect.DeepEqual(f.Interface(), reflect.Zero(f.Type()).Interface()) { 152 continue 153 } 154 155 if f.Type().Kind() == reflect.Interface { 156 f = f.Elem() 157 } 158 159 if f.Type().Kind() == reflect.Ptr { 160 f = f.Elem() 161 } 162 163 if f.Type().Kind() == reflect.Struct { 164 setDefaults(prop.Properties, f) 165 } else { 166 prop.Default = fmt.Sprintf("%v", f.Interface()) 167 } 168 } 169 } 170 171 func stringArrayEqual(a, b []string) bool { 172 if len(a) != len(b) { 173 return false 174 } 175 176 for i := range a { 177 if a[i] != b[i] { 178 return false 179 } 180 } 181 182 return true 183 } 184 185 func (docs *PropertyDocs) SameSubProperties(other PropertyDocs) bool { 186 if len(docs.Properties) != len(other.Properties) { 187 return false 188 } 189 190 for i := range docs.Properties { 191 if !docs.Properties[i].Equal(other.Properties[i]) { 192 return false 193 } 194 } 195 196 return true 197 } 198 199 func (docs *PropertyStructDocs) GetByName(name string) *PropertyDocs { 200 return getByName(name, "", &docs.Properties) 201 } 202 203 func getByName(name string, prefix string, props *[]PropertyDocs) *PropertyDocs { 204 for i := range *props { 205 if prefix+(*props)[i].Name == name { 206 return &(*props)[i] 207 } else if strings.HasPrefix(name, prefix+(*props)[i].Name+".") { 208 return getByName(name, prefix+(*props)[i].Name+".", &(*props)[i].Properties) 209 } 210 } 211 return nil 212 } 213 214 func (prop *PropertyDocs) Nest(nested *PropertyStructDocs) { 215 //prop.Name += "(" + nested.Name + ")" 216 //prop.Text += "(" + nested.Text + ")" 217 prop.Properties = append(prop.Properties, nested.Properties...) 218 } 219 220 func newDocs(t *doc.Type) (*PropertyStructDocs, error) { 221 typeSpec := t.Decl.Specs[0].(*ast.TypeSpec) 222 docs := PropertyStructDocs{ 223 Name: t.Name, 224 Text: t.Doc, 225 } 226 227 structType, ok := typeSpec.Type.(*ast.StructType) 228 if !ok { 229 return nil, fmt.Errorf("type of %q is not a struct", t.Name) 230 } 231 232 var err error 233 docs.Properties, err = structProperties(structType) 234 if err != nil { 235 return nil, err 236 } 237 238 return &docs, nil 239 } 240 241 func structProperties(structType *ast.StructType) (props []PropertyDocs, err error) { 242 for _, f := range structType.Fields.List { 243 names := f.Names 244 if names == nil { 245 // Anonymous fields have no name, use the type as the name 246 // TODO: hide the name and make the properties show up in the embedding struct 247 if t, ok := f.Type.(*ast.Ident); ok { 248 names = append(names, t) 249 } 250 } 251 for _, n := range names { 252 var name, typ, tag, text string 253 var innerProps []PropertyDocs 254 if n != nil { 255 name = proptools.PropertyNameForField(n.Name) 256 } 257 if f.Doc != nil { 258 text = f.Doc.Text() 259 } 260 if f.Tag != nil { 261 tag, err = strconv.Unquote(f.Tag.Value) 262 if err != nil { 263 return nil, err 264 } 265 } 266 switch a := f.Type.(type) { 267 case *ast.ArrayType: 268 typ = "list of strings" 269 case *ast.InterfaceType: 270 typ = "interface" 271 case *ast.Ident: 272 typ = a.Name 273 case *ast.StructType: 274 innerProps, err = structProperties(a) 275 if err != nil { 276 return nil, err 277 } 278 default: 279 typ = fmt.Sprintf("%T", f.Type) 280 } 281 282 props = append(props, PropertyDocs{ 283 Name: name, 284 Type: typ, 285 Tag: reflect.StructTag(tag), 286 Text: text, 287 Properties: innerProps, 288 }) 289 } 290 } 291 292 return props, nil 293 } 294 295 func (docs *PropertyStructDocs) ExcludeByTag(key, value string) { 296 filterPropsByTag(&docs.Properties, key, value, true) 297 } 298 299 func (docs *PropertyStructDocs) IncludeByTag(key, value string) { 300 filterPropsByTag(&docs.Properties, key, value, false) 301 } 302 303 func filterPropsByTag(props *[]PropertyDocs, key, value string, exclude bool) { 304 // Create a slice that shares the storage of props but has 0 length. Appending up to 305 // len(props) times to this slice will overwrite the original slice contents 306 filtered := (*props)[:0] 307 for _, x := range *props { 308 tag := x.Tag.Get(key) 309 for _, entry := range strings.Split(tag, ",") { 310 if (entry == value) == !exclude { 311 filtered = append(filtered, x) 312 } 313 } 314 } 315 316 *props = filtered 317 } 318 319 // Package AST generation and storage 320 func (dc *DocCollector) packageDocs(pkg string) (*doc.Package, error) { 321 pkgDocs := dc.getPackageDocs(pkg) 322 if pkgDocs == nil { 323 if files, ok := dc.pkgFiles[pkg]; ok { 324 var err error 325 pkgAST, err := NewPackageAST(files) 326 if err != nil { 327 return nil, err 328 } 329 pkgDocs = doc.New(pkgAST, pkg, doc.AllDecls) 330 pkgDocs = dc.putPackageDocs(pkg, pkgDocs) 331 } else { 332 return nil, fmt.Errorf("unknown package %q", pkg) 333 } 334 } 335 return pkgDocs, nil 336 } 337 338 func (dc *DocCollector) getPackageDocs(pkg string) *doc.Package { 339 dc.mutex.Lock() 340 defer dc.mutex.Unlock() 341 342 return dc.pkgDocs[pkg] 343 } 344 345 func (dc *DocCollector) putPackageDocs(pkg string, pkgDocs *doc.Package) *doc.Package { 346 dc.mutex.Lock() 347 defer dc.mutex.Unlock() 348 349 if dc.pkgDocs[pkg] != nil { 350 return dc.pkgDocs[pkg] 351 } else { 352 dc.pkgDocs[pkg] = pkgDocs 353 return pkgDocs 354 } 355 } 356 357 func NewPackageAST(files []string) (*ast.Package, error) { 358 asts := make(map[string]*ast.File) 359 360 fset := token.NewFileSet() 361 for _, file := range files { 362 ast, err := parser.ParseFile(fset, file, nil, parser.ParseComments) 363 if err != nil { 364 return nil, err 365 } 366 asts[file] = ast 367 } 368 369 pkg, _ := ast.NewPackage(fset, asts, nil, nil) 370 return pkg, nil 371 } 372 373 func Write(filename string, pkgFiles map[string][]string, 374 moduleTypePropertyStructs map[string][]interface{}) error { 375 376 docSet := NewDocCollector(pkgFiles) 377 378 var moduleTypeList []*moduleTypeDoc 379 for moduleType, propertyStructs := range moduleTypePropertyStructs { 380 mtDoc, err := getModuleTypeDoc(docSet, moduleType, propertyStructs) 381 if err != nil { 382 return err 383 } 384 removeEmptyPropertyStructs(mtDoc) 385 collapseDuplicatePropertyStructs(mtDoc) 386 collapseNestedPropertyStructs(mtDoc) 387 combineDuplicateProperties(mtDoc) 388 moduleTypeList = append(moduleTypeList, mtDoc) 389 } 390 391 sort.Sort(moduleTypeByName(moduleTypeList)) 392 393 buf := &bytes.Buffer{} 394 395 unique := 0 396 397 tmpl, err := template.New("file").Funcs(map[string]interface{}{ 398 "unique": func() int { 399 unique++ 400 return unique 401 }}).Parse(fileTemplate) 402 if err != nil { 403 return err 404 } 405 406 err = tmpl.Execute(buf, moduleTypeList) 407 if err != nil { 408 return err 409 } 410 411 err = ioutil.WriteFile(filename, buf.Bytes(), 0666) 412 if err != nil { 413 return err 414 } 415 416 return nil 417 } 418 419 func getModuleTypeDoc(docSet *DocCollector, moduleType string, 420 propertyStructs []interface{}) (*moduleTypeDoc, error) { 421 mtDoc := &moduleTypeDoc{ 422 Name: moduleType, 423 //Text: docSet.ModuleTypeDocs(moduleType), 424 } 425 426 for _, s := range propertyStructs { 427 v := reflect.ValueOf(s).Elem() 428 t := v.Type() 429 430 // Ignore property structs with unexported or unnamed types 431 if t.PkgPath() == "" { 432 continue 433 } 434 psDoc, err := docSet.Docs(t.PkgPath(), t.Name(), v) 435 if err != nil { 436 return nil, err 437 } 438 psDoc.ExcludeByTag("blueprint", "mutated") 439 440 for nested, nestedValue := range nestedPropertyStructs(v) { 441 nestedType := nestedValue.Type() 442 443 // Ignore property structs with unexported or unnamed types 444 if nestedType.PkgPath() == "" { 445 continue 446 } 447 nestedDoc, err := docSet.Docs(nestedType.PkgPath(), nestedType.Name(), nestedValue) 448 if err != nil { 449 return nil, err 450 } 451 nestedDoc.ExcludeByTag("blueprint", "mutated") 452 nestPoint := psDoc.GetByName(nested) 453 if nestPoint == nil { 454 return nil, fmt.Errorf("nesting point %q not found", nested) 455 } 456 457 key, value, err := blueprint.HasFilter(nestPoint.Tag) 458 if err != nil { 459 return nil, err 460 } 461 if key != "" { 462 nestedDoc.IncludeByTag(key, value) 463 } 464 465 nestPoint.Nest(nestedDoc) 466 } 467 mtDoc.PropertyStructs = append(mtDoc.PropertyStructs, psDoc) 468 } 469 470 return mtDoc, nil 471 } 472 473 func nestedPropertyStructs(s reflect.Value) map[string]reflect.Value { 474 ret := make(map[string]reflect.Value) 475 var walk func(structValue reflect.Value, prefix string) 476 walk = func(structValue reflect.Value, prefix string) { 477 typ := structValue.Type() 478 for i := 0; i < structValue.NumField(); i++ { 479 field := typ.Field(i) 480 if field.PkgPath != "" { 481 // The field is not exported so just skip it. 482 continue 483 } 484 485 fieldValue := structValue.Field(i) 486 487 switch fieldValue.Kind() { 488 case reflect.Bool, reflect.String, reflect.Slice, reflect.Int, reflect.Uint: 489 // Nothing 490 case reflect.Struct: 491 walk(fieldValue, prefix+proptools.PropertyNameForField(field.Name)+".") 492 case reflect.Ptr, reflect.Interface: 493 if !fieldValue.IsNil() { 494 // We leave the pointer intact and zero out the struct that's 495 // pointed to. 496 elem := fieldValue.Elem() 497 if fieldValue.Kind() == reflect.Interface { 498 if elem.Kind() != reflect.Ptr { 499 panic(fmt.Errorf("can't get type of field %q: interface "+ 500 "refers to a non-pointer", field.Name)) 501 } 502 elem = elem.Elem() 503 } 504 if elem.Kind() == reflect.Struct { 505 nestPoint := prefix + proptools.PropertyNameForField(field.Name) 506 ret[nestPoint] = elem 507 walk(elem, nestPoint+".") 508 } 509 } 510 default: 511 panic(fmt.Errorf("unexpected kind for property struct field %q: %s", 512 field.Name, fieldValue.Kind())) 513 } 514 } 515 516 } 517 518 walk(s, "") 519 return ret 520 } 521 522 // Remove any property structs that have no exported fields 523 func removeEmptyPropertyStructs(mtDoc *moduleTypeDoc) { 524 for i := 0; i < len(mtDoc.PropertyStructs); i++ { 525 if len(mtDoc.PropertyStructs[i].Properties) == 0 { 526 mtDoc.PropertyStructs = append(mtDoc.PropertyStructs[:i], mtDoc.PropertyStructs[i+1:]...) 527 i-- 528 } 529 } 530 } 531 532 // Squashes duplicates of the same property struct into single entries 533 func collapseDuplicatePropertyStructs(mtDoc *moduleTypeDoc) { 534 var collapsedDocs []*PropertyStructDocs 535 536 propertyStructLoop: 537 for _, from := range mtDoc.PropertyStructs { 538 for _, to := range collapsedDocs { 539 if from.Name == to.Name { 540 collapseDuplicateProperties(&to.Properties, &from.Properties) 541 continue propertyStructLoop 542 } 543 } 544 collapsedDocs = append(collapsedDocs, from) 545 } 546 mtDoc.PropertyStructs = collapsedDocs 547 } 548 549 func collapseDuplicateProperties(to, from *[]PropertyDocs) { 550 propertyLoop: 551 for _, f := range *from { 552 for i := range *to { 553 t := &(*to)[i] 554 if f.Name == t.Name { 555 collapseDuplicateProperties(&t.Properties, &f.Properties) 556 continue propertyLoop 557 } 558 } 559 *to = append(*to, f) 560 } 561 } 562 563 // Find all property structs that only contain structs, and move their children up one with 564 // a prefixed name 565 func collapseNestedPropertyStructs(mtDoc *moduleTypeDoc) { 566 for _, ps := range mtDoc.PropertyStructs { 567 collapseNestedProperties(&ps.Properties) 568 } 569 } 570 571 func collapseNestedProperties(p *[]PropertyDocs) { 572 var n []PropertyDocs 573 574 for _, parent := range *p { 575 var containsProperty bool 576 for j := range parent.Properties { 577 child := &parent.Properties[j] 578 if len(child.Properties) > 0 { 579 collapseNestedProperties(&child.Properties) 580 } else { 581 containsProperty = true 582 } 583 } 584 if containsProperty || len(parent.Properties) == 0 { 585 n = append(n, parent) 586 } else { 587 for j := range parent.Properties { 588 child := parent.Properties[j] 589 child.Name = parent.Name + "." + child.Name 590 n = append(n, child) 591 } 592 } 593 } 594 *p = n 595 } 596 597 func combineDuplicateProperties(mtDoc *moduleTypeDoc) { 598 for _, ps := range mtDoc.PropertyStructs { 599 combineDuplicateSubProperties(&ps.Properties) 600 } 601 } 602 603 func combineDuplicateSubProperties(p *[]PropertyDocs) { 604 var n []PropertyDocs 605 propertyLoop: 606 for _, child := range *p { 607 if len(child.Properties) > 0 { 608 combineDuplicateSubProperties(&child.Properties) 609 for i := range n { 610 s := &n[i] 611 if s.SameSubProperties(child) { 612 s.OtherNames = append(s.OtherNames, child.Name) 613 s.OtherTexts = append(s.OtherTexts, child.Text) 614 continue propertyLoop 615 } 616 } 617 } 618 n = append(n, child) 619 } 620 621 *p = n 622 } 623 624 type moduleTypeByName []*moduleTypeDoc 625 626 func (l moduleTypeByName) Len() int { return len(l) } 627 func (l moduleTypeByName) Less(i, j int) bool { return l[i].Name < l[j].Name } 628 func (l moduleTypeByName) Swap(i, j int) { l[i], l[j] = l[j], l[i] } 629 630 type moduleTypeDoc struct { 631 Name string 632 Text string 633 PropertyStructs []*PropertyStructDocs 634 } 635 636 var ( 637 fileTemplate = ` 638 <html> 639 <head> 640 <title>Build Docs</title> 641 <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css"> 642 <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js"></script> 643 <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script> 644 </head> 645 <body> 646 <h1>Build Docs</h1> 647 <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true"> 648 {{range .}} 649 {{ $collapseIndex := unique }} 650 <div class="panel panel-default"> 651 <div class="panel-heading" role="tab" id="heading{{$collapseIndex}}"> 652 <h2 class="panel-title"> 653 <a class="collapsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse{{$collapseIndex}}" aria-expanded="false" aria-controls="collapse{{$collapseIndex}}"> 654 {{.Name}} 655 </a> 656 </h2> 657 </div> 658 </div> 659 <div id="collapse{{$collapseIndex}}" class="panel-collapse collapse" role="tabpanel" aria-labelledby="heading{{$collapseIndex}}"> 660 <div class="panel-body"> 661 <p>{{.Text}}</p> 662 {{range .PropertyStructs}} 663 <p>{{.Text}}</p> 664 {{template "properties" .Properties}} 665 {{end}} 666 </div> 667 </div> 668 {{end}} 669 </div> 670 </body> 671 </html> 672 673 {{define "properties"}} 674 <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true"> 675 {{range .}} 676 {{$collapseIndex := unique}} 677 {{if .Properties}} 678 <div class="panel panel-default"> 679 <div class="panel-heading" role="tab" id="heading{{$collapseIndex}}"> 680 <h4 class="panel-title"> 681 <a class="collapsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapse{{$collapseIndex}}" aria-expanded="false" aria-controls="collapse{{$collapseIndex}}"> 682 {{.Name}}{{range .OtherNames}}, {{.}}{{end}} 683 </a> 684 </h4> 685 </div> 686 </div> 687 <div id="collapse{{$collapseIndex}}" class="panel-collapse collapse" role="tabpanel" aria-labelledby="heading{{$collapseIndex}}"> 688 <div class="panel-body"> 689 <p>{{.Text}}</p> 690 {{range .OtherTexts}}<p>{{.}}</p>{{end}} 691 {{template "properties" .Properties}} 692 </div> 693 </div> 694 {{else}} 695 <div> 696 <h4>{{.Name}}{{range .OtherNames}}, {{.}}{{end}}</h4> 697 <p>{{.Text}}</p> 698 {{range .OtherTexts}}<p>{{.}}</p>{{end}} 699 <p><i>Type: {{.Type}}</i></p> 700 {{if .Default}}<p><i>Default: {{.Default}}</i></p>{{end}} 701 </div> 702 {{end}} 703 {{end}} 704 </div> 705 {{end}} 706 ` 707 ) 708