1 # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # ============================================================================== 15 """A `traverse` visitor for processing documentation.""" 16 17 from __future__ import absolute_import 18 from __future__ import division 19 from __future__ import print_function 20 21 import six 22 23 from tensorflow.python.util import tf_export 24 from tensorflow.python.util import tf_inspect 25 26 27 class DocGeneratorVisitor(object): 28 """A visitor that generates docs for a python object when __call__ed.""" 29 30 def __init__(self, root_name=''): 31 """Make a visitor. 32 33 As this visitor is starting its traversal at a module or class, it will not 34 be told the name of that object during traversal. `root_name` is the name it 35 should use for that object, effectively prefixing all names with 36 "root_name.". 37 38 Args: 39 root_name: The name of the root module/class. 40 """ 41 self.set_root_name(root_name) 42 self._index = {} 43 self._tree = {} 44 self._reverse_index = None 45 self._duplicates = None 46 self._duplicate_of = None 47 48 def set_root_name(self, root_name): 49 """Sets the root name for subsequent __call__s.""" 50 self._root_name = root_name or '' 51 self._prefix = (root_name + '.') if root_name else '' 52 53 @property 54 def index(self): 55 """A map from fully qualified names to objects to be documented. 56 57 The index is filled when the visitor is passed to `traverse`. 58 59 Returns: 60 The index filled by traversal. 61 """ 62 return self._index 63 64 @property 65 def tree(self): 66 """A map from fully qualified names to all its child names for traversal. 67 68 The full name to member names map is filled when the visitor is passed to 69 `traverse`. 70 71 Returns: 72 The full name to member name map filled by traversal. 73 """ 74 return self._tree 75 76 @property 77 def reverse_index(self): 78 """A map from `id(object)` to the preferred fully qualified name. 79 80 This map only contains non-primitive objects (no numbers or strings) present 81 in `index` (for primitive objects, `id()` doesn't quite do the right thing). 82 83 It is computed when it, `duplicate_of`, or `duplicates` are first accessed. 84 85 Returns: 86 The `id(object)` to full name map. 87 """ 88 self._maybe_find_duplicates() 89 return self._reverse_index 90 91 @property 92 def duplicate_of(self): 93 """A map from duplicate full names to a preferred fully qualified name. 94 95 This map only contains names that are not themself a preferred name. 96 97 It is computed when it, `reverse_index`, or `duplicates` are first accessed. 98 99 Returns: 100 The map from duplicate name to preferred name. 101 """ 102 self._maybe_find_duplicates() 103 return self._duplicate_of 104 105 @property 106 def duplicates(self): 107 """A map from preferred full names to a list of all names for this symbol. 108 109 This function returns a map from preferred (master) name for a symbol to a 110 lexicographically sorted list of all aliases for that name (incl. the master 111 name). Symbols without duplicate names do not appear in this map. 112 113 It is computed when it, `reverse_index`, or `duplicate_of` are first 114 accessed. 115 116 Returns: 117 The map from master name to list of all duplicate names. 118 """ 119 self._maybe_find_duplicates() 120 return self._duplicates 121 122 def _add_prefix(self, name): 123 """Adds the root name to a name.""" 124 return self._prefix + name if name else self._root_name 125 126 def __call__(self, parent_name, parent, children): 127 """Visitor interface, see `tensorflow/tools/common:traverse` for details. 128 129 This method is called for each symbol found in a traversal using 130 `tensorflow/tools/common:traverse`. It should not be called directly in 131 user code. 132 133 Args: 134 parent_name: The fully qualified name of a symbol found during traversal. 135 parent: The Python object referenced by `parent_name`. 136 children: A list of `(name, py_object)` pairs enumerating, in alphabetical 137 order, the children (as determined by `tf_inspect.getmembers`) of 138 `parent`. `name` is the local name of `py_object` in `parent`. 139 140 Raises: 141 RuntimeError: If this visitor is called with a `parent` that is not a 142 class or module. 143 """ 144 parent_name = self._add_prefix(parent_name) 145 self._index[parent_name] = parent 146 self._tree[parent_name] = [] 147 148 if not (tf_inspect.ismodule(parent) or tf_inspect.isclass(parent)): 149 raise RuntimeError('Unexpected type in visitor -- %s: %r' % (parent_name, 150 parent)) 151 152 for i, (name, child) in enumerate(list(children)): 153 # Don't document __metaclass__ 154 if name in ['__metaclass__']: 155 del children[i] 156 continue 157 158 full_name = '.'.join([parent_name, name]) if parent_name else name 159 self._index[full_name] = child 160 self._tree[parent_name].append(name) 161 162 def _score_name(self, name): 163 """Return a tuple of scores indicating how to sort for the best name. 164 165 This function is meant to be used as the `key` to the `sorted` function. 166 167 This sorting in order: 168 Prefers names refering to the defining class, over a subclass. 169 Prefers names that are not in "contrib". 170 prefers submodules to the root namespace. 171 Prefers short names `tf.thing` over `tf.a.b.c.thing` 172 Sorts lexicographically on name parts. 173 174 Args: 175 name: the full name to score, for example `tf.estimator.Estimator` 176 177 Returns: 178 A tuple of scores. When sorted the preferred name will have the lowest 179 value. 180 """ 181 parts = name.split('.') 182 short_name = parts[-1] 183 184 container = self._index['.'.join(parts[:-1])] 185 186 defining_class_score = 1 187 if tf_inspect.isclass(container): 188 if short_name in container.__dict__: 189 # prefer the defining class 190 defining_class_score = -1 191 192 contrib_score = -1 193 if 'contrib' in parts: 194 contrib_score = 1 195 196 while parts: 197 container = self._index['.'.join(parts)] 198 if tf_inspect.ismodule(container): 199 break 200 parts.pop() 201 202 module_length = len(parts) 203 if len(parts) == 2: 204 # `tf.submodule.thing` is better than `tf.thing` 205 module_length_score = -1 206 else: 207 # shorter is better 208 module_length_score = module_length 209 210 return (defining_class_score, contrib_score, module_length_score, name) 211 212 def _maybe_find_duplicates(self): 213 """Compute data structures containing information about duplicates. 214 215 Find duplicates in `index` and decide on one to be the "master" name. 216 217 Computes a reverse_index mapping each object id to its master name. 218 219 Also computes a map `duplicate_of` from aliases to their master name (the 220 master name itself has no entry in this map), and a map `duplicates` from 221 master names to a lexicographically sorted list of all aliases for that name 222 (incl. the master name). 223 224 All these are computed and set as fields if they haven't already. 225 """ 226 if self._reverse_index is not None: 227 return 228 229 # Maps the id of a symbol to its fully qualified name. For symbols that have 230 # several aliases, this map contains the first one found. 231 # We use id(py_object) to get a hashable value for py_object. Note all 232 # objects in _index are in memory at the same time so this is safe. 233 reverse_index = {} 234 235 # Make a preliminary duplicates map. For all sets of duplicate names, it 236 # maps the first name found to a list of all duplicate names. 237 raw_duplicates = {} 238 for full_name, py_object in six.iteritems(self._index): 239 # We cannot use the duplicate mechanism for some constants, since e.g., 240 # id(c1) == id(c2) with c1=1, c2=1. This is unproblematic since constants 241 # have no usable docstring and won't be documented automatically. 242 if (py_object is not None and 243 not isinstance(py_object, six.integer_types + six.string_types + 244 (six.binary_type, six.text_type, float, complex, bool)) 245 and py_object is not ()): # pylint: disable=literal-comparison 246 object_id = id(py_object) 247 if object_id in reverse_index: 248 master_name = reverse_index[object_id] 249 if master_name in raw_duplicates: 250 raw_duplicates[master_name].append(full_name) 251 else: 252 raw_duplicates[master_name] = [master_name, full_name] 253 else: 254 reverse_index[object_id] = full_name 255 # Decide on master names, rewire duplicates and make a duplicate_of map 256 # mapping all non-master duplicates to the master name. The master symbol 257 # does not have an entry in this map. 258 duplicate_of = {} 259 # Duplicates maps the main symbols to the set of all duplicates of that 260 # symbol (incl. itself). 261 duplicates = {} 262 for names in raw_duplicates.values(): 263 names = sorted(names) 264 master_name = ( 265 tf_export.get_canonical_name_for_symbol(self._index[names[0]]) 266 if names else None) 267 if master_name: 268 master_name = 'tf.%s' % master_name 269 else: 270 # Choose the master name with a lexical sort on the tuples returned by 271 # by _score_name. 272 master_name = min(names, key=self._score_name) 273 274 duplicates[master_name] = names 275 for name in names: 276 if name != master_name: 277 duplicate_of[name] = master_name 278 279 # Set the reverse index to the canonical name. 280 reverse_index[id(self._index[master_name])] = master_name 281 282 self._duplicate_of = duplicate_of 283 self._duplicates = duplicates 284 self._reverse_index = reverse_index 285
