1  # epydoc -- Docstring processing 
  2  # 
  3  # Copyright (C) 2005 Edward Loper 
  4  # Author: Edward Loper <edloper@loper.org> 
  5  # URL: <http://epydoc.sf.net> 
  6  # 
  7  # $Id: docstringparser.py 1163 2006-04-05 02:28:48Z edloper $ 
  8   
  9  """ 
 10  Parse docstrings and handle any fields it defines, such as C{@type} 
 11  and C{@author}.  Fields are used to describe specific information 
 12  about an object.  There are two classes of fields: X{simple fields} 
 13  and X{special fields}. 
 14   
 15  Simple fields are fields that get stored directly in an C{APIDoc}'s 
 16  metadata dictionary, without any special processing.  The set of 
 17  simple fields is defined by the list L{STANDARD_FIELDS}, whose 
 18  elements are L{DocstringField}s. 
 19   
 20  Special fields are fields that perform some sort of processing on the 
 21  C{APIDoc}, or add information to attributes other than the metadata 
 22  dictionary.  Special fields are are handled by field handler 
 23  functions, which are registered using L{register_field_handler}. 
 24  """ 
 25  __docformat__ = 'epytext en' 
 26   
 27   
 28  ###################################################################### 
 29  ## Imports 
 30  ###################################################################### 
 31   
 32  import re, sys 
 33  from epydoc import markup 
 34  from epydoc.apidoc import * 
 35  from epydoc.docintrospecter import introspect_docstring_lineno 
 36  from epydoc.util import py_src_filename 
 37  from epydoc import log 
 38  import __builtin__, exceptions 
 39   
 40  ###################################################################### 
 41  # Docstring Fields 
 42  ###################################################################### 
 43   
 44 -class DocstringField: 
 45      """ 
 46      A simple docstring field, which can be used to describe specific 
 47      information about an object, such as its author or its version. 
 48      Simple docstring fields are fields that take no arguments, and 
 49      are displayed as simple sections. 
 50   
 51      @ivar tags: The set of tags that can be used to identify this 
 52          field. 
 53      @ivar singular: The label that should be used to identify this 
 54          field in the output, if the field contains one value. 
 55      @ivar plural: The label that should be used to identify this 
 56          field in the output, if the field contains multiple values. 
 57      @ivar short: If true, then multiple values should be combined 
 58          into a single comma-delimited list.  If false, then 
 59          multiple values should be listed separately in a bulleted 
 60          list. 
 61      @ivar multivalue: If true, then multiple values may be given 
 62          for this field; if false, then this field can only take a 
 63          single value, and a warning should be issued if it is 
 64          redefined. 
 65      @ivar takes_arg: If true, then this field expects an argument; 
 66          and a separate field section will be constructed for each 
 67          argument value.  The label (and plural label) should include 
 68          a '%s' to mark where the argument's string rep should be 
 69          added. 
 70      """ 
 71 -    def __init__(self, tags, label, plural=None, 
 72                   short=0, multivalue=1, takes_arg=0): 
 73          if type(tags) in (list, tuple): 
 74              self.tags = tuple(tags) 
 75          elif type(tags) is str: 
 76              self.tags = (tags,) 
 77          else: raise TypeError('Bad tags: %s' % tags) 
 78          self.singular = label 
 79          if plural is None: self.plural = label 
 80          else: self.plural = plural 
 81          self.multivalue = multivalue 
 82          self.short = short 
 83          self.takes_arg = takes_arg 
 84   
 85 -    def __cmp__(self, other): 
 86          if not isinstance(other, DocstringField): return -1 
 87          return cmp(self.tags, other.tags) 
 88       
 89 -    def __hash__(self): 
 90          return hash(self.tags) 
 91   
 92 -    def __repr__(self): 
 93          return '<Field: %s>' % self.tags[0] 
 94   
 95  STANDARD_FIELDS = [ 
 96      #: A list of the standard simple fields accepted by epydoc.  This 
 97      #: list can be augmented at run-time by a docstring with the special 
 98      #: C{@deffield} field.  The order in which fields are listed here 
 99      #: determines the order in which they will be displayed in the 
100      #: output. 
101       
102      # If it's deprecated, put that first. 
103      DocstringField(['deprecated', 'depreciated'], 
104               'Deprecated', multivalue=0), 
105   
106      # Status info 
107      DocstringField(['version'], 'Version', multivalue=0), 
108      DocstringField(['date'], 'Date', multivalue=0), 
109      DocstringField(['status'], 'Status', multivalue=0), 
110       
111      # Bibliographic Info 
112      DocstringField(['author', 'authors'], 'Author', 'Authors', short=1), 
113      DocstringField(['contact'], 'Contact', 'Contacts', short=1), 
114      DocstringField(['organization', 'org'], 
115                     'Organization', 'Organizations'), 
116      DocstringField(['copyright', '(c)'], 'Copyright', multivalue=0), 
117      DocstringField(['license'], 'License', multivalue=0), 
118   
119      # Various warnings etc. 
120      DocstringField(['bug'], 'Bug', 'Bugs'), 
121      DocstringField(['warning', 'warn'], 'Warning', 'Warnings'), 
122      DocstringField(['attention'], 'Attention'), 
123      DocstringField(['note'], 'Note', 'Notes'), 
124   
125      # Formal conditions 
126      DocstringField(['requires', 'require', 'requirement'], 'Requires'), 
127      DocstringField(['precondition', 'precond'], 
128               'Precondition', 'Preconditions'), 
129      DocstringField(['postcondition', 'postcond'], 
130               'Postcondition', 'Postconditions'), 
131      DocstringField(['invariant'], 'Invariant'), 
132   
133      # When was it introduced (version # or date) 
134      DocstringField(['since'], 'Since', multivalue=0), 
135   
136      # Crossreferences 
137      DocstringField(['see', 'seealso'], 'See Also', short=1), 
138   
139      # Future Work 
140      DocstringField(['todo'], 'To Do', takes_arg=True), 
141   
142      # Permissions (used by zope-based projects) 
143      DocstringField(['permission', 'permissions'], 'Permission', 'Permissions') 
144      ] 
145   
146  ###################################################################### 
147  #{ Docstring Parsing 
148  ###################################################################### 
149   
150  DEFAULT_DOCFORMAT = 'epytext' 
151  """The name of the default markup languge used to process docstrings.""" 
152   
153  # [xx] keep track of which ones we've already done, in case we're 
154  # asked to process one twice?  e.g., for @include we might have to 
155  # parse the included docstring earlier than we might otherwise..?? 
156   
157 -def parse_docstring(api_doc, docindex): 
158      """ 
159      Process the given C{APIDoc}'s docstring.  In particular, populate 
160      the C{APIDoc}'s C{descr} and C{summary} attributes, and add any 
161      information provided by fields in the docstring. 
162       
163      @param docindex: A DocIndex, used to find the containing 
164          module (to look up the docformat); and to find any 
165          user docfields defined by containing objects. 
166      """ 
167      if api_doc.metadata is not UNKNOWN: 
168          log.debug("%s's docstring processed twice" % api_doc.canonical_name) 
169          return 
170           
171      initialize_api_doc(api_doc) 
172   
173      # If there's no docstring, then there's nothing more to do. 
174      if (api_doc.docstring in (None, UNKNOWN)): 
175          return 
176   
177      # Remove leading indentation from the docstring. 
178      api_doc.docstring = unindent_docstring(api_doc.docstring) 
179   
180      # Extract a signature from the docstring, if it has one.  This 
181      # overrides any signature we got via introspection/parsing. 
182      if isinstance(api_doc, RoutineDoc): 
183          parse_function_signature(api_doc) 
184   
185      # Parse the docstring.  Any errors encountered are stored as 
186      # `ParseError` objects in the errors list. 
187      docformat = get_docformat(api_doc, docindex) 
188      parse_errors = [] 
189      parsed_docstring = markup.parse(api_doc.docstring, docformat, 
190                                      parse_errors) 
191           
192      # Divide the docstring into a description and a list of 
193      # fields. 
194      descr, fields = parsed_docstring.split_fields(parse_errors) 
195      api_doc.descr = descr 
196   
197      # Process fields 
198      field_warnings = [] 
199      for field in fields: 
200          try: 
201              process_field(api_doc, docindex, field.tag(), 
202                                 field.arg(), field.body()) 
203          except ValueError, e: field_warnings.append(str(e)) 
204   
205      # Extract a summary 
206      if api_doc.summary is None and api_doc.descr is not None: 
207          api_doc.summary = api_doc.descr.summary() 
208   
209      # [XX] Make sure we don't have types/param descrs for unknown 
210      # vars/params? 
211   
212      # Report any errors that occured 
213      report_errors(api_doc, docindex, parse_errors, field_warnings) 
214   
215 -def initialize_api_doc(api_doc): 
216      """A helper function for L{parse_docstring()} that initializes 
217      the attributes that C{parse_docstring()} will write to.""" 
218      if api_doc.descr is UNKNOWN: 
219          api_doc.descr = None 
220      if api_doc.summary is UNKNOWN: 
221          api_doc.summary = None 
222      if api_doc.metadata is UNKNOWN: 
223          api_doc.metadata = [] 
224      if isinstance(api_doc, RoutineDoc): 
225          if api_doc.arg_descrs is UNKNOWN: 
226              api_doc.arg_descrs = [] 
227          if api_doc.arg_types is UNKNOWN: 
228              api_doc.arg_types = {} 
229          if api_doc.return_descr is UNKNOWN: 
230              api_doc.return_descr = None 
231          if api_doc.return_type is UNKNOWN: 
232              api_doc.return_type = None 
233          if api_doc.exception_descrs is UNKNOWN: 
234              api_doc.exception_descrs = [] 
235      if isinstance(api_doc, (VariableDoc, PropertyDoc)): 
236          if api_doc.type_descr is UNKNOWN: 
237              api_doc.type_descr = None 
238      if isinstance(api_doc, NamespaceDoc): 
239          if api_doc.group_specs is UNKNOWN: 
240              api_doc.group_specs = [] 
241          if api_doc.sort_spec is UNKNOWN: 
242              api_doc.sort_spec = [] 
243   
244 -def report_errors(api_doc, docindex, parse_errors, field_warnings): 
245      """A helper function for L{parse_docstring()} that reports any 
246      markup warnings and field warnings that we encountered while 
247      processing C{api_doc}'s docstring.""" 
248      if not parse_errors and not field_warnings: return 
249   
250      # Get the name of the item containing the error, and the 
251      # filename of its containing module. 
252      name = api_doc.canonical_name 
253      module = api_doc.defining_module 
254      if module != UNKNOWN and module.filename not in (None, UNKNOWN): 
255          try: filename = py_src_filename(module.filename) 
256          except: filename = module.filename 
257      else: 
258          filename = '??' 
259   
260      # [xx] Don't report markup errors for standard builtins. 
261      if (isinstance(api_doc, ValueDoc) and api_doc != module and 
262          (api_doc.pyval in __builtin__.__dict__.values() or 
263           (module is not None and  
264            module.pyval in (__builtin__, exceptions)))): 
265          return 
266   
267      # Get the start line of the docstring containing the error. 
268      startline = api_doc.docstring_lineno 
269      if startline in (None, UNKNOWN): 
270          startline = introspect_docstring_lineno(api_doc) 
271          if startline in (None, UNKNOWN): 
272              startline = None 
273   
274      # Display a block header. 
275      header = 'File %s, ' % filename 
276      if startline is not None: 
277          header += 'line %d, ' % startline 
278      header += 'in %s' % name 
279      log.start_block(header) 
280       
281   
282      # Display all parse errors.  But first, combine any errors 
283      # with duplicate description messages. 
284      if startline is None: 
285          # remove dups, but keep original order: 
286          dups = {} 
287          for error in parse_errors: 
288              message = error.descr() 
289              if message not in dups: 
290                  log.docstring_warning(message) 
291                  dups[message] = 1 
292      else: 
293          # Combine line number fields for dup messages: 
294          messages = {} # maps message -> list of linenum 
295          for error in parse_errors: 
296              error.set_linenum_offset(startline) 
297              message = error.descr() 
298              messages.setdefault(message, []).append(error.linenum()) 
299          message_items = messages.items() 
300          message_items.sort(lambda a,b:cmp(min(a[1]), min(b[1]))) 
301          for message, linenums in message_items: 
302              linenums = [n for n in linenums if n is not None] 
303              if len(linenums) == 0: 
304                  log.docstring_warning(message) 
305              elif len(linenums) == 1: 
306                  log.docstring_warning("Line %s: %s" % (linenums[0], message)) 
307              else: 
308                  linenums = ', '.join(['%s' % l for l in linenums]) 
309                  log.docstring_warning("Lines %s: %s" % (linenums, message)) 
310   
311      # Display all field warnings. 
312      for warning in field_warnings: 
313          log.docstring_warning(warning) 
314   
315      # End the message block. 
316      log.end_block() 
317   
318  ###################################################################### 
319  #{ Field Processing Error Messages 
320  ###################################################################### 
321   
322  UNEXPECTED_ARG = '%r did not expect an argument' 
323  EXPECTED_ARG = '%r expected an argument' 
324  EXPECTED_SINGLE_ARG = '%r expected a single argument' 
325  BAD_CONTEXT = 'Invalid context for %r' 
326  REDEFINED = 'Redefinition of %s' 
327  UNKNOWN_TAG = 'Unknown field tag %r' 
328   
329  ###################################################################### 
330  #{ Field Processing 
331  ###################################################################### 
332   
333 -def process_field(api_doc, docindex, tag, arg, descr): 
334      """ 
335      Process a single field, and use it to update C{api_doc}.  If 
336      C{tag} is the name of a special field, then call its handler 
337      function.  If C{tag} is the name of a simple field, then use 
338      C{process_simple_field} to process it.  Otherwise, check if it's a 
339      user-defined field, defined in this docstring or the docstring of 
340      a containing object; and if so, process it with 
341      C{process_simple_field}. 
342   
343      @param tag: The field's tag, such as C{'author'} 
344      @param arg: The field's optional argument 
345      @param descr: The description following the field tag and 
346          argument. 
347      @raise ValueError: If a problem was encountered while processing 
348          the field.  The C{ValueError}'s string argument is an 
349          explanation of the problem, which should be displayed as a 
350          warning message. 
351      """ 
352      # standard special fields 
353      if tag in _field_dispatch_table: 
354          handler = _field_dispatch_table[tag] 
355          handler(api_doc, docindex, tag, arg, descr) 
356          return 
357   
358      # standard simple fields & user-defined fields 
359      for field in STANDARD_FIELDS + user_docfields(api_doc, docindex): 
360          if tag in field.tags: 
361              # [xx] check if it's redefined if it's not multivalue?? 
362              if not field.takes_arg: 
363                  _check(api_doc, tag, arg, expect_arg=False) 
364              api_doc.metadata.append((field, arg, descr)) 
365              return 
366   
367      # If we didn't handle the field, then report a warning. 
368      raise ValueError(UNKNOWN_TAG % tag) 
369   
370 -def user_docfields(api_doc, docindex): 
371      """ 
372      Return a list of user defined fields that can be used for the 
373      given object.  This list is taken from the given C{api_doc}, and 
374      any of its containing C{NamepaceDoc}s. 
375       
376      @bug: If a child's docstring is parsed before its parents, then 
377          its parent won't yet have had its C{extra_docstring_fields} 
378          attribute initialized. 
379      """ 
380      docfields = [] 
381      # Get any docfields from `api_doc` itself 
382      if api_doc.extra_docstring_fields not in (None, UNKNOWN): 
383          docfields += api_doc.extra_docstring_fields 
384      # Get any docfields from `api_doc`'s ancestors 
385      for i in range(len(api_doc.canonical_name)-1, 0, -1): 
386          ancestor = docindex.get_valdoc(api_doc.canonical_name.container()) 
387          if ancestor is not None \ 
388          and ancestor.extra_docstring_fields not in (None, UNKNOWN): 
389              docfields += ancestor.extra_docstring_fields 
390      return docfields 
391   
392  _field_dispatch_table = {} 
393 -def register_field_handler(handler, *field_tags): 
394      """ 
395      Register the given field handler function for processing any 
396      of the given field tags.  Field handler functions should 
397      have the following signature: 
398   
399          >>> def field_handler(api_doc, docindex, tag, arg, descr): 
400          ...     '''update api_doc in response to the field.''' 
401   
402      Where C{api_doc} is the documentation object to update; 
403      C{docindex} is a L{DocIndex} that can be used to look up the 
404      documentation for related objects; C{tag} is the field tag that 
405      was used; C{arg} is the optional argument; and C{descr} is the 
406      description following the field tag and argument. 
407      """ 
408      for field_tag in field_tags: 
409          _field_dispatch_table[field_tag] = handler 
410   
411  ###################################################################### 
412  #{ Field Handler Functions 
413  ###################################################################### 
414   
415 -def process_summary_field(api_doc, docindex, tag, arg, descr): 
416      """Store C{descr} in C{api_doc.summary}""" 
417      _check(api_doc, tag, arg, expect_arg=False) 
418      if api_doc.summary is not None: 
419          raise ValueError(REDEFINED % tag) 
420      api_doc.summary = descr 
421   
422 -def process_include_field(api_doc, docindex, tag, arg, descr): 
423      """Copy the docstring contents from the object named in C{descr}""" 
424      _check(api_doc, tag, arg, expect_arg=False) 
425      # options: 
426      #   a. just append the descr to our own 
427      #   b. append descr and update metadata 
428      #   c. append descr and process all fields. 
429      # in any case, mark any errors we may find as coming from an 
430      # imported docstring. 
431       
432      # how does this interact with documentation inheritance?? 
433      raise ValueError('%s not implemented yet' % tag) 
434   
435 -def process_undocumented_field(api_doc, docindex, tag, arg, descr): 
436      """Remove any documentation for the variables named in C{descr}""" 
437      _check(api_doc, tag, arg, context=NamespaceDoc, expect_arg=False) 
438      for ident in _descr_to_identifiers(descr): 
439          var_name_re = re.compile('^%s$' % ident.replace('*', '(.*)')) 
440          for var_name, var_doc in api_doc.variables.items(): 
441              if var_name_re.match(var_name): 
442                  # Remove the variable from `variables`. 
443                  api_doc.variables.pop(var_name, None) 
444   
445 -def process_group_field(api_doc, docindex, tag, arg, descr): 
446      """Define a group named C{arg} containing the variables whose 
447      names are listed in C{descr}.""" 
448      _check(api_doc, tag, arg, context=NamespaceDoc, expect_arg=True) 
449      api_doc.group_specs.append( (arg, _descr_to_identifiers(descr)) ) 
450      # [xx] should this also set sort order? 
451   
452 -def process_deffield_field(api_doc, docindex, tag, arg, descr): 
453      """Define a new custom field.""" 
454      _check(api_doc, tag, arg, expect_arg=True) 
455      if api_doc.extra_docstring_fields is UNKNOWN: 
456          api_doc.extra_docstring_fields = [] 
457      try: 
458          docstring_field = _descr_to_docstring_field(arg, descr) 
459          api_doc.extra_docstring_fields.append(docstring_field) 
460      except ValueError, e: 
461          raise ValueError('Bad %s: %s' % (tag, e)) 
462   
463 -def process_raise_field(api_doc, docindex, tag, arg, descr): 
464      """Record the fact that C{api_doc} can raise the exception named 
465      C{tag} in C{api_doc.exception_descrs}.""" 
466      _check(api_doc, tag, arg, context=RoutineDoc, expect_arg='single') 
467      try: name = DottedName(arg) 
468      except DottedName.InvalidDottedName: name = arg 
469      api_doc.exception_descrs.append( (name, descr) ) 
470   
471 -def process_sort_field(api_doc, docindex, tag, arg, descr): 
472      _check(api_doc, tag, arg, context=NamespaceDoc, expect_arg=False) 
473      api_doc.sort_spec = _descr_to_identifiers(descr) + api_doc.sort_spec 
474   
475  # [xx] should I notice when they give a type for an unknown var? 
476 -def process_type_field(api_doc, docindex, tag, arg, descr): 
477      # In namespace, "@type var: ..." describes the type of a var. 
478      if isinstance(api_doc, NamespaceDoc): 
479          _check(api_doc, tag, arg, expect_arg='single') 
480          set_var_type(api_doc, arg, descr) 
481   
482      # For variables & properties, "@type: ..." describes the variable. 
483      elif isinstance(api_doc, (VariableDoc, PropertyDoc)): 
484          _check(api_doc, tag, arg, expect_arg=False) 
485          if api_doc.type_descr is not None: 
486              raise ValueError(REDEFINED % tag) 
487          api_doc.type_descr = descr 
488   
489      # For routines, "@type param: ..." describes a parameter. 
490      elif isinstance(api_doc, RoutineDoc): 
491          _check(api_doc, tag, arg, expect_arg='single') 
492          if arg in api_doc.arg_types: 
493              raise ValueError(REDEFINED % ('type for '+arg)) 
494          api_doc.arg_types[arg] = descr 
495      else: 
496          raise ValueError(BAD_CONTEXT % arg) 
497   
498 -def process_var_field(api_doc, docindex, tag, arg, descr): 
499      _check(api_doc, tag, arg, context=ModuleDoc, expect_arg=True) 
500      for ident in re.split('[:;, ] *', arg): 
501          set_var_descr(api_doc, ident, descr) 
502           
503 -def process_cvar_field(api_doc, docindex, tag, arg, descr): 
504      # If @cvar is used *within* a variable, then use it as the 
505      # variable's description, and treat the variable as a class var. 
506      if (isinstance(api_doc, VariableDoc) and 
507          isinstance(api_doc.container, ClassDoc)): 
508          _check(api_doc, tag, arg, expect_arg=False) 
509          api_doc.is_instvar = False 
510          api_doc.descr = api_doc.descr.concatenate(descr) 
511          api_doc.summary = descr.summary() 
512   
513      # Otherwise, @cvar should be used in a class. 
514      else: 
515          _check(api_doc, tag, arg, context=ClassDoc, expect_arg=True) 
516          for ident in re.split('[:;, ] *', arg): 
517              set_var_descr(api_doc, ident, descr) 
518              api_doc.variables[ident].is_instvar = False 
519           
520 -def process_ivar_field(api_doc, docindex, tag, arg, descr): 
521      # If @ivar is used *within* a variable, then use it as the 
522      # variable's description, and treat the variable as an instvar. 
523      if (isinstance(api_doc, VariableDoc) and 
524          isinstance(api_doc.container, ClassDoc)): 
525          _check(api_doc, tag, arg, expect_arg=False) 
526          # require that there be no other descr? 
527          api_doc.is_instvar = True 
528          api_doc.descr = api_doc.descr.concatenate(descr) 
529          api_doc.summary = descr.summary() 
530   
531      # Otherwise, @ivar should be used in a class. 
532      else: 
533          _check(api_doc, tag, arg, context=ClassDoc, expect_arg=True) 
534          for ident in re.split('[:;, ] *', arg): 
535              set_var_descr(api_doc, ident, descr) 
536              api_doc.variables[ident].is_instvar = True 
537   
538  # [xx] '@return: foo' used to get used as a descr if no other 
539  # descr was present.  is that still true? 
540 -def process_return_field(api_doc, docindex, tag, arg, descr): 
541      _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=False) 
542      if api_doc.return_descr is not None: 
543          raise ValueError(REDEFINED % 'return value description') 
544      api_doc.return_descr = descr 
545   
546 -def process_rtype_field(api_doc, docindex, tag, arg, descr): 
547      _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=False) 
548      if api_doc.return_type is not None: 
549          raise ValueError(REDEFINED % 'return value type') 
550      api_doc.return_type = descr 
551   
552 -def process_arg_field(api_doc, docindex, tag, arg, descr): 
553      _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=True) 
554      idents = re.split('[:;, ] *', arg) 
555      api_doc.arg_descrs.append( (idents, descr) ) 
556   
557 -def process_kwarg_field(api_doc, docindex, tag, arg, descr): 
558      # [xx] these should -not- be checked if they exist.. 
559      # and listed separately or not?? 
560      _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=True) 
561      idents = re.split('[:;, ] *', arg) 
562      api_doc.arg_descrs.append( (idents, descr) ) 
563   
564  register_field_handler(process_group_field, 'group') 
565  register_field_handler(process_deffield_field, 'deffield', 'newfield') 
566  register_field_handler(process_sort_field, 'sort') 
567  register_field_handler(process_summary_field, 'summary') 
568  register_field_handler(process_undocumented_field, 'undocumented') 
569  register_field_handler(process_include_field, 'include') 
570  register_field_handler(process_var_field, 'var', 'variable') 
571  register_field_handler(process_type_field, 'type') 
572  register_field_handler(process_cvar_field, 'cvar', 'cvariable') 
573  register_field_handler(process_ivar_field, 'ivar', 'ivariable') 
574  register_field_handler(process_return_field, 'return', 'returns') 
575  register_field_handler(process_rtype_field, 'rtype', 'returntype') 
576  register_field_handler(process_arg_field, 'arg', 'argument', 
577                                            'parameter', 'param') 
578  register_field_handler(process_kwarg_field, 'kwarg', 'keyword', 'kwparam') 
579  register_field_handler(process_raise_field, 'raise', 'raises', 
580                                              'except', 'exception') 
581   
582  ###################################################################### 
583  #{ Helper Functions 
584  ###################################################################### 
585   
586 -def set_var_descr(api_doc, ident, descr): 
587      if ident not in api_doc.variables: 
588          api_doc.variables[ident] = VariableDoc(container=api_doc, 
589                                                 name=ident) 
590                                         
591      var_doc = api_doc.variables[ident] 
592      if var_doc.descr not in (None, UNKNOWN): 
593          raise ValueError(REDEFINED % ('description for '+ident)) 
594      var_doc.descr = descr 
595      if var_doc.summary in (None, UNKNOWN): 
596          var_doc.summary = var_doc.descr.summary() 
597   
598 -def set_var_type(api_doc, ident, descr): 
599      if ident not in api_doc.variables: 
600          api_doc.variables[ident] = VariableDoc(container=api_doc, 
601                                                 name=ident) 
602      var_doc = api_doc.variables[ident] 
603      if var_doc.type_descr not in (None, UNKNOWN): 
604          raise ValueError(REDEFINED % ('type for '+ident)) 
605      var_doc.type_descr = descr 
606           
607 -def _check(api_doc, tag, arg, context=None, expect_arg=None): 
608      if context is not None: 
609          if not isinstance(api_doc, context): 
610              raise ValueError(BAD_CONTEXT % tag) 
611      if expect_arg is not None: 
612          if expect_arg == True: 
613              if arg is None: 
614                  raise ValueError(EXPECTED_ARG % tag) 
615          elif expect_arg == False: 
616              if arg is not None: 
617                  raise ValueError(UNEXPECTED_ARG % tag) 
618          elif expect_arg == 'single': 
619              if (arg is None or ' ' in arg): 
620                  raise ValueError(EXPECTED_SINGLE_ARG % tag) 
621          else: 
622              assert 0, 'bad value for expect_arg' 
623   
624 -def get_docformat(api_doc, docindex): 
625      """ 
626      Return the name of the markup language that should be used to 
627      parse the API documentation for the given object. 
628      """ 
629      # Find the module that defines api_doc. 
630      module = api_doc.defining_module 
631      # Look up its docformat. 
632      if module != UNKNOWN and module.docformat not in (None, UNKNOWN): 
633          docformat = module.docformat 
634      else: 
635          docformat = DEFAULT_DOCFORMAT 
636      # Convert to lower case & strip region codes. 
637      try: return docformat.lower().split()[0] 
638      except: return DEFAULT_DOCFORMAT 
639   
640 -def unindent_docstring(docstring): 
641      # [xx] copied from inspect.getdoc(); we can't use inspect.getdoc() 
642      # itself, since it expects an object, not a string. 
643       
644      if docstring == '': return '' 
645      lines = docstring.expandtabs().split('\n') 
646   
647      # Find minimum indentation of any non-blank lines after first line. 
648      margin = sys.maxint 
649      for line in lines[1:]: 
650          content = len(line.lstrip()) 
651          if content: 
652              indent = len(line) - content 
653              margin = min(margin, indent) 
654      # Remove indentation. 
655      if lines: 
656          lines[0] = lines[0].lstrip() 
657      if margin < sys.maxint: 
658          for i in range(1, len(lines)): lines[i] = lines[i][margin:] 
659      # Remove any trailing (but not leading!) blank lines. 
660      while lines and not lines[-1]: 
661          lines.pop() 
662      #while lines and not lines[0]: 
663      #    lines.pop(0) 
664      return '\n'.join(lines) 
665                              
666  _IDENTIFIER_LIST_REGEXP = re.compile(r'^[\w.\*]+([\s,:;]\s*[\w.\*]+)*$') 
667 -def _descr_to_identifiers(descr): 
668      """ 
669      Given a C{ParsedDocstring} that contains a list of identifiers, 
670      return a list of those identifiers.  This is used by fields such 
671      as C{@group} and C{@sort}, which expect lists of identifiers as 
672      their values.  To extract the identifiers, the docstring is first 
673      converted to plaintext, and then split.  The plaintext content of 
674      the docstring must be a a list of identifiers, separated by 
675      spaces, commas, colons, or semicolons. 
676       
677      @rtype: C{list} of C{string} 
678      @return: A list of the identifier names contained in C{descr}. 
679      @type descr: L{markup.ParsedDocstring} 
680      @param descr: A C{ParsedDocstring} containing a list of 
681          identifiers. 
682      @raise ValueError: If C{descr} does not contain a valid list of 
683          identifiers. 
684      """ 
685      idents = descr.to_plaintext(None).strip() 
686      idents = re.sub(r'\s+', ' ', idents) 
687      if not _IDENTIFIER_LIST_REGEXP.match(idents): 
688          raise ValueError, 'Bad Identifier list: %r' % idents 
689      rval = re.split('[:;, ] *', idents) 
690      return rval 
691       
692 -def _descr_to_docstring_field(arg, descr): 
693      tags = [s.lower() for s in re.split('[:;, ] *', arg)] 
694      descr = descr.to_plaintext(None).strip() 
695      args = re.split('[:;,] *', descr) 
696      if len(args) == 0 or len(args) > 3: 
697          raise ValueError, 'Wrong number of arguments' 
698      singular = args[0] 
699      if len(args) >= 2: plural = args[1] 
700      else: plural = None 
701      short = 0 
702      if len(args) >= 3: 
703          if args[2] == 'short': short = 1 
704          else: raise ValueError('Bad arg 2 (expected "short")') 
705      return DocstringField(tags, singular, plural, short) 
706   
707  ###################################################################### 
708  #{ Function Signature Extraction 
709  ###################################################################### 
710   
711  # [XX] todo: add optional type modifiers? 
712  _SIGNATURE_RE = re.compile( 
713      # Class name (for builtin methods) 
714      r'^\s*((?P<self>\w+)\.)?' + 
715      # The function name (must match exactly) [XX] not anymore! 
716      r'(?P<func>\w+)' + 
717      # The parameters 
718      r'\((?P<params>(\s*\[?\s*\*{0,2}[\w\-\.]+(=.+?)?'+ 
719      r'(\s*\[?\s*,\s*\]?\s*\*{0,2}[\w\-\.]+(=.+?)?)*\]*)?)\s*\)' + 
720      # The return value (optional) 
721      r'(\s*(->)\s*(?P<return>\S.*?))?'+ 
722      # The end marker 
723      r'\s*(\n|\s+(--|<=+>)\s+|$|\.\s+|\.\n)') 
724  """A regular expression that is used to extract signatures from 
725  docstrings.""" 
726       
727 -def parse_function_signature(func_doc): 
728      """ 
729      Construct the signature for a builtin function or method from 
730      its docstring.  If the docstring uses the standard convention 
731      of including a signature in the first line of the docstring 
732      (and formats that signature according to standard 
733      conventions), then it will be used to extract a signature. 
734      Otherwise, the signature will be set to a single varargs 
735      variable named C{"..."}. 
736   
737      @rtype: C{None} 
738      """ 
739      # If there's no docstring, then don't do anything. 
740      if not func_doc.docstring: return False 
741   
742      m = _SIGNATURE_RE.match(func_doc.docstring) 
743      if m is None: return False 
744   
745      # Do I want to be this strict? 
746  #     if not (m.group('func') == func_doc.canonical_name[-1] or 
747  #             '_'+m.group('func') == func_doc.canonical_name[-1]): 
748  #         log.warning("Not extracting function signature from %s's " 
749  #                     "docstring, since the name doesn't match." % 
750  #                     func_doc.canonical_name) 
751  #         return False 
752       
753      params = m.group('params') 
754      rtype = m.group('return') 
755      selfparam = m.group('self') 
756       
757      # Extract the parameters from the signature. 
758      func_doc.posargs = [] 
759      func_doc.vararg = None 
760      func_doc.kwarg = None 
761      if func_doc.posarg_defaults is UNKNOWN: 
762          func_doc.posarg_defaults = [] 
763      if params: 
764          # Figure out which parameters are optional. 
765          while '[' in params or ']' in params: 
766              m2 = re.match(r'(.*)\[([^\[\]]+)\](.*)', params) 
767              if not m2: return False 
768              (start, mid, end) = m2.groups() 
769              mid = re.sub(r'((,|^)\s*[\w\-\.]+)', r'\1=...', mid) 
770              params = start+mid+end 
771   
772          params = re.sub(r'=...=' , r'=', params) 
773          for name in params.split(','): 
774              if '=' in name: 
775                  (name, default_repr) = name.split('=',1) 
776                  default = GenericValueDoc(parse_repr=default_repr) 
777              else: 
778                  default = None 
779              name = name.strip() 
780              if name == '...': 
781                  func_doc.vararg = '...' 
782              elif name.startswith('**'): 
783                  func_doc.kwarg = name[2:] 
784              elif name.startswith('*'): 
785                  func_doc.vararg = name[1:] 
786              else: 
787                  func_doc.posargs.append(name) 
788                  if len(func_doc.posarg_defaults) < len(func_doc.posargs): 
789                      func_doc.posarg_defaults.append(default) 
790                  elif default is not None: 
791                      argnum = len(func_doc.posargs)-1 
792                      func_doc.posarg_defaults[argnum] = default 
793   
794      # Extract the return type/value from the signature 
795      if rtype: 
796          func_doc.return_descr = markup.parse(rtype, 'plaintext') 
797   
798      # Add the self parameter, if it was specified. 
799      if selfparam: 
800          func_doc.posargs.insert(0, selfparam) 
801          func_doc.posarg_defaults.insert(0, None) 
802   
803      # Remove the signature from the docstring. 
804      func_doc.docstring = func_doc.docstring[m.end():] 
805           
806      # We found a signature. 
807      return True 
808