Added Python (Thanks to Beholder) - it fails to build properly using my build system,

so there's a precompiled binary included, with a hack in Android.mk to make it work on NDK r4b

project/jni/python/src/Doc/tools/roman.py

@@ -0,0 +1,80 @@
+"""Convert to and from Roman numerals"""
+
+__author__ = "Mark Pilgrim (f8dy@diveintopython.org)"
+__version__ = "1.4"
+__date__ = "8 August 2001"
+__copyright__ = """Copyright (c) 2001 Mark Pilgrim
+
+This program is part of "Dive Into Python", a free Python tutorial for
+experienced programmers.  Visit http://diveintopython.org/ for the
+latest version.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the Python 2.1.1 license, available at
+http://www.python.org/2.1.1/license.html
+"""
+
+import re
+
+#Define exceptions
+class RomanError(Exception): pass
+class OutOfRangeError(RomanError): pass
+class NotIntegerError(RomanError): pass
+class InvalidRomanNumeralError(RomanError): pass
+
+#Define digit mapping
+romanNumeralMap = (('M',  1000),
+                   ('CM', 900),
+                   ('D',  500),
+                   ('CD', 400),
+                   ('C',  100),
+                   ('XC', 90),
+                   ('L',  50),
+                   ('XL', 40),
+                   ('X',  10),
+                   ('IX', 9),
+                   ('V',  5),
+                   ('IV', 4),
+                   ('I',  1))
+
+def toRoman(n):
+    """convert integer to Roman numeral"""
+    if not (0 < n < 5000):
+        raise OutOfRangeError, "number out of range (must be 1..4999)"
+    if int(n) <> n:
+        raise NotIntegerError, "decimals can not be converted"
+
+    result = ""
+    for numeral, integer in romanNumeralMap:
+        while n >= integer:
+            result += numeral
+            n -= integer
+    return result
+
+#Define pattern to detect valid Roman numerals
+romanNumeralPattern = re.compile("""
+    ^                   # beginning of string
+    M{0,4}              # thousands - 0 to 4 M's
+    (CM|CD|D?C{0,3})    # hundreds - 900 (CM), 400 (CD), 0-300 (0 to 3 C's),
+                        #            or 500-800 (D, followed by 0 to 3 C's)
+    (XC|XL|L?X{0,3})    # tens - 90 (XC), 40 (XL), 0-30 (0 to 3 X's),
+                        #        or 50-80 (L, followed by 0 to 3 X's)
+    (IX|IV|V?I{0,3})    # ones - 9 (IX), 4 (IV), 0-3 (0 to 3 I's),
+                        #        or 5-8 (V, followed by 0 to 3 I's)
+    $                   # end of string
+    """ ,re.VERBOSE)
+
+def fromRoman(s):
+    """convert Roman numeral to integer"""
+    if not s:
+        raise InvalidRomanNumeralError, 'Input can not be blank'
+    if not romanNumeralPattern.search(s):
+        raise InvalidRomanNumeralError, 'Invalid Roman numeral: %s' % s
+
+    result = 0
+    index = 0
+    for numeral, integer in romanNumeralMap:
+        while s[index:index+len(numeral)] == numeral:
+            result += integer
+            index += len(numeral)
+    return result

project/jni/python/src/Doc/tools/rstlint.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Check for stylistic and formal issues in .rst and .py
+# files included in the documentation.
+#
+# 01/2009, Georg Brandl
+
+# TODO: - wrong versions in versionadded/changed
+#       - wrong markup after versionchanged directive
+
+from __future__ import with_statement
+
+import os
+import re
+import sys
+import getopt
+import subprocess
+from os.path import join, splitext, abspath, exists
+from collections import defaultdict
+
+directives = [
+    # standard docutils ones
+    'admonition', 'attention', 'caution', 'class', 'compound', 'container',
+    'contents', 'csv-table', 'danger', 'date', 'default-role', 'epigraph',
+    'error', 'figure', 'footer', 'header', 'highlights', 'hint', 'image',
+    'important', 'include', 'line-block', 'list-table', 'meta', 'note',
+    'parsed-literal', 'pull-quote', 'raw', 'replace',
+    'restructuredtext-test-directive', 'role', 'rubric', 'sectnum', 'sidebar',
+    'table', 'target-notes', 'tip', 'title', 'topic', 'unicode', 'warning',
+    # Sphinx custom ones
+    'acks', 'attribute', 'autoattribute', 'autoclass', 'autodata',
+    'autoexception', 'autofunction', 'automethod', 'automodule', 'centered',
+    'cfunction', 'class', 'classmethod', 'cmacro', 'cmdoption', 'cmember',
+    'code-block', 'confval', 'cssclass', 'ctype', 'currentmodule', 'cvar',
+    'data', 'deprecated', 'describe', 'directive', 'doctest', 'envvar', 'event',
+    'exception', 'function', 'glossary', 'highlight', 'highlightlang', 'index',
+    'literalinclude', 'method', 'module', 'moduleauthor', 'productionlist',
+    'program', 'role', 'sectionauthor', 'seealso', 'sourcecode', 'staticmethod',
+    'tabularcolumns', 'testcode', 'testoutput', 'testsetup', 'toctree', 'todo',
+    'todolist', 'versionadded', 'versionchanged'
+]
+
+all_directives = '(' + '|'.join(directives) + ')'
+seems_directive_re = re.compile(r'\.\. %s([^a-z:]|:(?!:))' % all_directives)
+default_role_re = re.compile(r'(^| )`\w([^`]*?\w)?`($| )')
+leaked_markup_re = re.compile(r'[a-z]::[^=]|:[a-z]+:|`|\.\.\s*\w+:')
+
+
+checkers = {}
+
+checker_props = {'severity': 1, 'falsepositives': False}
+
+def checker(*suffixes, **kwds):
+    """Decorator to register a function as a checker."""
+    def deco(func):
+        for suffix in suffixes:
+            checkers.setdefault(suffix, []).append(func)
+        for prop in checker_props:
+            setattr(func, prop, kwds.get(prop, checker_props[prop]))
+        return func
+    return deco
+
+
+@checker('.py', severity=4)
+def check_syntax(fn, lines):
+    """Check Python examples for valid syntax."""
+    code = ''.join(lines)
+    if '\r' in code:
+        if os.name != 'nt':
+            yield 0, '\\r in code file'
+        code = code.replace('\r', '')
+    try:
+        compile(code, fn, 'exec')
+    except SyntaxError, err:
+        yield err.lineno, 'not compilable: %s' % err
+
+
+@checker('.rst', severity=2)
+def check_suspicious_constructs(fn, lines):
+    """Check for suspicious reST constructs."""
+    inprod = False
+    for lno, line in enumerate(lines):
+        if seems_directive_re.match(line):
+            yield lno+1, 'comment seems to be intended as a directive'
+        if '.. productionlist::' in line:
+            inprod = True
+        elif not inprod and default_role_re.search(line):
+            yield lno+1, 'default role used'
+        elif inprod and not line.strip():
+            inprod = False
+
+
+@checker('.py', '.rst')
+def check_whitespace(fn, lines):
+    """Check for whitespace and line length issues."""
+    for lno, line in enumerate(lines):
+        if '\r' in line:
+            yield lno+1, '\\r in line'
+        if '\t' in line:
+            yield lno+1, 'OMG TABS!!!1'
+        if line[:-1].rstrip(' \t') != line[:-1]:
+            yield lno+1, 'trailing whitespace'
+
+
+@checker('.rst', severity=0)
+def check_line_length(fn, lines):
+    """Check for line length; this checker is not run by default."""
+    for lno, line in enumerate(lines):
+        if len(line) > 81:
+            # don't complain about tables, links and function signatures
+            if line.lstrip()[0] not in '+|' and \
+               'http://' not in line and \
+               not line.lstrip().startswith(('.. function',
+                                             '.. method',
+                                             '.. cfunction')):
+                yield lno+1, "line too long"
+
+
+@checker('.html', severity=2, falsepositives=True)
+def check_leaked_markup(fn, lines):
+    """Check HTML files for leaked reST markup; this only works if
+    the HTML files have been built.
+    """
+    for lno, line in enumerate(lines):
+        if leaked_markup_re.search(line):
+            yield lno+1, 'possibly leaked markup: %r' % line
+
+
+def main(argv):
+    usage = '''\
+Usage: %s [-v] [-f] [-s sev] [-i path]* [path]
+
+Options:  -v       verbose (print all checked file names)
+          -f       enable checkers that yield many false positives
+          -s sev   only show problems with severity >= sev
+          -i path  ignore subdir or file path
+''' % argv[0]
+    try:
+        gopts, args = getopt.getopt(argv[1:], 'vfs:i:')
+    except getopt.GetoptError:
+        print usage
+        return 2
+
+    verbose = False
+    severity = 1
+    ignore = []
+    falsepos = False
+    for opt, val in gopts:
+        if opt == '-v':
+            verbose = True
+        elif opt == '-f':
+            falsepos = True
+        elif opt == '-s':
+            severity = int(val)
+        elif opt == '-i':
+            ignore.append(abspath(val))
+
+    if len(args) == 0:
+        path = '.'
+    elif len(args) == 1:
+        path = args[0]
+    else:
+        print usage
+        return 2
+
+    if not exists(path):
+        print 'Error: path %s does not exist' % path
+        return 2
+
+    count = defaultdict(int)
+    out = sys.stdout
+
+    for root, dirs, files in os.walk(path):
+        # ignore subdirs controlled by svn
+        if '.svn' in dirs:
+            dirs.remove('.svn')
+
+        # ignore subdirs in ignore list
+        if abspath(root) in ignore:
+            del dirs[:]
+            continue
+
+        for fn in files:
+            fn = join(root, fn)
+            if fn[:2] == './':
+                fn = fn[2:]
+
+            # ignore files in ignore list
+            if abspath(fn) in ignore:
+                continue
+
+            ext = splitext(fn)[1]
+            checkerlist = checkers.get(ext, None)
+            if not checkerlist:
+                continue
+
+            if verbose:
+                print 'Checking %s...' % fn
+
+            try:
+                with open(fn, 'r') as f:
+                    lines = list(f)
+            except (IOError, OSError), err:
+                print '%s: cannot open: %s' % (fn, err)
+                count[4] += 1
+                continue
+
+            for checker in checkerlist:
+                if checker.falsepositives and not falsepos:
+                    continue
+                csev = checker.severity
+                if csev >= severity:
+                    for lno, msg in checker(fn, lines):
+                        print >>out, '[%d] %s:%d: %s' % (csev, fn, lno, msg)
+                        count[csev] += 1
+    if verbose:
+        print
+    if not count:
+        if severity > 1:
+            print 'No problems with severity >= %d found.' % severity
+        else:
+            print 'No problems found.'
+    else:
+        for severity in sorted(count):
+            number = count[severity]
+            print '%d problem%s with severity %d found.' % \
+                  (number, number > 1 and 's' or '', severity)
+    return int(bool(count))
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

project/jni/python/src/Doc/tools/sphinx-build.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+    Sphinx - Python documentation toolchain
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: 2007 by Georg Brandl.
+    :license: Python license.
+"""
+
+import sys
+
+if __name__ == '__main__':
+
+    if sys.version_info[:3] < (2, 4, 0):
+        print >>sys.stderr, """\
+Error: Sphinx needs to be executed with Python 2.4 or newer
+(If you run this from the Makefile, you can set the PYTHON variable
+to the path of an alternative interpreter executable, e.g.,
+``make html PYTHON=python2.5``).
+"""
+        sys.exit(1)
+
+    from sphinx import main
+    sys.exit(main(sys.argv))

project/jni/python/src/Doc/tools/sphinxext/download.html

@@ -0,0 +1,60 @@
+{% extends "layout.html" %}
+{% set title = 'Download' %}
+{% set dlbase = 'http://docs.python.org/ftp/python/doc/current' %}
+{% block body %}
+
+<h1>Download Python {{ release }} Documentation</h1>
+
+{% if 'a' in release or 'b' in release or 'c' in release %}
+<p>We don't package the documentation for development releases for download.
+  Downloads will be available for the final release.</p>
+
+{% else %}
+{% if last_updated %}<p><b>Last updated on: {{ last_updated }}.</b></p>{% endif %}
+
+<p>To download an archive containing all the documents for this version of
+Python in one of various formats, follow one of links in this table. The numbers
+in the table are the size of the download files in megabytes.</p>
+
+<table class="docutils">
+  <tr><th>Format</th><th>Packed as .zip</th><th>Packed as .tar.bz2</th></tr>
+  <tr><td>PDF (US-Letter paper size)</td>
+    <td><a href="{{ dlbase }}/python-{{ release }}-docs-pdf-letter.zip">Download</a> (ca. 8 MB)</td>
+    <td><a href="{{ dlbase }}/python-{{ release }}-docs-pdf-letter.tar.bz2">Download</a> (ca. 8 MB)</td>
+  </tr>
+  <tr><td>PDF (A4 paper size)</td>
+    <td><a href="{{ dlbase }}/python-{{ release }}-docs-pdf-a4.zip">Download</a> (ca. 8 MB)</td>
+    <td><a href="{{ dlbase }}/python-{{ release }}-docs-pdf-a4.tar.bz2">Download</a> (ca. 8 MB)</td>
+  </tr>
+  <tr><td>HTML</td>
+    <td><a href="{{ dlbase }}/python-{{ release }}-docs-html.zip">Download</a> (ca. 6 MB)</td>
+    <td><a href="{{ dlbase }}/python-{{ release }}-docs-html.tar.bz2">Download</a> (ca. 4 MB)</td>
+  </tr>
+  <tr><td>Plain Text</td>
+    <td><a href="{{ dlbase }}/python-{{ release }}-docs-text.zip">Download</a> (ca. 2 MB)</td>
+    <td><a href="{{ dlbase }}/python-{{ release }}-docs-text.tar.bz2">Download</a> (ca. 1.5 MB)</td>
+  </tr>
+</table>
+
+
+<p>These archives contain all the content in the documentation.</p>
+
+<h2>Unpacking</h2>
+
+<p>Unix users should download the .tar.bz2 archives; these are bzipped tar
+archives and can be handled in the usual way using tar and the bzip2
+program. The <a href="http://www.info-zip.org">InfoZIP</a> unzip program can be
+used to handle the ZIP archives if desired. The .tar.bz2 archives provide the
+best compression and fastest download times.</p>
+
+<p>Windows users can use the ZIP archives since those are customary on that
+platform. These are created on Unix using the InfoZIP zip program.</p>
+
+
+<h2>Problems</h2>
+
+<p>If you have comments or suggestions for the Python documentation, please send
+email to <a href="mailto:docs@python.org">docs@python.org</a>.</p>
+{% endif %}
+
+{% endblock %}

project/jni/python/src/Doc/tools/sphinxext/indexcontent.html

@@ -0,0 +1,59 @@
+{% extends "defindex.html" %}
+{% block tables %}
+  <p><strong>Parts of the documentation:</strong></p>
+  <table class="contentstable" align="center"><tr>
+    <td width="50%">
+      <p class="biglink"><a class="biglink" href="{{ pathto("whatsnew/" + version) }}">What's new in Python {{ version }}?</a><br/>
+         <span class="linkdescr">or <a href="{{ pathto("whatsnew/index") }}">all "What's new" documents</a> since 2.0</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("tutorial/index") }}">Tutorial</a><br/>
+         <span class="linkdescr">start here</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("using/index") }}">Using Python</a><br/>
+         <span class="linkdescr">how to use Python on different platforms</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("reference/index") }}">Language Reference</a><br/>
+         <span class="linkdescr">describes syntax and language elements</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("library/index") }}">Library Reference</a><br/>
+         <span class="linkdescr">keep this under your pillow</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("howto/index") }}">Python HOWTOs</a><br/>
+         <span class="linkdescr">in-depth documents on specific topics</span></p>
+    </td><td width="50%">
+      <p class="biglink"><a class="biglink" href="{{ pathto("extending/index") }}">Extending and Embedding</a><br/>
+         <span class="linkdescr">tutorial for C/C++ programmers</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("c-api/index") }}">Python/C API</a><br/>
+         <span class="linkdescr">reference for C/C++ programmers</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("install/index") }}">Installing Python Modules</a><br/>
+         <span class="linkdescr">information for installers &amp; sys-admins</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("distutils/index") }}">Distributing Python Modules</a><br/>
+         <span class="linkdescr">sharing modules with others</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("documenting/index") }}">Documenting Python</a><br/>
+         <span class="linkdescr">guide for documentation authors</span></p>
+    </td></tr>
+  </table>
+
+  <p><strong>Indices and tables:</strong></p>
+  <table class="contentstable" align="center"><tr>
+    <td width="50%">
+      <p class="biglink"><a class="biglink" href="{{ pathto("modindex") }}">Global Module Index</a><br/>
+         <span class="linkdescr">quick access to all modules</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("genindex") }}">General Index</a><br/>
+         <span class="linkdescr">all functions, classes, terms</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("glossary") }}">Glossary</a><br/>
+         <span class="linkdescr">the most important terms explained</span></p>
+    </td><td width="50%">
+      <p class="biglink"><a class="biglink" href="{{ pathto("search") }}">Search page</a><br/>
+         <span class="linkdescr">search this documentation</span></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("contents") }}">Complete Table of Contents</a><br/>
+         <span class="linkdescr">lists all sections and subsections</span></p>
+    </td></tr>
+  </table>
+
+  <p><strong>Meta information:</strong></p>
+  <table class="contentstable" align="center"><tr>
+    <td width="50%">
+      <p class="biglink"><a class="biglink" href="{{ pathto("bugs") }}">Reporting bugs</a></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("about") }}">About the documentation</a></p>
+    </td><td width="50%">
+      <p class="biglink"><a class="biglink" href="{{ pathto("license") }}">History and License of Python</a></p>
+      <p class="biglink"><a class="biglink" href="{{ pathto("copyright") }}">Copyright</a></p>
+    </td></tr>
+  </table>
+{% endblock %}

project/jni/python/src/Doc/tools/sphinxext/indexsidebar.html

@@ -0,0 +1,24 @@
+            <h3>Download</h3>
+            <p><a href="{{ pathto('download') }}">Download these documents</a></p>
+	    <h3>Docs for other versions</h3>
+	    <ul>
+	      <li><a href="http://docs.python.org/3.0/">Python 3.0 (stable)</a></li>
+	      <li><a href="http://docs.python.org/2.7/">Python 2.7 (in development)</a></li>
+	      <li><a href="http://docs.python.org/3.1/">Python 3.1 (in development)</a></li>
+              <li><a href="http://www.python.org/doc/versions/">Old versions</a></li>
+            </ul>
+
+            <h3>Other resources</h3>
+            <ul>
+              {# XXX: many of these should probably be merged in the main docs #}
+              <li><a href="http://www.python.org/doc/faq/">FAQs</a></li>
+              <li><a href="http://www.python.org/doc/intros/">Introductions</a></li>
+              <li><a href="http://www.python.org/doc/essays/">Guido's Essays</a></li>
+              <li><a href="http://www.python.org/doc/newstyle/">New-style Classes</a></li>
+              <li><a href="http://www.python.org/dev/peps/">PEP Index</a></li>
+              <li><a href="http://wiki.python.org/moin/BeginnersGuide">Beginner's Guide</a></li>
+              <li><a href="http://www.python.org/topics/">Topic Guides</a></li>
+              <li><a href="http://wiki.python.org/moin/PythonBooks">Book List</a></li>
+              <li><a href="http://www.python.org/doc/av/">Audio/Visual Talks</a></li>
+              <li><a href="http://www.python.org/doc/other/">Other Doc Collections</a></li>
+            </ul>

project/jni/python/src/Doc/tools/sphinxext/layout.html

@@ -0,0 +1,10 @@
+{% extends "!layout.html" %}
+{% block rootrellink %}
+        <li><img src="{{ pathto('_static/py.png', 1) }}" alt=""
+                 style="vertical-align: middle; margin-top: -1px"/></li>
+        <li><a href="{{ pathto('index') }}">{{ shorttitle }}</a>{{ reldelim1 }}</li>
+{% endblock %}
+{% block extrahead %}
+    <link rel="shortcut icon" type="image/png" href="{{ pathto('_static/py.png', 1) }}" />
+{{ super() }}
+{% endblock %}

project/jni/python/src/Doc/tools/sphinxext/opensearch.xml

@@ -0,0 +1,4 @@
+{% extends "!opensearch.xml" %}
+{% block extra -%}
+<Image height="16" width="16" type="image/x-icon">http://www.python.org/images/favicon16x16.ico</Image>
+{%- endblock %}

project/jni/python/src/Doc/tools/sphinxext/patchlevel.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+"""
+    patchlevel.py
+    ~~~~~~~~~~~~~
+
+    Extract version info from Include/patchlevel.h.
+    Adapted from Doc/tools/getversioninfo.
+
+    :copyright: 2007-2008 by Georg Brandl.
+    :license: Python license.
+"""
+
+import os
+import re
+import sys
+
+def get_header_version_info(srcdir):
+    patchlevel_h = os.path.join(srcdir, '..', 'Include', 'patchlevel.h')
+
+    # This won't pick out all #defines, but it will pick up the ones we
+    # care about.
+    rx = re.compile(r'\s*#define\s+([a-zA-Z][a-zA-Z_0-9]*)\s+([a-zA-Z_0-9]+)')
+
+    d = {}
+    f = open(patchlevel_h)
+    try:
+        for line in f:
+            m = rx.match(line)
+            if m is not None:
+                name, value = m.group(1, 2)
+                d[name] = value
+    finally:
+        f.close()
+
+    release = version = '%s.%s' % (d['PY_MAJOR_VERSION'], d['PY_MINOR_VERSION'])
+    micro = int(d['PY_MICRO_VERSION'])
+    if micro != 0:
+        release += '.' + str(micro)
+
+    level = d['PY_RELEASE_LEVEL']
+    suffixes = {
+        'PY_RELEASE_LEVEL_ALPHA': 'a',
+        'PY_RELEASE_LEVEL_BETA':  'b',
+        'PY_RELEASE_LEVEL_GAMMA': 'c',
+        }
+    if level != 'PY_RELEASE_LEVEL_FINAL':
+        release += suffixes[level] + str(int(d['PY_RELEASE_SERIAL']))
+    return version, release
+
+
+def get_sys_version_info():
+    major, minor, micro, level, serial = sys.version_info
+    release = version = '%s.%s' % (major, minor)
+    if micro:
+        release += '.%s' % micro
+    if level != 'final':
+        release += '%s%s' % (level[0], serial)
+    return version, release
+
+
+def get_version_info():
+    try:
+        return get_header_version_info('.')
+    except (IOError, OSError):
+        version, release = get_sys_version_info()
+        print >>sys.stderr, 'Can\'t get version info from Include/patchlevel.h, ' \
+              'using version of this interpreter (%s).' % release
+        return version, release
+
+if __name__ == '__main__':
+    print get_header_version_info('.')[1]

project/jni/python/src/Doc/tools/sphinxext/pyspecific.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+"""
+    pyspecific.py
+    ~~~~~~~~~~~~~
+
+    Sphinx extension with Python doc-specific markup.
+
+    :copyright: 2008, 2009 by Georg Brandl.
+    :license: Python license.
+"""
+
+ISSUE_URI = 'http://bugs.python.org/issue%s'
+
+from docutils import nodes, utils
+
+# monkey-patch reST parser to disable alphabetic and roman enumerated lists
+from docutils.parsers.rst.states import Body
+Body.enum.converters['loweralpha'] = \
+    Body.enum.converters['upperalpha'] = \
+    Body.enum.converters['lowerroman'] = \
+    Body.enum.converters['upperroman'] = lambda x: None
+
+
+def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
+    issue = utils.unescape(text)
+    text = 'issue ' + issue
+    refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue)
+    return [refnode], []
+
+
+# Support for building "topic help" for pydoc
+
+pydoc_topic_labels = [
+    'assert', 'assignment', 'atom-identifiers', 'atom-literals',
+    'attribute-access', 'attribute-references', 'augassign', 'binary',
+    'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object',
+    'bltin-file-objects', 'bltin-null-object', 'bltin-type-objects', 'booleans',
+    'break', 'callable-types', 'calls', 'class', 'coercion-rules',
+    'comparisons', 'compound', 'context-managers', 'continue', 'conversions',
+    'customization', 'debugger', 'del', 'dict', 'dynamic-features', 'else',
+    'exceptions', 'exec', 'execmodel', 'exprlists', 'floating', 'for',
+    'formatstrings', 'function', 'global', 'id-classes', 'identifiers', 'if',
+    'imaginary', 'import', 'in', 'integers', 'lambda', 'lists', 'naming',
+    'numbers', 'numeric-types', 'objects', 'operator-summary', 'pass', 'power',
+    'print', 'raise', 'return', 'sequence-methods', 'sequence-types',
+    'shifting', 'slicings', 'specialattrs', 'specialnames',
+    'string-conversions', 'string-methods', 'strings', 'subscriptions', 'truth',
+    'try', 'types', 'typesfunctions', 'typesmapping', 'typesmethods',
+    'typesmodules', 'typesseq', 'typesseq-mutable', 'unary', 'while', 'with',
+    'yield'
+]
+
+from os import path
+from time import asctime
+from pprint import pformat
+from docutils.io import StringOutput
+from docutils.utils import new_document
+
+try:
+    from sphinx.builders import Builder
+except ImportError:
+    # using Sphinx < 0.6, which has a different package layout
+    from sphinx.builder import Builder
+    # monkey-patch toctree directive to accept (and ignore) the :numbered: flag
+    from sphinx.directives.other import toctree_directive
+    toctree_directive.options['numbered'] = toctree_directive.options['glob']
+
+try:
+    from sphinx.writers.text import TextWriter
+except ImportError:
+    from sphinx.textwriter import TextWriter
+
+
+class PydocTopicsBuilder(Builder):
+    name = 'pydoc-topics'
+
+    def init(self):
+        self.topics = {}
+
+    def get_outdated_docs(self):
+        return 'all pydoc topics'
+
+    def get_target_uri(self, docname, typ=None):
+        return ''  # no URIs
+
+    def write(self, *ignored):
+        writer = TextWriter(self)
+        for label in self.status_iterator(pydoc_topic_labels, 'building topics... '):
+            if label not in self.env.labels:
+                self.warn('label %r not in documentation' % label)
+                continue
+            docname, labelid, sectname = self.env.labels[label]
+            doctree = self.env.get_and_resolve_doctree(docname, self)
+            document = new_document('<section node>')
+            document.append(doctree.ids[labelid])
+            destination = StringOutput(encoding='utf-8')
+            writer.write(document, destination)
+            self.topics[label] = writer.output
+
+    def finish(self):
+        f = open(path.join(self.outdir, 'pydoc_topics.py'), 'w')
+        try:
+            f.write('# Autogenerated by Sphinx on %s\n' % asctime())
+            f.write('topics = ' + pformat(self.topics) + '\n')
+        finally:
+            f.close()
+
+# Support for checking for suspicious markup
+
+import suspicious
+
+# Support for documenting Opcodes
+
+import re
+from sphinx import addnodes
+
+opcode_sig_re = re.compile(r'(\w+(?:\+\d)?)\s*\((.*)\)')
+
+def parse_opcode_signature(env, sig, signode):
+    """Transform an opcode signature into RST nodes."""
+    m = opcode_sig_re.match(sig)
+    if m is None:
+        raise ValueError
+    opname, arglist = m.groups()
+    signode += addnodes.desc_name(opname, opname)
+    paramlist = addnodes.desc_parameterlist()
+    signode += paramlist
+    paramlist += addnodes.desc_parameter(arglist, arglist)
+    return opname.strip()
+
+
+def setup(app):
+    app.add_role('issue', issue_role)
+    app.add_builder(PydocTopicsBuilder)
+    app.add_builder(suspicious.CheckSuspiciousMarkupBuilder)
+    app.add_description_unit('opcode', 'opcode', '%s (opcode)',
+                             parse_opcode_signature)

project/jni/python/src/Doc/tools/sphinxext/static/basic.css

@@ -0,0 +1,421 @@
+/**
+ * Sphinx stylesheet -- basic theme
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ */
+
+/* -- main layout ----------------------------------------------------------- */
+
+div.documentwrapper {
+    float: left;
+    width: 100%;
+}
+
+div.bodywrapper {
+    margin: 0 0 0 230px;
+}
+
+div.clearer {
+    clear: both;
+}
+
+/* -- relbar ---------------------------------------------------------------- */
+
+div.related {
+    width: 100%;
+    font-size: 90%;
+}
+
+div.related h3 {
+    display: none;
+}
+
+div.related ul {
+    margin: 0;
+    padding: 0 0 0 10px;
+    list-style: none;
+}
+
+div.related li {
+    display: inline;
+}
+
+div.related li.right {
+    float: right;
+    margin-right: 5px;
+}
+
+/* -- sidebar --------------------------------------------------------------- */
+
+div.sphinxsidebarwrapper {
+    padding: 10px 5px 0 10px;
+}
+
+div.sphinxsidebar {
+    float: left;
+    width: 230px;
+    margin-left: -100%;
+    font-size: 90%;
+}
+
+div.sphinxsidebar ul {
+    list-style: none;
+}
+
+div.sphinxsidebar ul ul,
+div.sphinxsidebar ul.want-points {
+    margin-left: 20px;
+    list-style: square;
+}
+
+div.sphinxsidebar ul ul {
+    margin-top: 0;
+    margin-bottom: 0;
+}
+
+div.sphinxsidebar form {
+    margin-top: 10px;
+}
+
+div.sphinxsidebar input {
+    border: 1px solid #98dbcc;
+    font-family: sans-serif;
+    font-size: 1em;
+}
+
+img {
+    border: 0;
+}
+
+/* -- search page ----------------------------------------------------------- */
+
+ul.search {
+    margin: 10px 0 0 20px;
+    padding: 0;
+}
+
+ul.search li {
+    padding: 5px 0 5px 20px;
+    background-image: url(file.png);
+    background-repeat: no-repeat;
+    background-position: 0 7px;
+}
+
+ul.search li a {
+    font-weight: bold;
+}
+
+ul.search li div.context {
+    color: #888;
+    margin: 2px 0 0 30px;
+    text-align: left;
+}
+
+ul.keywordmatches li.goodmatch a {
+    font-weight: bold;
+}
+
+/* -- index page ------------------------------------------------------------ */
+
+table.contentstable {
+    width: 90%;
+}
+
+table.contentstable p.biglink {
+    line-height: 150%;
+}
+
+a.biglink {
+    font-size: 1.3em;
+}
+
+span.linkdescr {
+    font-style: italic;
+    padding-top: 5px;
+    font-size: 90%;
+}
+
+/* -- general index --------------------------------------------------------- */
+
+table.indextable td {
+    text-align: left;
+    vertical-align: top;
+}
+
+table.indextable dl, table.indextable dd {
+    margin-top: 0;
+    margin-bottom: 0;
+}
+
+table.indextable tr.pcap {
+    height: 10px;
+}
+
+table.indextable tr.cap {
+    margin-top: 10px;
+    background-color: #f2f2f2;
+}
+
+img.toggler {
+    margin-right: 3px;
+    margin-top: 3px;
+    cursor: pointer;
+}
+
+/* -- general body styles --------------------------------------------------- */
+
+a.headerlink {
+    visibility: hidden;
+}
+
+h1:hover > a.headerlink,
+h2:hover > a.headerlink,
+h3:hover > a.headerlink,
+h4:hover > a.headerlink,
+h5:hover > a.headerlink,
+h6:hover > a.headerlink,
+dt:hover > a.headerlink {
+    visibility: visible;
+}
+
+div.body p.caption {
+    text-align: inherit;
+}
+
+div.body td {
+    text-align: left;
+}
+
+.field-list ul {
+    padding-left: 1em;
+}
+
+.first {
+    margin-top: 0 !important;
+}
+
+p.rubric {
+    margin-top: 30px;
+    font-weight: bold;
+}
+
+/* -- sidebars -------------------------------------------------------------- */
+
+div.sidebar {
+    margin: 0 0 0.5em 1em;
+    border: 1px solid #ddb;
+    padding: 7px 7px 0 7px;
+    background-color: #ffe;
+    width: 40%;
+    float: right;
+}
+
+p.sidebar-title {
+    font-weight: bold;
+}
+
+/* -- topics ---------------------------------------------------------------- */
+
+div.topic {
+    border: 1px solid #ccc;
+    padding: 7px 7px 0 7px;
+    margin: 10px 0 10px 0;
+}
+
+p.topic-title {
+    font-size: 1.1em;
+    font-weight: bold;
+    margin-top: 10px;
+}
+
+/* -- admonitions ----------------------------------------------------------- */
+
+div.admonition {
+    margin-top: 10px;
+    margin-bottom: 10px;
+    padding: 7px;
+}
+
+div.admonition dt {
+    font-weight: bold;
+}
+
+div.admonition dl {
+    margin-bottom: 0;
+}
+
+p.admonition-title {
+    margin: 0px 10px 5px 0px;
+    font-weight: bold;
+}
+
+div.body p.centered {
+    text-align: center;
+    margin-top: 25px;
+}
+
+/* -- tables ---------------------------------------------------------------- */
+
+table.docutils {
+    border: 0 solid #dce;
+    border-collapse: collapse;
+}
+
+table.docutils td, table.docutils th {
+    padding: 2px 5px 2px 5px;
+    border-left: 0;    
+    background-color: #eef;    
+}
+
+table.field-list td, table.field-list th {
+    border: 0 !important;
+}
+
+table.footnote td, table.footnote th {
+    border: 0 !important;
+}
+
+table.docutils th {
+    border-top: 1px solid #cac;    
+    background-color: #ede;
+}
+
+th {
+    text-align: left;
+    padding-right: 5px;
+}
+
+th.head {
+    text-align: center;    
+}
+
+/* -- other body styles ----------------------------------------------------- */
+
+dl {
+    margin-bottom: 15px;
+}
+
+dd p {
+    margin-top: 0px;
+}
+
+dd ul, dd table {
+    margin-bottom: 10px;
+}
+
+dd {
+    margin-top: 3px;
+    margin-bottom: 10px;
+    margin-left: 30px;
+}
+
+dt:target, .highlight {
+    background-color: #fbe54e;
+}
+
+dl.glossary dt {
+    font-weight: bold;
+    font-size: 1.1em;
+}
+
+.field-list ul {
+    margin: 0;
+    padding-left: 1em;
+}
+
+.field-list p {
+    margin: 0;
+}
+
+.refcount {
+    color: #060;
+}
+
+.optional {
+    font-size: 1.3em;
+}
+
+.versionmodified {
+    font-style: italic;
+}
+
+.system-message {
+    background-color: #fda;
+    padding: 5px;
+    border: 3px solid red;
+}
+
+.footnote:target  {
+    background-color: #ffa
+}
+
+/* -- code displays --------------------------------------------------------- */
+
+pre {
+    overflow: auto;
+}
+
+td.linenos pre {
+    padding: 5px 0px;
+    border: 0;
+    background-color: transparent;
+    color: #aaa;
+}
+
+table.highlighttable {
+    margin-left: 0.5em;
+}
+
+table.highlighttable td {
+    padding: 0 0.5em 0 0.5em;
+}
+
+tt.descname {
+    background-color: transparent;
+    font-weight: bold;
+    font-size: 1.2em;
+}
+
+tt.descclassname {
+    background-color: transparent;
+}
+
+tt.xref, a tt {
+    background-color: transparent;
+    font-weight: bold;
+}
+
+h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt {
+    background-color: transparent;
+}
+
+/* -- math display ---------------------------------------------------------- */
+
+img.math {
+    vertical-align: middle;
+}
+
+div.math p {
+    text-align: center;
+}
+
+span.eqno {
+    float: right;
+}
+
+/* -- printout stylesheet --------------------------------------------------- */
+
+@media print {
+    div.document,
+    div.documentwrapper,
+    div.bodywrapper {
+        margin: 0;
+        width: 100%;
+    }
+
+    div.sphinxsidebar,
+    div.related,
+    div.footer,
+    #top-link {
+        display: none;
+    }
+}

project/jni/python/src/Doc/tools/sphinxext/susp-ignored.csv

@@ -0,0 +1,164 @@
+c-api/arg,,:ref,"PyArg_ParseTuple(args, ""O|O:ref"", &object, &callback)"
+c-api/list,,:high,list[low:high]
+c-api/list,,:high,list[low:high] = itemlist
+c-api/sequence,,:i2,o[i1:i2]
+c-api/sequence,,:i2,o[i1:i2] = v
+c-api/sequence,,:i2,del o[i1:i2]
+c-api/unicode,,:end,str[start:end]
+distutils/apiref,,:action,http://pypi.python.org/pypi?:action=list_classifiers
+distutils/setupscript,,::,
+extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))"
+extending/extending,,:set,"if (PyArg_ParseTuple(args, ""O:set_callback"", &temp)) {"
+extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);"
+extending/newtypes,,:call,"if (!PyArg_ParseTuple(args, ""sss:call"", &arg1, &arg2, &arg3)) {"
+extending/windows,,:initspam,/export:initspam
+howto/cporting,,:add,"if (!PyArg_ParseTuple(args, ""ii:add_ints"", &one, &two))"
+howto/cporting,,:encode,"if (!PyArg_ParseTuple(args, ""O:encode_object"", &myobj))"
+howto/cporting,,:say,"if (!PyArg_ParseTuple(args, ""U:say_hello"", &name))"
+howto/curses,,:black,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and"
+howto/curses,,:blue,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and"
+howto/curses,,:cyan,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and"
+howto/curses,,:green,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and"
+howto/curses,,:magenta,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and"
+howto/curses,,:red,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and"
+howto/curses,,:white,"7:white."
+howto/curses,,:yellow,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and"
+howto/regex,,::,
+howto/regex,,:foo,(?:foo)
+howto/urllib2,,:example,"for example ""joe@password:example.com"""
+howto/webservers,,.. image:,.. image:: http.png
+library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)],"
+library/datetime,,:MM,
+library/datetime,,:SS,
+library/decimal,,:optional,"trailneg:optional trailing minus indicator"
+library/difflib,,:ahi,a[alo:ahi]
+library/difflib,,:bhi,b[blo:bhi]
+library/difflib,,:i2,
+library/difflib,,:j2,
+library/difflib,,:i1,
+library/dis,,:TOS,
+library/dis,,`,TOS = `TOS`
+library/doctest,,`,``factorial`` from the ``example`` module:
+library/doctest,,`,The ``example`` module
+library/doctest,,`,Using ``factorial``
+library/functions,,:step,a[start:stop:step]
+library/functions,,:stop,"a[start:stop, i]"
+library/functions,,:stop,a[start:stop:step]
+library/hotshot,,:lineno,"ncalls  tottime  percall  cumtime  percall filename:lineno(function)"
+library/httplib,,:port,host:port
+library/imaplib,,:MM,"""DD-Mmm-YYYY HH:MM:SS +HHMM"""
+library/imaplib,,:SS,"""DD-Mmm-YYYY HH:MM:SS +HHMM"""
+library/linecache,,:sys,"sys:x:3:3:sys:/dev:/bin/sh"
+library/logging,,:And,
+library/logging,,:package1,
+library/logging,,:package2,
+library/logging,,:root,
+library/logging,,:This,
+library/logging,,:port,host:port
+library/mmap,,:i2,obj[i1:i2]
+library/multiprocessing,,:queue,">>> QueueManager.register('get_queue', callable=lambda:queue)"
+library/multiprocessing,,`,">>> l._callmethod('__getitem__', (20,))     # equiv to `l[20]`"
+library/multiprocessing,,`,">>> l._callmethod('__getslice__', (2, 7))   # equiv to `l[2:7]`"
+library/multiprocessing,,`,# `BaseManager`.
+library/multiprocessing,,`,# `Pool.imap()` (which will save on the amount of code needed anyway).
+library/multiprocessing,,`,# A test file for the `multiprocessing` package
+library/multiprocessing,,`,# A test of `multiprocessing.Pool` class
+library/multiprocessing,,`,# Add more tasks using `put()`
+library/multiprocessing,,`,# create server for a `HostManager` object
+library/multiprocessing,,`,# Depends on `multiprocessing` package -- tested with `processing-0.60`
+library/multiprocessing,,`,# in the original order then consider using `Pool.map()` or
+library/multiprocessing,,`,# Not sure if we should synchronize access to `socket.accept()` method by
+library/multiprocessing,,`,# object.  (We import `multiprocessing.reduction` to enable this pickling.)
+library/multiprocessing,,`,# register the Foo class; make `f()` and `g()` accessible via proxy
+library/multiprocessing,,`,# register the Foo class; make `g()` and `_h()` accessible via proxy
+library/multiprocessing,,`,# register the generator function baz; use `GeneratorProxy` to make proxies
+library/multiprocessing,,`,`Cluster` is a subclass of `SyncManager` so it allows creation of
+library/multiprocessing,,`,`hostname` gives the name of the host.  If hostname is not
+library/multiprocessing,,`,`slots` is used to specify the number of slots for processes on
+library/optparse,,:len,"del parser.rargs[:len(value)]"
+library/os.path,,:foo,c:foo
+library/parser,,`,"""Make a function that raises an argument to the exponent `exp`."""
+library/posix,,`,"CFLAGS=""`getconf LFS_CFLAGS`"" OPT=""-g -O2 $CFLAGS"""
+library/profile,,:lineno,ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+library/profile,,:lineno,filename:lineno(function)
+library/pyexpat,,:elem1,<py:elem1 />
+library/pyexpat,,:py,"xmlns:py = ""http://www.python.org/ns/"">"
+library/repr,,`,"return `obj`"
+library/smtplib,,:port,"as well as a regular host:port server."
+library/socket,,::,'5aef:2b::8'
+library/sqlite3,,:memory,
+library/sqlite3,,:age,"select name_last, age from people where name_last=:who and age=:age"
+library/sqlite3,,:who,"select name_last, age from people where name_last=:who and age=:age"
+library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc."
+library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group"
+library/ssl,,:myserver,"Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com"
+library/ssl,,:MyState,State or Province Name (full name) [Some-State]:MyState
+library/ssl,,:ops,Email Address []:ops@myserver.mygroup.myorganization.com
+library/ssl,,:Some,"Locality Name (eg, city) []:Some City"
+library/ssl,,:US,Country Name (2 letter code) [AU]:US
+library/stdtypes,,:len,s[len(s):len(s)]
+library/stdtypes,,:len,s[len(s):len(s)]
+library/string,,:end,s[start:end]
+library/string,,:end,s[start:end]
+library/subprocess,,`,"output=`mycmd myarg`"
+library/subprocess,,`,"output=`dmesg | grep hda`"
+library/tarfile,,:compression,filemode[:compression]
+library/tarfile,,:gz,
+library/tarfile,,:bz2,
+library/time,,:mm,
+library/time,,:ss,
+library/turtle,,::,Example::
+library/urllib,,:port,:port
+library/urllib2,,:password,"""joe:password@python.org"""
+library/uuid,,:uuid,urn:uuid:12345678-1234-5678-1234-567812345678
+library/xmlrpclib,,:pass,http://user:pass@host:port/path
+library/xmlrpclib,,:pass,user:pass
+library/xmlrpclib,,:port,http://user:pass@host:port/path
+license,,`,THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
+license,,:zooko,mailto:zooko@zooko.com
+license,,`,THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+reference/datamodel,,:step,a[i:j:step]
+reference/datamodel,,:max,
+reference/expressions,,:index,x[index:index]
+reference/expressions,,:datum,{key:datum...}
+reference/expressions,,`,`expressions...`
+reference/grammar,,:output,#diagram:output
+reference/grammar,,:rules,#diagram:rules
+reference/grammar,,:token,#diagram:token
+reference/grammar,,`,'`' testlist1 '`'
+reference/lexical_analysis,,:fileencoding,# vim:fileencoding=<encoding-name>
+reference/lexical_analysis,,`,",       :       .       `       =       ;"
+tutorial/datastructures,,:value,key:value pairs within the braces adds initial key:value pairs
+tutorial/datastructures,,:value,It is also possible to delete a key:value
+tutorial/stdlib2,,:start,"fields = struct.unpack('<IIIHH', data[start:start+16])"
+tutorial/stdlib2,,:start,extra = data[start:start+extra_size]
+tutorial/stdlib2,,:start,filename = data[start:start+filenamesize]
+tutorial/stdlib2,,:config,"logging.warning('Warning:config file %s not found', 'server.conf')"
+tutorial/stdlib2,,:config,WARNING:root:Warning:config file server.conf not found
+tutorial/stdlib2,,:Critical,CRITICAL:root:Critical error -- shutting down
+tutorial/stdlib2,,:Error,ERROR:root:Error occurred
+tutorial/stdlib2,,:root,CRITICAL:root:Critical error -- shutting down
+tutorial/stdlib2,,:root,ERROR:root:Error occurred
+tutorial/stdlib2,,:root,WARNING:root:Warning:config file server.conf not found
+tutorial/stdlib2,,:Warning,WARNING:root:Warning:config file server.conf not found
+using/cmdline,,:line,file:line: category: message
+using/cmdline,,:category,action:message:category:module:line
+using/cmdline,,:line,action:message:category:module:line
+using/cmdline,,:message,action:message:category:module:line
+using/cmdline,,:module,action:message:category:module:line
+using/cmdline,,:errorhandler,:errorhandler
+using/windows,162,`,`` this fixes syntax highlighting errors in some editors due to the \\\\ hackery
+using/windows,170,`,``
+whatsnew/2.0,418,:len,
+whatsnew/2.3,,::,
+whatsnew/2.3,,:config,
+whatsnew/2.3,,:Critical,
+whatsnew/2.3,,:Error,
+whatsnew/2.3,,:Problem,
+whatsnew/2.3,,:root,
+whatsnew/2.3,,:Warning,
+whatsnew/2.4,,::,
+whatsnew/2.4,,:System,
+whatsnew/2.5,,:memory,:memory:
+whatsnew/2.5,,:step,[start:stop:step]
+whatsnew/2.5,,:stop,[start:stop:step]

project/jni/python/src/Doc/tools/sphinxext/suspicious.py

@@ -0,0 +1,242 @@
+"""
+Try to detect suspicious constructs, resembling markup
+that has leaked into the final output.
+
+Suspicious lines are reported in a comma-separated-file,
+``suspicious.csv``, located in the output directory.
+
+The file is utf-8 encoded, and each line contains four fields:
+
+ * document name (normalized)
+ * line number in the source document
+ * problematic text
+ * complete line showing the problematic text in context
+
+It is common to find many false positives. To avoid reporting them
+again and again, they may be added to the ``ignored.csv`` file
+(located in the configuration directory). The file has the same
+format as ``suspicious.csv`` with a few differences:
+
+  - each line defines a rule; if the rule matches, the issue
+    is ignored.
+  - line number may be empty (that is, nothing between the
+    commas: ",,"). In this case, line numbers are ignored (the
+    rule matches anywhere in the file).
+  - the last field does not have to be a complete line; some
+    surrounding text (never more than a line) is enough for
+    context.
+
+Rules are processed sequentially. A rule matches when:
+
+ * document names are the same
+ * problematic texts are the same
+ * line numbers are close to each other (5 lines up or down)
+ * the rule text is completely contained into the source line
+
+The simplest way to create the ignored.csv file is by copying
+undesired entries from suspicious.csv (possibly trimming the last
+field.)
+
+Copyright 2009 Gabriel A. Genellina
+
+"""
+
+import os, sys
+import csv
+import re
+from docutils import nodes
+
+try:
+    from sphinx.builders import Builder
+except ImportError:
+    from sphinx.builder import Builder
+
+
+detect_all = re.compile(ur'''
+    ::(?=[^=])|            # two :: (but NOT ::=)
+    :[a-zA-Z][a-zA-Z0-9]+| # :foo
+    `|                     # ` (seldom used by itself)
+    (?<!\.)\.\.[ \t]*\w+:  # .. foo: (but NOT ... else:)
+    ''', re.UNICODE | re.VERBOSE).finditer
+
+class Rule:
+    def __init__(self, docname, lineno, issue, line):
+        "A rule for ignoring issues"
+        self.docname = docname # document to which this rule applies
+        self.lineno = lineno   # line number in the original source;
+                               # this rule matches only near that.
+                               # None -> don't care
+        self.issue = issue     # the markup fragment that triggered this rule
+        self.line = line       # text of the container element (single line only)
+
+
+class CheckSuspiciousMarkupBuilder(Builder):
+    """
+    Checks for possibly invalid markup that may leak into the output
+    """
+    name = 'suspicious'
+
+    def init(self):
+        # create output file
+        self.log_file_name = os.path.join(self.outdir, 'suspicious.csv')
+        open(self.log_file_name, 'w').close()
+        # load database of previously ignored issues
+        self.load_rules(os.path.join(os.path.dirname(__file__), 'susp-ignored.csv'))
+
+    def get_outdated_docs(self):
+        return self.env.found_docs
+
+    def get_target_uri(self, docname, typ=None):
+        return ''
+
+    def prepare_writing(self, docnames):
+        ### PYTHON PROJECT SPECIFIC ###
+        for name in set(docnames):
+            if name.split('/', 1)[0] == 'documenting':
+                docnames.remove(name)
+        ### PYTHON PROJECT SPECIFIC ###
+
+    def write_doc(self, docname, doctree):
+        self.any_issue = False # set when any issue is encountered in this document
+        self.docname = docname
+        visitor = SuspiciousVisitor(doctree, self)
+        doctree.walk(visitor)
+
+    def finish(self):
+        return
+
+    def check_issue(self, line, lineno, issue):
+        if not self.is_ignored(line, lineno, issue):
+            self.report_issue(line, lineno, issue)
+
+    def is_ignored(self, line, lineno, issue):
+        """Determine whether this issue should be ignored.
+        """
+        docname = self.docname
+        for rule in self.rules:
+            if rule.docname != docname: continue
+            if rule.issue != issue: continue
+            # Both lines must match *exactly*. This is rather strict,
+            # and probably should be improved.
+            # Doing fuzzy matches with levenshtein distance could work,
+            # but that means bringing other libraries...
+            # Ok, relax that requirement: just check if the rule fragment
+            # is contained in the document line
+            if rule.line not in line: continue
+            # Check both line numbers. If they're "near"
+            # this rule matches. (lineno=None means "don't care")
+            if (rule.lineno is not None) and \
+                abs(rule.lineno - lineno) > 5: continue
+            # if it came this far, the rule matched
+            return True
+        return False
+
+    def report_issue(self, text, lineno, issue):
+        if not self.any_issue: self.info()
+        self.any_issue = True
+        self.write_log_entry(lineno, issue, text)
+        self.warn('[%s:%d] "%s" found in "%-.120s"' % (
+                self.docname.encode(sys.getdefaultencoding(),'replace'),
+                lineno,
+                issue.encode(sys.getdefaultencoding(),'replace'),
+                text.strip().encode(sys.getdefaultencoding(),'replace')))
+        self.app.statuscode = 1
+
+    def write_log_entry(self, lineno, issue, text):
+        f = open(self.log_file_name, 'ab')
+        writer = csv.writer(f)
+        writer.writerow([self.docname.encode('utf-8'),
+                lineno,
+                issue.encode('utf-8'),
+                text.strip().encode('utf-8')])
+        del writer
+        f.close()
+
+    def load_rules(self, filename):
+        """Load database of previously ignored issues.
+
+        A csv file, with exactly the same format as suspicious.csv
+        Fields: document name (normalized), line number, issue, surrounding text
+        """
+        self.info("loading ignore rules... ", nonl=1)
+        self.rules = rules = []
+        try: f = open(filename, 'rb')
+        except IOError: return
+        for i, row in enumerate(csv.reader(f)):
+            if len(row) != 4:
+                raise ValueError, "wrong format in %s, line %d: %s" % (filename, i+1, row)
+            docname, lineno, issue, text = row
+            docname = docname.decode('utf-8')
+            if lineno: lineno = int(lineno)
+            else: lineno = None
+            issue = issue.decode('utf-8')
+            text = text.decode('utf-8')
+            rule = Rule(docname, lineno, issue, text)
+            rules.append(rule)
+        f.close()
+        self.info('done, %d rules loaded' % len(self.rules))
+
+
+def get_lineno(node):
+    "Obtain line number information for a node"
+    lineno = None
+    while lineno is None and node:
+        node = node.parent
+        lineno = node.line
+    return lineno
+
+
+def extract_line(text, index):
+    """text may be a multiline string; extract
+    only the line containing the given character index.
+
+    >>> extract_line("abc\ndefgh\ni", 6)
+    >>> 'defgh'
+    >>> for i in (0, 2, 3, 4, 10):
+    ...   print extract_line("abc\ndefgh\ni", i)
+    abc
+    abc
+    abc
+    defgh
+    defgh
+    i
+    """
+    p = text.rfind('\n', 0, index) + 1
+    q = text.find('\n', index)
+    if q<0: q = len(text)
+    return text[p:q]
+
+
+class SuspiciousVisitor(nodes.GenericNodeVisitor):
+
+    lastlineno = 0
+
+    def __init__(self, document, builder):
+        nodes.GenericNodeVisitor.__init__(self, document)
+        self.builder = builder
+
+    def default_visit(self, node):
+        if isinstance(node, (nodes.Text, nodes.image)): # direct text containers
+            text = node.astext()
+            # lineno seems to go backwards sometimes (?)
+            self.lastlineno = lineno = max(get_lineno(node) or 0, self.lastlineno)
+            seen = set() # don't report the same issue more than only once per line
+            for match in detect_all(text):
+                #import pdb; pdb.set_trace()
+                issue = match.group()
+                line = extract_line(text, match.start())
+                if (issue, line) not in seen:
+                    self.builder.check_issue(line, lineno, issue)
+                    seen.add((issue, line))
+
+    unknown_visit = default_visit
+
+    def visit_document(self, node):
+        self.lastlineno = 0
+
+    def visit_comment(self, node):
+        # ignore comments -- too much false positives.
+        # (although doing this could miss some errors;
+        # there were two sections "commented-out" by mistake
+        # in the Python docs that would not be catched)
+        raise nodes.SkipNode