Annotate.py 11.8 KB
Newer Older
1 2
# Note: Work in progress

3 4
from __future__ import absolute_import

Robert Bradshaw's avatar
Robert Bradshaw committed
5
import os
6
import os.path
7
import re
8
import codecs
9
import textwrap
10
from datetime import datetime
11 12
from functools import partial
from collections import defaultdict
Stefan Behnel's avatar
Stefan Behnel committed
13
from xml.sax.saxutils import escape as html_escape
14 15
from StringIO import StringIO

16 17 18
from . import Version
from .Code import CCodeWriter
from .. import Utils
19 20 21 22


class AnnotationCCodeWriter(CCodeWriter):

Robert Bradshaw's avatar
Robert Bradshaw committed
23
    def __init__(self, create_from=None, buffer=None, copy_formatting=True):
24
        CCodeWriter.__init__(self, create_from, buffer, copy_formatting=copy_formatting)
25
        if create_from is None:
26
            self.annotation_buffer = StringIO()
27
            self.annotations = []
28 29
            self.last_annotated_pos = None
            self.code = defaultdict(partial(defaultdict, str))
30
        else:
31
            # When creating an insertion point, keep references to the same database
32 33 34
            self.annotation_buffer = create_from.annotation_buffer
            self.annotations = create_from.annotations
            self.code = create_from.code
35
            self.last_annotated_pos = create_from.last_annotated_pos
36

37 38
    def create_new(self, create_from, buffer, copy_formatting):
        return AnnotationCCodeWriter(create_from, buffer, copy_formatting)
39

40
    def write(self, s):
41 42
        CCodeWriter.write(self, s)
        self.annotation_buffer.write(s)
43

44
    def mark_pos(self, pos, trace=True):
45
        if pos is not None:
46
            CCodeWriter.mark_pos(self, pos, trace)
47 48 49 50
        if self.last_annotated_pos:
            source_desc, line, _ = self.last_annotated_pos
            pos_code = self.code[source_desc.filename]
            pos_code[line] += self.annotation_buffer.getvalue()
51
        self.annotation_buffer = StringIO()
52
        self.last_annotated_pos = pos
53 54 55

    def annotate(self, pos, item):
        self.annotations.append((pos, item))
56

57 58
    def _css(self):
        """css template will later allow to choose a colormap"""
59
        css = [self._css_template]
60 61
        for i in range(255):
            color = u"FFFF%02x" % int(255/(1+i/10.0))
62
            css.append('.cython.score-%d {background-color: #%s;}' % (i, color))
63 64 65 66
        try:
            from pygments.formatters import HtmlFormatter
        except ImportError:
            pass
67 68 69
        else:
            css.append(HtmlFormatter().get_style_defs('.cython'))
        return '\n'.join(css)
70

71
    _js = """
72 73 74 75 76
    function toggleDiv(id) {
        theDiv = id.nextElementSibling
        if (theDiv.style.display != 'block') theDiv.style.display = 'block';
        else theDiv.style.display = 'none';
    }
77
    """.strip()
78

79 80
    _css_template = textwrap.dedent("""
        body.cython { font-family: courier; font-size: 12; }
81

82 83
        .cython.tag  {  }
        .cython.line { margin: 0em }
84
        .cython.code { font-size: 9; color: #444444; display: none; margin: 0px 0px 0px 8px; border-left: 8px none; }
Robert Bradshaw's avatar
Robert Bradshaw committed
85

86 87
        .cython.line .run { background-color: #B0FFB0; }
        .cython.line .mis { background-color: #FFB0B0; }
88 89
        .cython.code.run  { border-left: 8px solid #B0FFB0; }
        .cython.code.mis  { border-left: 8px solid #FFB0B0; }
90

91 92 93 94 95
        .cython.code .py_c_api  { color: red; }
        .cython.code .py_macro_api  { color: #FF7000; }
        .cython.code .pyx_c_api  { color: #FF3000; }
        .cython.code .pyx_macro_api  { color: #FF7000; }
        .cython.code .refnanny  { color: #FFA000; }
96
        .cython.code .trace  { color: #FFA000; }
97
        .cython.code .error_goto  { color: #FFA000; }
98

99 100 101 102 103 104
        .cython.code .coerce  { color: #008000; border: 1px dotted #008000 }
        .cython.code .py_attr { color: #FF0000; font-weight: bold; }
        .cython.code .c_attr  { color: #0000FF; }
        .cython.code .py_call { color: #FF0000; font-weight: bold; }
        .cython.code .c_call  { color: #0000FF; }
    """)
105

106
    def save_annotation(self, source_filename, target_filename, coverage_xml=None):
107
        with Utils.open_source_file(source_filename) as f:
108 109
            code = f.read()
        generated_code = self.code.get(source_filename, {})
110 111
        c_file = Utils.decode_filename(os.path.basename(target_filename))
        html_filename = os.path.splitext(target_filename)[0] + ".html"
112

113
        with codecs.open(html_filename, "w", encoding="UTF-8") as out_buffer:
114 115 116 117 118 119 120
            out_buffer.write(self._save_annotation(code, generated_code, c_file, source_filename, coverage_xml))

    def _save_annotation_header(self, c_file, source_filename, coverage_timestamp=None):
        coverage_info = ''
        if coverage_timestamp:
            coverage_info = u' with coverage data from {timestamp}'.format(
                timestamp=datetime.fromtimestamp(int(coverage_timestamp) // 1000))
121

122 123 124 125 126 127 128
        outlist = [
            textwrap.dedent(u'''\
            <!DOCTYPE html>
            <!-- Generated by Cython {watermark} -->
            <html>
            <head>
                <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
129
                <title>Cython: {filename}</title>
130 131 132 133 134 135 136 137
                <style type="text/css">
                {css}
                </style>
                <script>
                {js}
                </script>
            </head>
            <body class="cython">
138
            <p><span style="border-bottom: solid 1px grey;">Generated by Cython {watermark}</span>{more_info}</p>
139 140 141 142 143
            <p>
                <span style="background-color: #FFFF00">Yellow lines</span> hint at Python interaction.<br />
                Click on a line that starts with a "<code>+</code>" to see the C code that Cython generated for it.
            </p>
            ''').format(css=self._css(), js=self._js, watermark=Version.watermark,
144 145
                        filename=os.path.basename(source_filename) if source_filename else '',
                        more_info=coverage_info)
146
        ]
147
        if c_file:
148 149 150 151 152 153
            outlist.append(u'<p>Raw output: <a href="%s">%s</a></p>\n' % (c_file, c_file))
        return outlist

    def _save_annotation_footer(self):
        return (u'</body></html>\n',)

154
    def _save_annotation(self, code, generated_code, c_file=None, source_filename=None, coverage_xml=None):
155 156
        """
        lines : original cython source code split by lines
157
        generated_code : generated c code keyed by line number in original file
158 159 160
        target filename : name of the file in which to store the generated html
        c_file : filename in which the c_code has been written
        """
161 162 163 164 165 166
        if coverage_xml is not None and source_filename:
            coverage_timestamp = coverage_xml.get('timestamp', '').strip()
            covered_lines = self._get_line_coverage(coverage_xml, source_filename)
        else:
            coverage_timestamp = covered_lines = None

167
        outlist = []
168 169
        outlist.extend(self._save_annotation_header(c_file, source_filename, coverage_timestamp))
        outlist.extend(self._save_annotation_body(code, generated_code, covered_lines))
170 171 172
        outlist.extend(self._save_annotation_footer())
        return ''.join(outlist)

173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
    def _get_line_coverage(self, coverage_xml, source_filename):
        coverage_data = None
        for entry in coverage_xml.iterfind('.//class'):
            if not entry.get('filename'):
                continue
            if (entry.get('filename') == source_filename or
                    os.path.abspath(entry.get('filename')) == source_filename):
                coverage_data = entry
                break
            elif source_filename.endswith(entry.get('filename')):
                coverage_data = entry  # but we might still find a better match...
        if coverage_data is None:
            return None
        return dict(
            (int(line.get('number')), int(line.get('hits')))
            for line in coverage_data.iterfind('lines/line')
        )

191 192
    def _htmlify_code(self, code):
        try:
193 194 195 196
            from pygments import highlight
            from pygments.lexers import CythonLexer
            from pygments.formatters import HtmlFormatter
        except ImportError:
197 198
            # no Pygments, just escape the code
            return html_escape(code)
199

200 201 202 203
        html_code = highlight(
            code, CythonLexer(stripnl=False, stripall=False),
            HtmlFormatter(nowrap=True))
        return html_code
204

205
    def _save_annotation_body(self, cython_code, generated_code, covered_lines=None):
206
        outlist = [u'<div class="cython">']
207
        pos_comment_marker = u'/* \N{HORIZONTAL ELLIPSIS} */\n'
Stefan Behnel's avatar
Stefan Behnel committed
208 209
        new_calls_map = dict(
            (name, 0) for name in
210
            'refnanny trace py_macro_api py_c_api pyx_macro_api pyx_c_api error_goto'.split()
Stefan Behnel's avatar
Stefan Behnel committed
211
        ).copy
212

213 214
        self.mark_pos(None)

215 216 217
        def annotate(match):
            group_name = match.lastgroup
            calls[group_name] += 1
218
            return ur"<span class='%s'>%s</span>" % (
219
                group_name, match.group(group_name))
220

221
        lines = self._htmlify_code(cython_code).splitlines()
Stefan Behnel's avatar
Stefan Behnel committed
222
        lineno_width = len(str(len(lines)))
223 224
        if not covered_lines:
            covered_lines = None
225

226
        for k, line in enumerate(lines, 1):
227
            try:
228
                c_code = generated_code[k]
229
            except KeyError:
230
                c_code = ''
231
            else:
232 233 234 235
                c_code = _replace_pos_comment(pos_comment_marker, c_code)
                if c_code.startswith(pos_comment_marker):
                    c_code = c_code[len(pos_comment_marker):]
                c_code = html_escape(c_code)
236

Stefan Behnel's avatar
Stefan Behnel committed
237
            calls = new_calls_map()
238
            c_code = _parse_code(annotate, c_code)
239
            score = (5 * calls['py_c_api'] + 2 * calls['pyx_c_api'] +
240
                     calls['py_macro_api'] + calls['pyx_macro_api'])
241

242 243
            if c_code:
                onclick = " onclick='toggleDiv(this)'"
244
                expandsymbol = '+'
245 246 247 248
            else:
                onclick = ''
                expandsymbol = '&#xA0;'

249 250 251 252 253 254
            covered = ''
            if covered_lines is not None and k in covered_lines:
                hits = covered_lines[k]
                if hits is not None:
                    covered = 'run' if hits else 'mis'

255
            outlist.append(
256
                u'<pre class="cython line score-{score}"{onclick}>'
257 258
                # generate line number with expand symbol in front,
                # and the right  number of digit
259
                u'{expandsymbol}<span class="{covered}">{line:0{lineno_width}d}</span>: {code}</pre>\n'.format(
260 261
                    score=score,
                    expandsymbol=expandsymbol,
262
                    covered=covered,
Stefan Behnel's avatar
Stefan Behnel committed
263
                    lineno_width=lineno_width,
264 265 266 267 268
                    line=k,
                    code=line.rstrip(),
                    onclick=onclick,
                ))
            if c_code:
269 270
                outlist.append(u"<pre class='cython code score-{score} {covered}'>{code}</pre>".format(
                    score=score, covered=covered, code=c_code))
271
        outlist.append(u"</div>")
272
        return outlist
273

274

275 276
_parse_code = re.compile(
    ur'(?P<refnanny>__Pyx_X?(?:GOT|GIVE)REF|__Pyx_RefNanny[A-Za-z]+)|'
277
    ur'(?P<trace>__Pyx_Trace[A-Za-z]+)|'
278 279
    ur'(?:'
    ur'(?P<pyx_macro_api>__Pyx_[A-Z][A-Z_]+)|'
Stefan Behnel's avatar
Stefan Behnel committed
280
    ur'(?P<pyx_c_api>__Pyx_[A-Z][a-z_][A-Za-z_]+)|'
281
    ur'(?P<py_macro_api>Py[A-Z][a-z]+_[A-Z][A-Z_]+)|'
Stefan Behnel's avatar
Stefan Behnel committed
282
    ur'(?P<py_c_api>Py[A-Z][a-z]+_[A-Z][a-z][A-Za-z_]+)'
283 284 285 286 287
    ur')(?=\()|'       # look-ahead to exclude subsequent '(' from replacement
    ur'(?P<error_goto>(?:(?<=;) *if .* +)?\{__pyx_filename = .*goto __pyx_L\w+;\})'
).sub


288 289 290 291 292 293 294
_replace_pos_comment = re.compile(
    # this matches what Cython generates as code line marker comment
    ur'^\s*/\*(?:(?:[^*]|\*[^/])*\n)+\s*\*/\s*\n',
    re.M
).sub


295
class AnnotationItem(object):
296

297 298 299 300 301
    def __init__(self, style, text, tag="", size=0):
        self.style = style
        self.text = text
        self.tag = tag
        self.size = size
302

303
    def start(self):
304
        return u"<span class='cython tag %s' title='%s'>%s" % (self.style, self.text, self.tag)
305

306
    def end(self):
307
        return self.size, u"</span>"