Move format parsing logic to ListFormatParser

ListFormatParser is now the core of PrettyPrinterFormatFilter, but can
also be easily used in another areas. Whole functionality of
PrettyPrinterAlignFilter was also moved into it.

Main advantage of having separate parser for list_format is possibility
of initializing it and resolving placeholders only once. Previous
implementation did that with each Todo object.

test/data/listformat.conf

+[ls]
+list_format                 = |%I| %x %{(}p{)} %c %S\t%K

test/test_list_format.py

@@ -52,7 +52,7 @@ class ListFormatTest(CommandTest):
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format02(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(80, 25)
 
@@ -69,7 +69,7 @@ class ListFormatTest(CommandTest):
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format03(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(100, 25)
 
@@ -86,7 +86,7 @@ class ListFormatTest(CommandTest):
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format04(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(100, 25)
 
@@ -103,7 +103,7 @@ class ListFormatTest(CommandTest):
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format05(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(80, 25)
 
@@ -119,7 +119,7 @@ class ListFormatTest(CommandTest):
 |  6| x 2014-12-12 Completed but with                           date:2014-12-12
 """
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format06(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(100, 25)
 
@@ -136,7 +136,7 @@ class ListFormatTest(CommandTest):
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format07(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(100, 25)
 
@@ -153,7 +153,7 @@ class ListFormatTest(CommandTest):
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format08(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(100, 25)
 
@@ -170,7 +170,7 @@ x 2014-12-12
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format09(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(100, 25)
 
@@ -243,7 +243,7 @@ just now | in 2 days | in a day |
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format14(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(40, 25)
         command = ListCommand(["-x", "-F", "|%I| %x %{(}p{)} %c %s	%K", "@Context1"],
@@ -412,7 +412,7 @@ Completed but with
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format27(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(50, 25)
 
@@ -519,7 +519,7 @@ ZZ
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format35(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(5, 25)
         command = ListCommand(["-x", "-s", "desc:priority", "-F", "%p{ }	%{ }p"], self.todolist, self.out, self.error)
@@ -534,10 +534,10 @@ Z  Z
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format36(self, mock_terminal_size):
         """Tab expands to 1 character."""
-        mock_terminal_size.return_value = self.terminal_size(6, 25)
+        mock_terminal_size.return_value =  self.terminal_size(6, 25)
         command = ListCommand(["-x", "-s", "desc:priority", "-F", "%p{ }	%{ }p"], self.todolist, self.out, self.error)
         command.execute()
 
@@ -550,7 +550,7 @@ Z   Z
 """
         self.assertEqual(self.output, result)
 
-    @mock.patch('topydo.lib.PrettyPrinterFilter.get_terminal_size')
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
     def test_list_format37(self, mock_terminal_size):
         mock_terminal_size.return_value = self.terminal_size(5, 25)
         command = ListCommand(["-x", "-s", "desc:priority", "-F", "	%{ }p"], self.todolist, self.out, self.error)
@@ -595,6 +595,39 @@ Z   Z
 
 
 
+"""
+        self.assertEqual(self.output, result)
+
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
+    def test_list_format40(self, mock_terminal_size):
+        mock_terminal_size.return_value = self.terminal_size(100, 25)
+
+        config('test/data/listformat.conf')
+        command = ListCommand(["-x"], self.todolist, self.out, self.error)
+        command.execute()
+
+        result = u"""|  1| (D) 2015-08-31 Bar @Context1 +Project2                            due:2015-09-30 t:2015-09-29
+|  2| (Z) 2015-11-06 Lorem ipsum dolorem sit amet. Red @fox... due:2015-11-08 lazy:bar t:2015-11-07
+|  3| (C) 2015-07-12 Foo @Context2 Not@Context +Project1 Not+Project
+|  4| (C) Baz @Context1 +Project1                                                         key:value
+|  5| Drink beer @ home                                                        ical:foobar id:1 p:2
+|  6| x 2014-12-12 Completed but with                                               date:2014-12-12
+"""
+        self.assertEqual(self.output, result)
+
+    @mock.patch('topydo.lib.ListFormat.get_terminal_size')
+    def test_list_format41(self, mock_terminal_size):
+        mock_terminal_size.return_value = self.terminal_size(100, 25)
+
+        command = ListCommand(["-x", "-F", "|%I| %x %{(}p{)} %c %S\\t%K"], self.todolist, self.out, self.error)
+        command.execute()
+
+        result = u"""|  1| (D) 2015-08-31 Bar @Context1 +Project2                            due:2015-09-30 t:2015-09-29
+|  2| (Z) 2015-11-06 Lorem ipsum dolorem sit amet. Red @fox... due:2015-11-08 lazy:bar t:2015-11-07
+|  3| (C) 2015-07-12 Foo @Context2 Not@Context +Project1 Not+Project
+|  4| (C) Baz @Context1 +Project1                                                         key:value
+|  5| Drink beer @ home                                                        ical:foobar id:1 p:2
+|  6| x 2014-12-12 Completed but with                                               date:2014-12-12
 """
         self.assertEqual(self.output, result)
 

topydo/commands/ListCommand.py

@@ -16,9 +16,9 @@
 
 from topydo.lib.Config import config
 from topydo.lib.ExpressionCommand import ExpressionCommand
+from topydo.lib.ListFormat import ListFormatParser
 from topydo.lib.PrettyPrinter import pretty_printer_factory
-from topydo.lib.PrettyPrinterFilter import (PrettyPrinterAlignFilter,
-                                            PrettyPrinterIndentFilter,
+from topydo.lib.PrettyPrinterFilter import (PrettyPrinterIndentFilter,
                                             PrettyPrinterFormatFilter)
 
 
@@ -90,9 +90,7 @@ class ListCommand(ExpressionCommand):
             hidden_tags = config().hidden_tags()
 
             filters = []
-            filters.append(PrettyPrinterFormatFilter(self.todolist,
-                           self.format))
-            filters.append(PrettyPrinterAlignFilter())
+            filters.append(PrettyPrinterFormatFilter(self.todolist, self.format))
             filters.append(PrettyPrinterIndentFilter(indent))
 
             self.printer = pretty_printer_factory(self.todolist, filters)

topydo/lib/ListFormat.py

@@ -17,6 +17,17 @@
 """ Ulities for formatting output with "list_format" option."""
 
 import arrow
+import re
+
+from six import u
+
+from topydo.lib.Config import config
+from topydo.lib.Utils import get_terminal_size
+
+MAIN_PATTERN = (r'^({{(?P<before>.+?)}})?'
+                r'(?P<placeholder>{ph}|\[{ph}\])'
+                r'({{(?P<after>.+?)}})?'
+                r'(?P<whitespace> *)')
 
 def filler(p_str, p_len):
     """
@@ -74,3 +85,188 @@ def strip_placeholder_braces(p_matchobj):
     whitespace = p_matchobj.group('whitespace') or ''
 
     return before + '%' + placeholder + after + whitespace
+
+def unescape_percent_sign(p_str):
+    """ Strips backslashes from escaped percent signs in p_str. """
+    unescaped_str = re.sub(r'\\%', '%', p_str)
+
+    return unescaped_str
+
+def remove_redundant_spaces(p_str):
+    """ Removes spaces surrunding <TAB> character (\t) from p_str. """
+    clean_str = re.sub(' *\t *', '\t', p_str)
+
+    return clean_str
+
+class ListFormatParser(object):
+    """ Parser of format string. """
+    def __init__(self, p_todolist, p_format=None):
+        self.format_string = re.sub(r'\\t', '\t', p_format or config().list_format())
+        self.todolist = p_todolist
+        self.one_line = False
+        self.line_width = get_terminal_size().columns
+        self.placeholders = {
+            # absolute creation date
+            'c': lambda t: t.creation_date().isoformat() if t.creation_date() else '',
+
+            # relative creation date
+            'C': lambda t: humanize_date(t.creation_date()) if t.creation_date() else '',
+
+            # absolute due date
+            'd': lambda t: t.due_date().isoformat() if t.due_date() else '',
+
+            # relative due date
+            'D': lambda t: humanize_date(t.due_date()) if t.due_date() else '',
+
+            # relative dates:  due, start
+            'h': lambda t: humanize_dates(t.due_date(), t.start_date()),
+
+            # relative dates in form:  creation, due, start
+            'H': lambda t: humanize_dates(t.due_date(), t.start_date(), t.creation_date()),
+            # todo ID
+            'i': lambda t: str(self.todolist.number(t)),
+
+            # todo ID pre-filled with 1 or 2 spaces if its length is <3
+            'I': lambda t: filler(str(self.todolist.number(t)), 3),
+
+
+            # list of tags (spaces) without hidden ones and due: and t:
+            'k': lambda t: ' '.join([u('{}:{}').format(tag, value)
+                                        for tag, value in sorted(t.tags()) if
+                                        tag not in config().hidden_tags() + [config().tag_start(), config().tag_due()]]),
+
+            # list of all tags (spaces)
+            'K': lambda t: ' '.join([u('{}:{}').format(tag, value)
+                                        for tag, value in sorted(t.tags())]),
+
+            # priority
+            'p': lambda t: t.priority() if t.priority() else '',
+
+            # text
+            's': lambda t: t.text(),
+
+            # text (truncated if necessary)
+            'S': lambda t: t.text(),
+
+            # absolute start date
+            't': lambda t: t.start_date().isoformat() if t.start_date() else '',
+
+            # relative start date
+            'T': lambda t: humanize_date(t.start_date()) if t.start_date() else '',
+
+            # absolute completion date
+            'x': lambda t: 'x ' + t.completion_date().isoformat() if t.is_completed() else '',
+
+            # relative completion date
+            'X': lambda t: 'x ' + humanize_date(t.completion_date()) if t.is_completed() else '',
+        }
+        self.format_list = self._preprocess_format()
+
+    def _preprocess_format(self):
+        """
+        Preprocess the format_string attribute.
+
+        Splits the format string on each placeholder and returns a list of
+        tuples containing substring, placeholder name, and function
+        retrieving content for placeholder (getter).
+
+        Relevant placeholder functions (getters) are taken from
+        'placeholders' attribute which is a dict. If no matching placeholder
+        is found in 'placeholders' getter is set to None. Getter and
+        placeholder are also always set to None in first element of the
+        returned list, because it never contain a real placeholder (read
+        re.split documentation for further information).
+        """
+        format_split = re.split(r'(?<!\\)%', self.format_string)
+        preprocessed_format = []
+
+        for idx, substr in enumerate(format_split):
+            if idx == 0:
+                getter = None
+                placeholder = None
+            else:
+                pattern = MAIN_PATTERN.format(ph=r'\S')
+                try:
+                    placeholder = re.match(pattern, substr).group('placeholder').strip('[]')
+                except AttributeError:
+                    placeholder = None
+
+                if placeholder == 'S':
+                    self.one_line = True
+
+                try:
+                    getter = self.placeholders[placeholder]
+                except KeyError:
+                    getter = None
+                    substr = re.sub(pattern, '', substr)
+
+            format_elem = (substr, placeholder, getter)
+            preprocessed_format.append(format_elem)
+
+        return preprocessed_format
+
+    def truncate(self, p_str, p_repl):
+        """
+        Returns p_str with truncated and ended with '...' version of p_repl.
+
+        Place of the truncation is calculated depending on 'line_width'
+        attribute.
+        """
+        text_lim = self.line_width - len(p_str) - 4
+        truncated_str = re.sub(re.escape(p_repl), p_repl[:text_lim] + '...', p_str)
+
+        return truncated_str
+
+    def right_align(self, p_str):
+        """
+        Returns p_str with content after <TAB> character aligned right.
+
+        Right alignment is done using proper number of spaces calculated from
+        'line_width' attribute.
+        """
+        to_fill = self.line_width - len(p_str)
+
+        if to_fill > 0:
+            p_str = re.sub('\t', ' '*to_fill, p_str)
+        else:
+            p_str = re.sub('\t', ' ', p_str)
+
+        return p_str
+
+    def parse(self, p_todo):
+        """
+        Returns fully parsed string from 'format_string' attribute with all
+        placeholders properly substituted by content obtained from p_todo.
+
+        It uses preprocessed form of 'format_string' (result of
+        ListFormatParser._preprocess_format) stored in 'format_list'
+        attribute.
+        """
+        parsed_list = []
+        repl_S = None
+
+        for substr, placeholder, getter in self.format_list:
+            repl = getter(p_todo) if getter else ''
+            pattern = MAIN_PATTERN.format(ph=placeholder)
+
+            if placeholder == 'S':
+                repl_S = repl
+
+            if repl == '':
+                substr = re.sub(pattern, '', substr)
+            else:
+                substr = re.sub(pattern, strip_placeholder_braces, substr)
+                substr = re.sub(r'(?<!\\)%({ph}|\[{ph}\])'.format(ph=placeholder), repl, substr)
+
+            parsed_list.append(substr)
+
+        parsed_str = unescape_percent_sign(''.join(parsed_list))
+        parsed_str = remove_redundant_spaces(parsed_str)
+
+        if self.one_line and len(parsed_str) >= self.line_width:
+            parsed_str = self.truncate(parsed_str, repl_S)
+
+        if re.search('.*\t', parsed_str):
+            parsed_str = self.right_align(parsed_str)
+
+        return parsed_str.rstrip()

topydo/lib/PrettyPrinterFilter.py

@@ -23,8 +23,7 @@ from six import u
 
 from topydo.lib.Colors import NEUTRAL_COLOR, Colors
 from topydo.lib.Config import config
-from topydo.lib.ListFormat import (filler, humanize_date, humanize_dates,
-                                   strip_placeholder_braces)
+from topydo.lib.ListFormat import ListFormatParser
 from topydo.lib.Utils import get_terminal_size
 
 
@@ -118,133 +117,9 @@ class PrettyPrinterNumbers(PrettyPrinterFilter):
 class PrettyPrinterFormatFilter(PrettyPrinterFilter):
     def __init__(self, p_todolist, p_format=None):
         super(PrettyPrinterFormatFilter, self).__init__()
-        self.todolist = p_todolist
-        self.format = p_format or config().list_format()
+        self.parser = ListFormatParser(p_todolist, p_format)
 
     def filter(self, p_todo_str, p_todo):
-        placeholders = OrderedDict()
-        # absolute creation date
-        placeholders['c'] = lambda t: t.creation_date().isoformat() if t.creation_date() else ''
-
-        # relative creation date
-        placeholders['C'] = lambda t: humanize_date(t.creation_date()) if t.creation_date() else ''
-
-        # absolute due date
-        placeholders['d'] = lambda t: t.due_date().isoformat() if t.due_date() else ''
-
-        # relative due date
-        placeholders['D'] = lambda t: humanize_date(t.due_date()) if t.due_date() else ''
-
-        # relative dates:  due, start
-        placeholders['h'] = lambda t: humanize_dates(t.due_date(), t.start_date())
-
-        # relative dates in form:  creation, due, start
-        placeholders['H'] = lambda t: humanize_dates(t.due_date(), t.start_date(), t.creation_date())
-
-        # todo ID
-        placeholders['i'] = lambda t: str(self.todolist.number(t))
-
-        # todo ID pre-filled with 1 or 2 spaces if its length is <3
-        placeholders['I'] = lambda t: filler(str(self.todolist.number(t)), 3)
-
-        # list of tags (spaces) without hidden ones and due: and t:
-        placeholders['k'] = lambda t: ' '.join([u('{}:{}').format(tag, value)
-                                    for tag, value in sorted(p_todo.tags()) if
-                                    tag not in config().hidden_tags() + [config().tag_start(), config().tag_due()]])
-
-        # list of all tags (spaces)
-        placeholders['K'] = lambda t: ' '.join([u('{}:{}').format(tag, value)
-                                    for tag, value in sorted(p_todo.tags())])
-
-        # priority
-        placeholders['p'] = lambda t: t.priority() if t.priority() else ''
-
-        # text
-        placeholders['s'] = lambda t: t.text()
-
-        # absolute start date
-        placeholders['t'] = lambda t: t.start_date().isoformat() if t.start_date() else ''
-
-        # relative start date
-        placeholders['T'] = lambda t: humanize_date(t.start_date()) if t.start_date() else ''
-
-        # absolute completion date
-        placeholders['x'] = lambda t: 'x ' + t.completion_date().isoformat() if t.is_completed() else ''
-
-        # relative completion date
-        placeholders['X'] = lambda t: 'x ' + humanize_date(t.completion_date()) if t.is_completed() else ''
-
-        # text (truncated if necessary)
-        placeholders['S'] = lambda t: t.text()
-
-        p_todo_str = re.sub(r'\\t', '\t', self.format)
-        p_todo_str_list = re.split(r'(?<!\\)%', p_todo_str)
-        main_pattern = (r'^({{(?P<before>.+?)}})?'
-                        r'(?P<placeholder>{ph}|\[{ph}\])'
-                        r'({{(?P<after>.+?)}})?'
-                        r'(?P<whitespace> *)')
-        truncate = False
-
-        for index, substr in enumerate(p_todo_str_list):
-            if index == 0:
-                continue    # first item in p_todo_str_list is surely not a placeholder
-            if not re.match(main_pattern.format(ph='['+''.join(placeholders.keys()) + ']'), substr):
-                substr = re.sub(main_pattern.format(ph='.'), '', substr) # remove nonexistent placeholder
-                p_todo_str_list[index] = substr
-                continue
-            for placeholder, getter in placeholders.items():
-                repl = getter(p_todo)
-                pattern = main_pattern.format(ph=placeholder)
-                match = re.match(pattern, substr)
-                if match:
-                    if repl == '':
-                        substr = re.sub(pattern, '', substr)
-                    else:
-                        substr = re.sub(pattern, strip_placeholder_braces, substr)
-                        substr = re.sub(r'(?<!\\)%({ph}|\[{ph}\])'.format(ph=placeholder), repl, substr)
-
-                        if placeholder == 'S':
-                            truncate = True
-                            repl_S = repl # copy for truncating final p_todo_str
-
-                    p_todo_str_list[index] = substr
-                    break
-
-        p_todo_str = ''.join(p_todo_str_list)
-        p_todo_str = re.sub(r'\\%', '%', p_todo_str)
-        p_todo_str = re.sub(' *\t *', '\t', p_todo_str)
-
-        if truncate:
-            line_width = get_terminal_size().columns
-            if len(p_todo_str) >= line_width:
-                text_lim = line_width - len(p_todo_str) - 4
-                p_todo_str = re.sub(re.escape(repl_S), repl_S[:text_lim] + '...', p_todo_str)
-
-        # cut trailing space left when last placeholder in p_todo_str is empty and its predecessor is not
-        return p_todo_str.rstrip()
-
-
-class PrettyPrinterAlignFilter(PrettyPrinterFilter):
-    """
-    Final make-up for todo item line.
-    Currently it only provides right alignment from place specified in
-    list_format config-option (subsitutes tab-character with as many spaces as
-    it is needed to fill the whole line).
-    """
-
-    def __init__(self):
-        super(PrettyPrinterAlignFilter, self).__init__()
-
-    def filter(self, p_todo_str, _):
-        tab = re.search('.*\t', p_todo_str)
-
-        if tab:
-            line_width = get_terminal_size().columns
-            to_fill = line_width - len(p_todo_str)
-
-            if to_fill > 0:
-                p_todo_str = re.sub('\t', ' '*to_fill, p_todo_str)
-            else:
-                p_todo_str = re.sub('\t', ' ', p_todo_str)
+        p_todo_str = self.parser.parse(p_todo)
 
         return p_todo_str