WWIP getHateoas - separated "field" value resolution into function

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_style/ERP5Document_getHateoas.py

+"""Hello. This will be long because this goodness script does almost everything.
+
+In general it always returns a JSON reponse in HATEOAS format specification.
+
+:param REQUEST: HttpRequest holding GET and/or POST data
+:param response:
+:param view: either "view" or absolute URL of an ERP5 Action
+:param mode: {str} help to decide what user wants from us "form" | "search" ...
+:param relative_url: an URL of `traversed_document` to operate on (it must have an object_view)
+
+Only in mode == 'search'
+:param query:
+:param select_list:
+:param limit:tr
+:param form_relative_url: {str} relative URL of a form FIELD issuing the search (listbox/relation field...)
+
+Only in mode == 'form'
+:param form:
+
+Only in mode == 'traverse'
+
+TBD.
+"""
 from ZTUtils import make_query
 import json
 from base64 import urlsafe_b64encode, urlsafe_b64decode
@@ -8,6 +31,8 @@ import time
 from email.Utils import formatdate
 import re
 from zExceptions import Unauthorized
+from Products.ERP5Type.Utils import UpperCase
+from Products.ZSQLCatalog.SQLCatalog import Query, ComplexQuery
 
 if REQUEST is None:
   REQUEST = context.REQUEST
@@ -26,20 +51,212 @@ def byteify(string):
   else:
     return string
 
+def ensure_serializable(obj):
+  """Ensure obj and all sub-objects are JSON serializable."""
+  if isinstance(obj, dict):
+    for key in obj:
+      obj[key] = ensure_serializable(obj[key])
+  # throw away date's type information and later reconstruct as Zope's DateTime
+  if isinstance(obj, DateTime):
+    return obj.ISO()
+  if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
+    return obj.isoformat()
+  # we don't check other isinstances - we believe that iterables don't contain unserializable objects
+  return obj
+
+datetime_iso_re = re.compile(r'^\d{4}-\d{2}-\d{2} |T\d{2}:\d{2}:\d{2}.*$')
+time_iso_re = re.compile(r'^(\d{2}):(\d{2}):(\d{2}).*$')
+def ensure_deserialized(obj):
+  """Deserialize classes serialized by our own `ensure_serializable`.
+
+  Method `biteify` must not be called on the result because it would revert out
+  deserialization by calling __str__ on constructed classes.
+  """
+  if isinstance(obj, dict):
+    for key in obj:
+      obj[key] = ensure_deserialized(obj[key])
+  # seems that default __str__ method is good enough
+  if isinstance(obj, str):
+    # Zope's DateTime must be good enough for everyone
+    if datetime_iso_re.match(obj):
+      return DateTime(obj)
+    if time_iso_re.match(obj):
+      match_obj = time_iso_re.match(obj)
+      return datetime.time(*tuple(map(int, match_obj.groups())))
+  return obj
+
+
 def getProtectedProperty(document, select):
+  """getProtectedProperty is a security-aware substitution for builtin `getattr`
+
+  It resolves Properties on Products (visible via Zope Formulator), which are
+  accessible as ordinary attributes as well, by following security rules.
+
+  See https://lab.nexedi.com/nexedi/erp5/blob/master/product/ERP5Form/ListBox.py#L2293
+  """
   try:
-    #see https://lab.nexedi.com/nexedi/erp5/blob/master/product/ERP5Form/ListBox.py#L2293
-    try:
+    if "." in select:
       select = select[select.rindex('.') + 1:]
-    except ValueError:
-      pass
-
     return document.getProperty(select, d=None)
   except (ConflictError, RuntimeError):
     raise
   except:
     return None
 
+
+def object_uids_and_accessors(search_result, result_index, traversed_document):
+  """Return unique ID, unique URL, getter and hasser for any combination of `search_result` and `index`.
+
+  You want to use this method when you need a unique reference to random object in iterable (for example
+  result of list_method or stat_method). This will give you UID and URL for identification within JIO and
+  accessors to test/access object's properties.
+
+  Usage::
+
+  for i, random_object in enumerate(unknown_iterable):
+    uid, url, getter, hasser = object_ids_and_access(random_object, i)
+    if hasser(random_object, "linkable"):
+      result[uid] = {'url': portal.abolute_url() + url}
+    value = getter(random_object, "value")
+  """
+  if hasattr(search_result, "getObject"):
+    # search_result = search_result.getObject()
+    contents_uid = search_result.uid
+    # every document indexed in catalog has to have relativeUrl
+    contents_relative_url = getRealRelativeUrl(search_result)
+    # get property in secure way from documents
+    search_property_getter = getProtectedProperty
+    def search_property_hasser (doc, attr):
+      """Brains cannot access Properties - they use permissioned getters."""
+      try:
+        return doc.hasProperty(attr)
+      except (AttributeError, Unauthorized) as e:
+        context.log('Cannot state ownership of property "{}" on {!s} because of "{!s}"'.format(
+          attr, doc, e))
+        return False
+  elif hasattr(search_result, "aq_self"):
+    # Zope products have at least ID thus we work with that
+    contents_uid = search_result.uid
+    # either we got a document with relativeUrl or we got product and use ID
+    contents_relative_url = getRealRelativeUrl(search_result) or search_result.getId()
+    # documents and products have the same way of accessing properties
+    search_property_getter = getProtectedProperty
+    search_property_hasser = lambda doc, attr: doc.hasProperty(attr)
+  else:
+    # In case of reports the `search_result` can be list of
+    # PythonScripts.standard._Object - a reimplementation of plain dictionary
+
+    # means we are iterating over plain objects
+    # list_method must be defined because POPOs can return only that
+    contents_uid = "{}#{:d}".format(list_method, result_index)
+    # JIO requires every item to have _links.self.href so it can construct
+    # links to the document. Here we have a object in RAM (which should
+    # never happen!) thus we provide temporary UID
+    contents_relative_url = "{}/{}".format(traversed_document.getRelativeUrl(), contents_uid)
+    # property getter must be simple __getattr__ implementation
+    search_property_getter = lambda obj, attr: getattr(obj, attr, None)
+    search_property_hasser = lambda obj, attr: hasattr(obj, attr)
+
+  return contents_uid, contents_relative_url, search_property_getter, search_property_hasser
+
+
+def resolve_field(search_result, select, search_property_getter, search_property_hasser):
+  """Given `data_source` extract fields defined in `field_list` and render them using `field_template_dict`.
+
+  :param data_source: any dict-like object (usually dict or Brain or Document)
+  :select: field name (can represent actual Properties or Scripts)
+  """
+
+  # if the variable does not have a field template we need to find its
+  # value by resolving value in the correct order. The code is copy&pasted
+  # from ListBoxRendererLine.getValueList because it is universal
+  contents_value = None
+
+  if not isinstance(select, (str, unicode)) or len(select) == 0:
+    context.log('There is an invalid column name "{!s}"!'.format(select), level=200)
+    return None
+
+  if "." in select:
+    select = select[select.rindex('.') + 1:]
+
+  # 1. resolve attribute on a raw object (all wrappers removed) using
+  # lowest-level secure getattr method given object type
+  raw_search_result = search_result
+  if hasattr(search_result, 'aq_base'):
+    raw_search_result = search_result.aq_base
+
+  if search_property_hasser(raw_search_result, select):
+    contents_value = search_property_getter(raw_search_result, select)
+
+  # 2. use the fact that wrappers (brain or acquisition wrapper) use
+  # permissioned getters
+  unwrapped_search_result = search_result
+  if hasattr(search_result, 'aq_self'):
+    unwrapped_search_result = search_result.aq_self
+
+  if contents_value is None:
+    if not select.startswith('get') and select[0] not in string.ascii_uppercase:
+      # maybe a hidden getter (variable accessible by a getter)
+      accessor_name = 'get' + UpperCase(select)
+    else:
+      # or obvious getter (starts with "get" or Capital letter - Script)
+      accessor_name = select
+    # again we check on a unwrapped object to avoid acquisition resolution
+    # which would certainly find something which we don't want
+    try:
+      if hasattr(raw_search_result, accessor_name) and callable(getattr(search_result, accessor_name)):
+        # test on raw object but get the actual accessor using wrapper and acquisition
+        # do not call it here - it will be done later in generic call part
+        contents_value = getattr(search_result, accessor_name)
+    except (AttributeError, KeyError, Unauthorized) as error:
+      context.log("Could not evaluate {} nor {} on {} with error {!s}".format(
+        select, accessor_name, search_result, error), level=100)  # WARNING
+
+  if contents_value is None and search_property_hasser(search_result, select):
+    # maybe it is just a attribute
+    contents_value = search_property_getter(search_result, select)
+
+  if contents_value is None:
+    try:
+      contents_value = getattr(search_result, select, None)
+    except (Unauthorized, AttributeError, KeyError) as error:
+      context.log("Cannot resolve {} on {!s} because {!s}".format(
+        select, raw_search_result, error), level=100)
+
+  if callable(contents_value):
+    has_mandatory_param = False
+    has_brain_param = False
+    if hasattr(contents_value, "params"):
+      has_mandatory_param = any(map(lambda param: '=' not in param and '*' not in param,
+                                    contents_value.params().split(","))) \
+                            if contents_value.params() \
+                            else False # because any([]) == True
+      has_brain_param = "brain" in contents_value.params()
+    try:
+      if has_mandatory_param:
+        contents_value = contents_value(search_result)
+      elif has_brain_param:
+        contents_value = contents_value(brain=search_result)
+      else:
+        contents_value = contents_value()
+    except (AttributeError, KeyError, Unauthorized) as error:
+      context.log("Could not evaluate {} on {} with error {!s}".format(
+        contents_value, search_result, error), level=100)  # WARNING
+
+  # make resulting value JSON serializable
+  if contents_value is not None:
+    if same_type(contents_value, DateTime()):
+      # Serialize DateTime
+      contents_value = contents_value.rfc822()
+    # XXX Kato: what exactly should the later mean?
+    elif isinstance(contents_value, datetime.date):
+      contents_value = formatdate(time.mktime(contents_value.timetuple()))
+    elif hasattr(contents_value, 'translate'):
+      contents_value = "%s" % contents_value
+
+  return contents_value
+
+
 url_template_dict = {
   "form_action": "%(traversed_document_url)s/%(action_id)s",
   "traverse_generator": "%(root_url)s/%(script_id)s?mode=traverse" + \
@@ -75,18 +292,21 @@ url_template_dict = {
 default_document_uri_template = url_template_dict["jio_get_template"]
 Base_translateString = context.getPortalObject().Base_translateString
 
+
 def getRealRelativeUrl(document):
   return '/'.join(portal.portal_url.getRelativeContentPath(document))
 
+
 def getFormRelativeUrl(form):
   return portal.portal_catalog(
-    portal_type="ERP5 Form",
+    portal_type=("ERP5 Form", "ERP5 Report"),
     uid=form.getUid(),
     id=form.getId(),
     limit=1,
     select_dict={'relative_url': None}
   )[0].relative_url
 
+
 def getFieldDefault(traversed_document, field, key, value=None):
   # REQUEST.get(field.id, field.get_value("default"))
   result = traversed_document.Field_getDefaultValue(field, key, value, REQUEST)
@@ -98,6 +318,9 @@ def getFieldDefault(traversed_document, field, key, value=None):
 def renderField(traversed_document, field, form, value=None, meta_type=None, key=None, key_prefix=None, selection_params=None):
   """Extract important field's attributes into `result` dictionary."""
 
+  if selection_params is None:
+    selection_params = {}
+
   if meta_type is None:
     meta_type = field.meta_type
   if key is None:
@@ -295,19 +518,23 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
       "url": field.get_value("gadget_url"),
       "sandbox": field.get_value("js_sandbox")
     })
-    try:
-      result["renderjs_extra"] = json.dumps(dict(field.get_value("renderjs_extra")))
-    except KeyError:
-      # Ensure compatibility if the products are not yet up to date
-      result["renderjs_extra"] = json.dumps({})
     return result
 
   if meta_type == "ListBox":
-    """Display list of objects with optional search/sort capabilities on columns from catalog."""
+    """Display list of objects with optional search/sort capabilities on columns from catalog.
+
+    We might be inside a ReportBox which is inside a parent form BUT we still have access to
+    the original REQUEST with sent POST values from the parent form. We can save those
+    values into our query method and reconstruct them meanwhile calling asynchronous jio.allDocs.
+    """
     _translate = Base_translateString
 
-    column_list = [(name, _translate(title)) for name, title in field.get_value("columns")]
-    editable_column_list = [(name, _translate(title)) for name, title in field.get_value("editable_columns")]
+    # column definition in ListBox own value 'columns' is superseded by dynamic
+    # column definition from Selection for specific Report ListBoxes; the same for editable_columns
+    column_list = [(name, _translate(title)) for name, title in (selection_params.get('selection_columns', [])
+                                                                 or field.get_value("columns"))]
+    editable_column_list = [(name, _translate(title)) for name, title in (selection_params.get('editable_columns', [])
+                                                                          or field.get_value("editable_columns"))]
     catalog_column_list = [(name, title)
                            for name, title in column_list
                            if sql_catalog.isValidColumn(name)]
@@ -319,28 +546,77 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
 
     # try to get specified sortable columns and fail back to searchable fields
     sort_column_list = [(name, _translate(title))
-                        for name, title in field.get_value("sort_columns")
+                        for name, title in (selection_params.get('selection_sort_order', [])
+                                            or field.get_value("sort_columns"))
                         if sql_catalog.isValidColumn(name)] or search_column_list
+    # portal_type list can be overriden by selection too
+    # since it can be intentionally empty we don't override with non-empty field value
+    portal_type_list = selection_params.get("portal_type", field.get_value('portal_types'))
 
     # requirement: get only sortable/searchable columns which are already displayed in listbox
     # see https://lab.nexedi.com/nexedi/erp5/blob/HEAD/product/ERP5Form/ListBox.py#L1004
     # implemented in javascript in the end
     # see https://lab.nexedi.com/nexedi/erp5/blob/master/bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_gadget_erp5_listbox_js.js#L163
 
-    portal_types = field.get_value('portal_types')
-    default_params = dict(field.get_value('default_params'))
+    default_params = dict(field.get_value('default_params'))  # default_params is a list of tuples
     default_params['ignore_unknown_columns'] = True
-    if selection_params is not None:
-      default_params.update(selection_params)
-    # How to implement pagination?
-    # default_params.update(REQUEST.form)
-    lines = field.get_value('lines')
-    list_method_name = traversed_document.Listbox_getListMethodName(field)
-    list_method_query_dict = dict(
-      portal_type=[x[1] for x in portal_types], **default_params
-    )
+    # we abandoned Selections in RJS thus we mix selection query parameters into
+    # listbox's default parameters
+    default_params.update(selection_params)
+
+    # ListBoxes in report view has portal_type defined already in default_params
+    # in that case we prefer non_empty version
+    list_method_query_dict = default_params.copy()
+    if not list_method_query_dict.get("portal_type", []):
+      list_method_query_dict["portal_type"] = [x for x, _ in portal_type_list]
     list_method_custom = None
 
+    # Search for non-editable documents - all reports goes here
+    # Reports have custom search scripts which wants parameters from the form
+    # thus we introspect such parameters and try to find them in REQUEST
+    list_method = None
+    list_method_name = traversed_document.Listbox_getListMethodName(field)
+    if list_method_name not in ("", "portal_catalog", "searchFolder", "objectValues"):
+      # we avoid accessing known protected objects and builtin functions above
+      try:
+        list_method = getattr(traversed_document, list_method_name)
+      except (Unauthorized, AttributeError, ValueError) as error:
+        # we are touching some specially protected (usually builtin) methods
+        # which we will not introspect
+        context.log('ListBox {!s} list_method {} is unavailable because of "{!s}"'.format(
+          field, list_method_name, error), level=100)
+
+    # Put all ListBox's search method params from REQUEST to `default_param_json`
+    # because old code expects synchronous render thus having all form's values
+    # still in the request which is not our case because we do asynchronous rendering
+    if list_method is not None and hasattr(list_method, "ZScriptHTML_tryParams"):
+      for list_method_param in list_method.ZScriptHTML_tryParams():
+        if list_method_param in REQUEST and list_method_param not in list_method_query_dict:
+          list_method_query_dict[list_method_param] = REQUEST.get(list_method_param)
+      # MIDDLE-DANGEROUS!
+      # In case of reports (later even exports) substitute None for unknown
+      # parameters. We suppose Python syntax for parameters!
+      # What we do here is literally putting every form field from REQUEST
+      # into search method parameters - this is later put back into REQUEST
+      # this way we can mimic synchronous rendering when all form field values
+      # were available in REQUEST. It is obviously wrong behaviour.
+      for list_method_param in list_method.params().split(","):
+        if "*" in list_method_param:
+          continue
+        if "=" in list_method_param:
+          continue
+        # now we have only mandatory parameters
+        list_method_param = list_method_param.strip()
+        if list_method_param not in list_method_query_dict:
+          list_method_query_dict[list_method_param] = None
+      # Now if the list_method does not specify **kwargs we need to remove
+      # unwanted parameters like "portal_type" which is everywhere
+      if "**" not in list_method.params():
+        _param_key_list = tuple(list_method_query_dict.keys()) # copy the keys
+        for param_key in _param_key_list:
+          if param_key not in list_method.params():  # we search in raw string
+            del list_method_query_dict[param_key]    # but it is enough
+
     if (editable_column_list):
       list_method_custom = url_template_dict["custom_search_template"] % {
         "root_url": site_root.absolute_url(),
@@ -348,8 +624,11 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
         "relative_url": traversed_document.getRelativeUrl().replace("/", "%2F"),
         "form_relative_url": "%s/%s" % (getFormRelativeUrl(form), field.id),
         "list_method": list_method_name,
-        "default_param_json": urlsafe_b64encode(json.dumps(list_method_query_dict))
+        "default_param_json": urlsafe_b64encode(
+          json.dumps(ensure_serializable(list_method_query_dict)))
       }
+      # once we imprint `default_params` into query string of 'list method' we
+      # don't want them to propagate to the query as well
       list_method_query_dict = {}
     elif (list_method_name == "portal_catalog"):
       pass
@@ -361,7 +640,7 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
         "script_id": script.id,
         "relative_url": traversed_document.getRelativeUrl().replace("/", "%2F"),
         "list_method": list_method_name,
-        "default_param_json": urlsafe_b64encode(json.dumps(list_method_query_dict))
+        "default_param_json": urlsafe_b64encode(json.dumps(ensure_serializable(list_method_query_dict)))
       }
       list_method_query_dict = {}
 
@@ -384,7 +663,6 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
 #         line[title] = prop
 #         line["_relative_url"] = document.getRelativeUrl()
 #       line_list.append(line)
-
     result.update({
       "column_list": column_list,
       "search_column_list": search_column_list,
@@ -392,10 +670,12 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
       "sort_column_list": sort_column_list,
       "editable_column_list": editable_column_list,
       "show_anchor": field.get_value("anchor"),
-      "portal_type": portal_types,
-      "lines": lines,
-      "default_params": default_params,
+      "portal_type": portal_type_list,
+      "lines": field.get_value('lines'),
+      "default_params": ensure_serializable(default_params),
       "list_method": list_method_name,
+      "stat_method": field.get_value('stat_method').getMethodName() if field.get_value('stat_method') != "" else "",
+      "count_method": field.get_value('count_method').getMethodName() if field.get_value('count_method') != "" else "",
       "query": url_template_dict["jio_search_template"] % {
         "query": make_query({
           "query": sql_catalog.buildQuery(
@@ -452,6 +732,12 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
 
 
 def renderForm(traversed_document, form, response_dict, key_prefix=None, selection_params=None):
+  """
+  :param selection_params: holds parameters to construct ERP5Form.Selection instance
+      for underlaying ListBox - since we do not use selections in RenderJS UI
+      we mitigate the functionality here by overriding ListBox's own values
+      for columns, editable columns, and sort with those found in `selection_params`
+  """
   REQUEST.set('here', traversed_document)
   field_errors = REQUEST.get('field_errors', {})
 
@@ -531,25 +817,78 @@ def renderForm(traversed_document, form, response_dict, key_prefix=None, selecti
   }
 
   if (form.pt == 'report_view'):
+    # reports are expected to return list of ReportSection which is a wrapper
+    # around a form - thus we will need to render those forms
     report_item_list = []
     report_result_list = []
     for field in form.get_fields():
       if field.getRecursiveTemplateField().meta_type == 'ReportBox':
+        # ReportBox.render returns a list of ReportSection classes which are
+        # just containers for FormId(s) usually containing one ListBox
+        # and its search/query parameters hidden in `selection_params`
+        # `path` contains relative_url of intended CONTEXT for underlaying ListBox
         report_item_list.extend(field.render())
-    j = 0
-    for report_item in report_item_list:
-      report_context = report_item.getObject(portal)
-      report_prefix = 'x%s' % j
-      j += 1
+    # ERP5 Report document differs from a ERP5 Form in only one thing: it has
+    # `report_method` attached to it - thus we call it right here
+    if hasattr(form, 'report_method') and getattr(form, 'report_method', ""):
+      report_method_name = getattr(form, 'report_method')
+      report_method = getattr(traversed_document, report_method_name)
+      report_item_list.extend(report_method())
+
+    for report_index, report_item in enumerate(report_item_list):
+      report_context = report_item.getObject(traversed_document)
+      report_prefix = 'x%s' % report_index
       report_title = report_item.getTitle()
       # report_class = "report_title_level_%s" % report_item.getLevel()
       report_form = report_item.getFormId()
       report_result = {'_links': {}}
-      renderForm(traversed_document, getattr(report_context, report_item.getFormId()),
-                 report_result, key_prefix=report_prefix,
-                 selection_params=report_item.selection_params)
+      # some reports save a lot of unserializable data (datetime.datetime) and
+      # key "portal_type" (don't confuse with "portal_types" in ListBox) into
+      # report_item.selection_params thus we need to take that into account in
+      # ListBox field
+      #
+      # Selection Params are parameters for embedded ListBox's List Method
+      # and it must be passed in `default_json_param` field (might contain
+      # unserializable data types thus we need to take care of that
+      # In order not to lose information we put all ReportSection attributes
+      # inside the report selection params
+      report_form_params = report_item.selection_params.copy() \
+                           if report_item.selection_params is not None \
+                           else {}
+
+      if report_item.selection_name:
+        selection_name = report_prefix + "_" + report_item.selection_name
+        report_form_params.update(selection_name=selection_name)
+        # this should load selections with correct values - since it is modifying
+        # global state in the backend we have nothing more to do here
+        # I could not find where the code stores params in selection with render
+        # prefix - maybe it in some `render` method where it should not be
+        # Of course it is ugly, terrible and should be removed!
+        selection_tool = context.getPortalObject().portal_selections
+        selection_tool.getSelectionFor(selection_name, REQUEST)
+        selection_tool.setSelectionParamsFor(selection_name, report_form_params)
+        selection_tool.setSelectionColumns(selection_name, report_item.selection_columns)
+
+      if report_item.selection_columns:
+        report_form_params.update(selection_columns=report_item.selection_columns)
+      if report_item.selection_sort_order:
+        report_form_params.update(selection_sort_order=report_item.selection_sort_order)
+
+      # Report section is just a wrapper around form thus we render it right
+      # we keep traversed_document because its Portal Type Class should be 
+      # addressable by the user = have actions (object_view) attached to it
+      # BUT! when Report Section defines `path` that is the new context for
+      # form rendering and subsequent searches...
+      renderForm(traversed_document if not report_item.path else report_context,
+                 getattr(report_context, report_item.getFormId()),
+                 report_result,
+                 key_prefix=report_prefix,
+                 selection_params=report_form_params)  # used to be only report_item.selection_params
+      # Report Title is important since there are more section on report page
+      # but often they render the same form with different data so we need to
+      # distinguish by the title at least.
+      report_result['title'] = report_title
       report_result_list.append(report_result)
-
     response_dict['report_section_list'] = report_result_list
 
 # XXX form action update, etc
@@ -593,6 +932,7 @@ def renderRawField(field):
 
 
 def renderFormDefinition(form, response_dict):
+  """Form "definition" is configurable in Zope admin: Form -> Order."""
   group_list = []
   for group in form.Form_getGroupTitleAndId():
 
@@ -711,19 +1051,23 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
     action_dict = {}
   #   result_dict['_relative_url'] = traversed_document.getRelativeUrl()
     result_dict['title'] = traversed_document.getTitle()
-  
+
     # Add a link to the portal type if possible
     if not is_portal:
-      result_dict['_links']['type'] = {
-        "href": default_document_uri_template % {
-          "root_url": site_root.absolute_url(),
-          "relative_url": portal.portal_types[traversed_document.getPortalType()]\
-                            .getRelativeUrl(), 
-          "script_id": script.id
-        },
-        "name": Base_translateString(traversed_document.getPortalType())
-      }
-      
+      # traversed_document should always have its Portal Type in ERP5 Portal Types
+      # thus attached actions to it so it is viewable
+      document_type_name = traversed_document.getPortalType()
+      document_type = getattr(portal.portal_types, document_type_name, None)
+      if document_type is not None:
+        result_dict['_links']['type'] = {
+          "href": default_document_uri_template % {
+            "root_url": site_root.absolute_url(),
+            "relative_url": document_type.getRelativeUrl(),
+            "script_id": script.id
+          },
+          "name": Base_translateString(traversed_document.getPortalType())
+        }
+
     # Return info about container
     if not is_portal:
       container = traversed_document.getParentValue()
@@ -834,7 +1178,6 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
       # renderer_form = traversed_document.restrictedTraverse(form_id, None)
       # XXX Proxy field are not correctly handled in traversed_document of web site
       renderer_form = getattr(traversed_document, form_id)
-  #     traversed_document.log(form_id)
       if (renderer_form is not None):
         embedded_dict = {
           '_links': {
@@ -844,19 +1187,31 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
           }
         }
         # Put all query parameters (?reset:int=1&workflow_action=start_action) in request to mimic usual form display
+        query_param_dict = {}
         query_split = embedded_url.split('?', 1)
         if len(query_split) == 2:
           for query_parameter in query_split[1].split("&"):
-            query_key, query_value = query_parameter.split("=")
-            REQUEST.set(query_key, query_value)
-  
+            query_key, query_value = query_parameter.split('=')
+            # often + is used instead of %20 so we replace for space here
+            query_param_dict[query_key] = query_value.replace("+", " ")
+
+        # set URL params into REQUEST (just like it was sent by form)
+        for query_key, query_value in query_param_dict.items():
+          REQUEST.set(query_key, query_value)
+
+        # unfortunatelly some people use Scripts as targets for Workflow
+        # transactions - thus we need to check and mitigate
+        if "Script" in renderer_form.meta_type:
+          # we suppose that the script takes only what is given in the URL params
+          return renderer_form(**query_param_dict)
+
         renderForm(traversed_document, renderer_form, embedded_dict)
         result_dict['_embedded'] = {
           '_view': embedded_dict
           # embedded_action_key: embedded_dict
         }
   #       result_dict['_links']["_view"] = {"href": embedded_url}
-  
+
         # Include properties in document JSON
         # XXX Extract from renderer form?
         """
@@ -938,7 +1293,7 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
   
     else:
       traversed_document_portal_type = traversed_document.getPortalType()
-      if traversed_document_portal_type == "ERP5 Form":
+      if traversed_document_portal_type in ("ERP5 Form", "ERP5 Report"):
         renderFormDefinition(traversed_document, result_dict)
         response.setHeader("Cache-Control", "private, max-age=1800")
         response.setHeader("Vary", "Cookie,Authorization,Accept-Encoding")
@@ -961,150 +1316,220 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
               "template": True
             }
           }
-  
+
     # Define document action
     if action_dict:
       result_dict['_actions'] = action_dict
-  
-  
+
   elif mode == 'search':
     #################################################
     # Portal catalog search
     #
     # Possible call arguments example:
     #  form_relative_url: portal_skins/erp5_web/WebSite_view/listbox
-    #  list_method: objectValues                      (Script providing listing)
+    #  list_method: objectValues                      (Script providing items)
     #  default_param_json: <base64 encoded JSON>      (Additional search params)
     #  query: <str>                                   (term for fulltext search)
     #  select_list: ['int_index', 'id', 'title', ...] (column names to select)
     #  limit: [15, 16]                                (begin_index, num_records)
     #  local_roles: TODO
+    #
+    # Default Param JSON contains
+    #  portal_type: list of Portal Types to include (singular form matches the
+    #                                                catalog column name)
+    #
+    # Discussion:
+    #
+    #  Why you didn't use ListBoxRendererLine?
+    #  > Method 'search' is used for getting related objects as well which are
+    #  > not backed up by a ListBox thus the value resolution would have to be
+    #  > there anyway. It is better to use one code for all in this case.
     #################################################
     if REQUEST.other['method'] != "GET":
       response.setStatus(405)
       return ""
-  
+
+    # in case we have custom list method
+    catalog_kw = {}
+
     # hardcoded responses for site and portal objects (which are not Documents!)
+    # we let the flow to continue because the result of a list_method call can
+    # be similar - they can in practice return anything
     if query == "__root__":
-      sql_list = [site_root]
+      search_result_iterable = [site_root]
     elif query == "__portal__":
-      sql_list = [portal]
+      search_result_iterable = [portal]
     else:
+      # otherwise gather kwargs for list_method and get whatever result it gives
+      callable_list_method = portal.portal_catalog
+      if list_method:
+        callable_list_method = getattr(traversed_document, list_method)
+
       catalog_kw = {
         "local_roles": local_roles,
         "limit": limit,
-        "sort_on": ()  # default is empty tuple
+        "sort_on": ()  # default is an empty tuple
       }
       if default_param_json is not None:
-        catalog_kw.update(byteify(json.loads(urlsafe_b64decode(default_param_json))))
+        catalog_kw.update(
+          ensure_deserialized(
+            byteify(
+              json.loads(urlsafe_b64decode(default_param_json)))))
       if query:
         catalog_kw["full_text"] = query
+
       if sort_on is not None:
+        def parse_sort_on(raw_string):
+          """Turn JSON serialized array into a tuple (col_name, order)."""
+          sort_col, sort_order = json.loads(raw_string)
+          sort_col, sort_order = byteify(sort_col), byteify(sort_order)
+          # JIO keeps sort order as whole word 'ascending' resp. 'descending'
+          if sort_order.lower().startswith("asc"):
+            sort_order = "ASC"
+          elif sort_order.lower().startswith("desc"):
+            sort_order = "DESC"
+          else:
+            # should raise an ValueError instead
+            context.log('Wrong sort order "{}" in {}! It must start with "asc" or "desc"'.format(sort_order, form_relative_url),
+                        level=200)  # error
+          return (sort_col, sort_order)
+
         if isinstance(sort_on, list):
-          catalog_kw['sort_on'] = tuple((byteify(sort_col), byteify(sort_order))
-                                         for sort_col, sort_order in map(json.loads, sort_on))
+          # sort_on argument is always a list of tuples(col_name, order)
+          catalog_kw['sort_on'] = list(map(parse_sort_on, sort_on))
         else:
-          sort_col, sort_order = json.loads(sort_on)
-          catalog_kw['sort_on'] = ((byteify(sort_col), byteify(sort_order)), )
-
-      if (list_method is None):
-        callable_list_method = portal.portal_catalog
-      else:
-        callable_list_method = getattr(traversed_document, list_method)
-
-      sql_list = callable_list_method(**catalog_kw)
-
-    result_list = []  # returned "content" of the search
-
-    # Cast to list if only one element is provided
-    editable_field_dict = {}
+          catalog_kw['sort_on'] = [parse_sort_on(sort_on), ]
+
+      # Some search scripts impertinently grab their arguments from REQUEST
+      # instead of being nice and specify them as their input parameters.
+      #
+      # We expect that wise and mighty ListBox did copy all form field values
+      # from its REQUEST into `default_param_json` so we can put them back.
+      #
+      # XXX Kato: Seems that current scripts are behaving nicely (using only
+      # specified input parameters). In case some list_method does not work
+      # this is the first place to try to uncomment.
+      #
+      # for k, v in catalog_kw.items():
+      #   REQUEST.set(k, v)
+
+      search_result_iterable = callable_list_method(**catalog_kw)
+
+        # Cast to list if only one element is provided
     if select_list is None:
       select_list = []
     elif same_type(select_list, ""):
       select_list = [select_list]
-  
-    if select_list:
-      if (form_relative_url is not None):
-        listbox_field = portal.restrictedTraverse(form_relative_url)
-        listbox_field_id = listbox_field.id
-        # XXX Proxy field are not correctly handled in traversed_document of web site
-        listbox_form = getattr(traversed_document, listbox_field.aq_parent.id)
-        for select in select_list:
-          # See Listbox.py getValueList --> getEditableField & getColumnAliasList method
-          tmp = select.replace('.', '_')
-          if listbox_form.has_field("%s_%s" % (listbox_field_id, tmp), include_disabled=1):
-            editable_field_dict[select] = listbox_form.get_field("%s_%s" % (listbox_field_id, tmp), include_disabled=1)
-  
+
+    # extract form field definition into `editable_field_dict`
+    editable_field_dict = {}
+    listbox_form = None
+    listbox_field_id = None
+    if form_relative_url is not None:
+      listbox_field = portal.restrictedTraverse(form_relative_url)
+      listbox_field_id = listbox_field.id
+      # XXX Proxy field are not correctly handled in traversed_document of web site
+      listbox_form = getattr(traversed_document, listbox_field.aq_parent.id)
+      for select in select_list:
+        # See Listbox.py getValueList --> getEditableField & getColumnAliasList method
+        # In short: there are Form Field definitions which names start with
+        # matching ListBox name - those are template fields to be rendered in
+        # cells with actual values defined by row and column
+        field_name = "{}_{}".format(listbox_field_id, select.replace(".", "_"))
+        if listbox_form.has_field(field_name, include_disabled=1):
+          editable_field_dict[select] = listbox_form.get_field(field_name, include_disabled=1)
+
     # handle the case when list-scripts are ignoring `limit` - paginate for them
     if limit is not None and isinstance(limit, (tuple, list)):
       start, num_items = map(int, limit)
-      if len(sql_list) <= num_items:
+      if len(search_result_iterable) <= num_items:
         # the limit was most likely taken into account thus we don't need to slice
-        start, num_items = 0, len(sql_list)
+        start, num_items = 0, len(search_result_iterable)
     else:
-      start, num_items = 0, len(sql_list)
+      start, num_items = 0, len(search_result_iterable)
+
+    contents_list = []  # resolved fields from the search result
 
-    for document_index, sql_document in enumerate(sql_list):
-      if document_index < start:
+    # now fill in `contents_list` with actual information
+    # beware that search_result_iterable can hide anything inside!
+    for result_index, search_result in enumerate(search_result_iterable):
+      # skip documents out of `limit`
+      if result_index < start:
         continue
-      if document_index >= start + num_items:
+      if result_index >= start + num_items:
         break
 
-      try:
-        document = sql_document.getObject()
-      except AttributeError:
-        # XXX ERP5 Site is not an ERP5 document
-        document = sql_document
-      document_uid = sql_document.uid
-      document_result = {
+      contents_uid, contents_relative_url, property_getter, property_hasser = \
+        object_uids_and_accessors(search_result, result_index, traversed_document)
+
+      # this dict will hold all resolved values
+      contents_item = {
+        # _links.self.href is mandatory for JIO so it can create reference to the
+        # (listbox) item alone
         '_links': {
           'self': {
             "href": default_document_uri_template % {
               "root_url": site_root.absolute_url(),
-              # XXX ERP5 Site is not an ERP5 document
-              "relative_url": getRealRelativeUrl(document) or document.getId(), 
+              "relative_url": contents_relative_url,
               "script_id": script.id
             },
           },
         }
       }
-      if editable_field_dict:
-        document_result['listbox_uid:list'] = {
+      # ERP5 stores&send the list of editable elements in a hidden field called
+      # only database results can be editable so it belongs here
+      if editable_field_dict and listbox_field_id:
+        contents_item['listbox_uid:list'] = {
           'key': "%s_uid:list" % listbox_field_id,
-          'value': document_uid
+          'value': contents_uid
         }
+
       for select in select_list:
+        # every `select` can have a template field or be just a exotic getter for a value
         if editable_field_dict.has_key(select):
-          REQUEST.set('cell', sql_document)
-  
-          if ('default' in editable_field_dict[select].tales):
-            tmp_value = None
-          else:
-            tmp_value = getProtectedProperty(document, select)
-  
-          property_value = renderField(
-            traversed_document, editable_field_dict[select], form, tmp_value,
-            key='field_%s_%s' % (editable_field_dict[select].id, document_uid))
+          # cell has a Form Field template thus render it using the field
+          # fields are nice because they are standard
+          REQUEST.set('cell', search_result)
+          # if default value is given by evaluating Tales expression then we only
+          # put "cell" to request (expected by tales) and let the field evaluate
+          default_field_value = None
+          if getattr(editable_field_dict[select].tales, "default", "") == "":
+            # if there is no tales expr (or is empty) we extract the value from search result
+            default_field_value = getProtectedProperty(search_result, select)
+
+          contents_item[select] = renderField(
+            traversed_document,
+            editable_field_dict[select],
+            listbox_form,
+            value=default_field_value,
+            key='field_%s_%s' % (editable_field_dict[select].id, contents_uid))
+
           REQUEST.other.pop('cell', None)
         else:
-          property_value = getProtectedProperty(document, select)
-        if property_value is not None:
-          if same_type(property_value, DateTime()):
-            # Serialize DateTime
-            property_value = property_value.rfc822()
-          elif isinstance(property_value, datetime.date):
-            property_value = formatdate(time.mktime(property_value.timetuple()))
-          elif getattr(property_value, 'translate', None) is not None:
-            property_value = "%s" % property_value
-          document_result[select] = property_value
-      result_list.append(document_result)
-    result_dict['_embedded'] = {"contents": result_list}
-  
-    result_dict['_query'] = query
-    result_dict['_local_roles'] = local_roles
-    result_dict['_limit'] = limit
-    result_dict['_select_list'] = select_list
-  
+          # most of the complicated magic happens here - we need to resolve field names
+          # given search_result. This name can unfortunately mean almost anything from
+          # a key name to Python Script with variable number of input parameters.
+          contents_item[select] = resolve_field(search_result, select, property_getter, property_hasser)
+
+      # endfor select
+      contents_list.append(contents_item)
+
+    result_dict.update({
+      '_query': query,
+      '_local_roles': local_roles,
+      '_limit': limit,
+      '_select_list': select_list,
+      '_embedded': {
+        'contents': contents_list
+      }
+    })
+
+    # We should cleanup the selection if it exists in catalog params BUT
+    # we cannot because it requires escalated Permission.'modifyPortal' so
+    # the correct solution would be to ReportSection.popReport but unfortunately
+    # we don't have it anymore because we are asynchronous
+    return result_dict
+
   elif mode == 'form':
     #################################################
     # Calculate form value
@@ -1212,6 +1637,7 @@ hateoas = calculateHateoas(is_portal=temp_is_portal, is_site_root=temp_is_site_r
                            restricted=restricted, list_method=list_method,
                            default_param_json=default_param_json,
                            form_relative_url=form_relative_url)
+
 if hateoas == "":
   return hateoas
 else: