Merge remote-tracking branch 'upstream/master' into zope4py2

bt5/erp5_accounting/SkinTemplateItem/portal_skins/erp5_accounting/AccountingTransaction_guessGroupedLines.py

@@ -14,8 +14,7 @@ lines_per_node = {}
 portal = context.getPortalObject()
 
 allow_grouping_with_different_quantity = portal.portal_preferences.getPreference(
-                                         'preferred_grouping_with_different_quantities', 0)
-
+  'preferred_grouping_with_different_quantities', 0)
 
 accounting_transaction_line_value_list = []
 if accounting_transaction_line_uid_list is None:
@@ -25,56 +24,71 @@ if accounting_transaction_line_uid_list is None:
                           accounting_transaction.getUid() != context.getUid():
       continue
     for line in accounting_transaction.getMovementList(
-                            portal.getPortalAccountingMovementTypeList()):
+        portal.getPortalAccountingMovementTypeList()):
       if line.getGroupingReference():
         continue
       accounting_transaction_line_value_list.append(line)
 else:
   if accounting_transaction_line_uid_list:
     accounting_transaction_line_value_list = [
-      brain.getObject() for brain in portal.portal_catalog(uid=accounting_transaction_line_uid_list)]
+      brain.getObject() for brain in portal.portal_catalog(
+        uid=accounting_transaction_line_uid_list)
+    ]
+
 
 for line in accounting_transaction_line_value_list:
-  accounting_transaction = line.getParentValue()
-  if accounting_transaction.AccountingTransaction_isSourceView():
-    section_relative_url = None
-    source_section = line.getSourceSectionValue(portal_type='Organisation')
-    if source_section is not None:
-      source_section = \
-        source_section.Organisation_getMappingRelatedOrganisation()
-      section_relative_url = source_section.getRelativeUrl()
+  # source
+  section_relative_url = None
+  source_section = line.getSourceSectionValue(portal_type='Organisation')
+  if source_section is not None:
+    source_section = \
+      source_section.Organisation_getMappingRelatedOrganisation()
+    section_relative_url = source_section.getRelativeUrl()
+  lines_per_node.setdefault(
+      (
+        line.getSource(portal_type='Account'),
+        section_relative_url,
+        line.getDestinationSection(),
+        line.AccountingTransactionLine_getGroupingExtraParameterList(
+          source=True),
+      ), []).append(
+        dict(
+          total_price=line.getSourceInventoriatedTotalAssetPrice() or 0,
+          date=line.getStartDate(),
+          path=line.getRelativeUrl()))
+
+  # destination
+  section_relative_url = None
+  destination_section = line.getDestinationSectionValue(
+    portal_type='Organisation')
+  if destination_section is not None:
+    destination_section = \
+      destination_section.Organisation_getMappingRelatedOrganisation()
+    section_relative_url = destination_section.getRelativeUrl()
+  lines_per_node.setdefault(
+    (
+      line.getDestination(portal_type='Account'),
+      section_relative_url,
+      line.getSourceSection(),
+      line.AccountingTransactionLine_getGroupingExtraParameterList(
+        source=False),
+    ), []).append(
+      dict(
+        total_price=line.getDestinationInventoriatedTotalAssetPrice() or 0,
+        date=line.getStopDate(),
+        path=line.getRelativeUrl()))
 
-    lines_per_node.setdefault(
-                  (line.getSource(portal_type='Account'),
-                   section_relative_url,
-                   line.getDestinationSection(),
-                   line.AccountingTransactionLine_getGroupingExtraParameterList(source=True),
-                   ), []).append(
-      dict(total_price=line.getSourceInventoriatedTotalAssetPrice() or 0,
-           date=line.getStartDate(),
-           path=line.getRelativeUrl()))
-  else:
-    section_relative_url = None
-    destination_section = line.getDestinationSectionValue(
-                                    portal_type='Organisation')
-    if destination_section is not None:
-      destination_section = \
-        destination_section.Organisation_getMappingRelatedOrganisation()
-      section_relative_url = destination_section.getRelativeUrl()
-    lines_per_node.setdefault(
-              (line.getDestination(portal_type='Account'),
-               section_relative_url,
-               line.getSourceSection(),
-               line.AccountingTransactionLine_getGroupingExtraParameterList(source=False),
-               ), []).append(
-    dict(total_price=line.getDestinationInventoriatedTotalAssetPrice() or 0,
-         date=line.getStopDate(),
-         path=line.getRelativeUrl()))
 
-changed_line_list = []
-for (node, section, mirror_section, _), line_info_list in lines_per_node.items():
+changed_line_set = set()
+for (
+    node,
+    section,
+    mirror_section,
+    _,
+), line_info_list in lines_per_node.items():
   if node is None:
     continue
+
   # get the currency rounding for this section, with a fallback that something that would
   # allow grouping in case precision is not defined.
   currency_precision = 5
@@ -87,10 +101,10 @@ for (node, section, mirror_section, _), line_info_list in lines_per_node.items()
     # we should include mirror node in the id_group, but this would reset
     # id generators and generate grouping references that were already used.
     id_group = ('grouping_reference', node, section, mirror_section)
-    previous_default = context.portal_ids.getLastGeneratedId(id_group=id_group, default=0)
-    grouping_reference = portal.portal_ids.generateNewId(id_generator='uid',
-                                                  id_group=id_group,
-                                                  default=previous_default + 1)
+    previous_default = context.portal_ids.getLastGeneratedId(
+      id_group=id_group, default=0)
+    grouping_reference = portal.portal_ids.generateNewId(
+      id_generator='uid', id_group=id_group, default=previous_default + 1)
 
     # convert from int to letters
     string_reference = int2letter(grouping_reference)
@@ -99,11 +113,14 @@ for (node, section, mirror_section, _), line_info_list in lines_per_node.items()
     date = max([line['date'] for line in line_info_list])
 
     for line in line_info_list:
+      if line['path'] in changed_line_set:
+        continue
       line_obj = portal.restrictedTraverse(line['path'])
       assert not line_obj.getGroupingReference(), line
       line_obj.setGroupingReference(string_reference)
       line_obj.setGroupingDate(date)
-      line_obj.reindexObject(activate_kw=dict(tag='accounting_grouping_reference'))
-      changed_line_list.append(line['path'])
+      line_obj.reindexObject(
+        activate_kw=dict(tag='accounting_grouping_reference'))
+      changed_line_set.add(line['path'])
 
-return changed_line_list
+return changed_line_set

bt5/erp5_accounting/TestTemplateItem/portal_components/test.erp5.testAccounting.py

@@ -6169,6 +6169,153 @@ class TestInternalInvoiceTransaction(AccountingTestCase):
     self.assertEqual(stat_internal_transaction.destination_credit, 1) # line1
     self.assertEqual(stat_internal_transaction.destination_asset_credit, 0.22) # line1
 
+  def test_grouping_reference_both_sides(self):
+    # Group together lines from two internal invoices:
+    #
+    # | Source Account | Debit | Credit | Grouping | Destination Account | Debit | Credit | Grouping |
+    # |----------------|-------|--------|----------|---------------------|-------|--------|----------|
+    # | receivable     | 10    |        | A        |                     |       |        |          |
+    # | sales          |       | 10     |          | purchase            | 10    |        |          |
+    # |                |       |        |          | payable             |       | 10     | B        |
+    # and
+    # | Source Account | Debit | Credit | Grouping | Destination Account | Debit | Credit | Grouping |
+    # |----------------|-------|--------|----------|---------------------|-------|--------|----------|
+    # | sales          | 10    |        |          | purchase            |       | 10     |          |
+    # | receivable     |       | 10     | A        |                     |       |        |          |
+    # |                |       |        |          | payable             | 10    |        | B        |
+    # This example does not really make sense from usage of internal invoices, because we usually
+    # use the same line for receivable and purchase in such a case, but it reproduces a case that did
+    # not group automatically.
+
+    internal_invoice1 = self.portal.accounting_module.newContent(
+      portal_type='Internal Invoice Transaction',
+      title='internal_invoice1',
+      source_section_value=self.section,
+      destination_section_value=self.main_section,
+      start_date=DateTime(2015, 1, 1),
+      created_by_builder=True,
+    )
+    # start before creating lines, because we don't want our lines to
+    # be initialized with mirror accounts.
+    internal_invoice1.start()
+    internal_invoice1.newContent(
+      id='line_a',
+      source_value=self.portal.account_module.receivable,
+      source_debit=10,
+    )
+    internal_invoice1.newContent(
+      source_value=self.portal.account_module.goods_sales,
+      destination_value=self.portal.account_module.goods_purchase,
+      source_credit=10,
+    )
+    internal_invoice1.newContent(
+      id='line_b',
+      destination_value=self.portal.account_module.payable,
+      destination_credit=10,
+    )
+    internal_invoice1.stop()
+    self.tic()
+
+    internal_invoice2 = self.portal.accounting_module.newContent(
+      portal_type='Internal Invoice Transaction',
+      title='internal_invoice2',
+      source_section_value=self.section,
+      destination_section_value=self.main_section,
+      start_date=DateTime(2015, 1, 1),
+      causality_value=internal_invoice1,
+      created_by_builder=True,
+    )
+    internal_invoice2.start()
+    internal_invoice2.newContent(
+      source_value=self.portal.account_module.goods_sales,
+      destination_value=self.portal.account_module.goods_purchase,
+      source_debit=10,
+    )
+    internal_invoice2.newContent(
+      id='line_a',
+      source_value=self.portal.account_module.receivable,
+      source_credit=10,
+    )
+    internal_invoice2.newContent(
+      id='line_b',
+      destination_value=self.portal.account_module.payable,
+      destination_debit=10,
+    )
+    internal_invoice2.stop()
+    self.tic()
+    self.assertTrue(internal_invoice1.line_a.getGroupingReference())
+    self.assertEqual(
+      internal_invoice1.line_a.getGroupingReference(),
+      internal_invoice2.line_a.getGroupingReference(),
+    )
+    self.assertTrue(internal_invoice1.line_b.getGroupingReference())
+    self.assertEqual(
+      internal_invoice1.line_b.getGroupingReference(),
+      internal_invoice2.line_b.getGroupingReference(),
+    )
+
+  def test_grouping_reference_no_group_when_mirror_accounts_are_different(self):
+    # Does not together lines from two internal invoices:
+    #
+    # | Source Account | Debit | Credit | Grouping | Destination Account | Debit | Credit | Grouping |
+    # |----------------|-------|--------|----------|---------------------|-------|--------|----------|
+    # | receivable     | 10    |        |  no  ->  | payable             |       | 10     |          |
+    # | sales          |       | 10     |          | purchase            | 10    |        |          |
+    # and
+    # | Source Account | Debit | Credit | Grouping | Destination Account | Debit | Credit | Grouping |
+    # |----------------|-------|--------|----------|---------------------|-------|--------|----------|
+    # | sales          | 10    |        |          | payable             |       | 10     |          |
+    # | receivable     |       | 10     |  no  ->  | purchase            | 10    |        |          |
+
+    internal_invoice1 = self.portal.accounting_module.newContent(
+      portal_type='Internal Invoice Transaction',
+      title='internal_invoice1',
+      source_section_value=self.section,
+      destination_section_value=self.main_section,
+      start_date=DateTime(2015, 1, 1),
+      created_by_builder=True,
+    )
+    internal_invoice1.start()
+    internal_invoice1.newContent(
+      id='not_grouped',
+      source_value=self.portal.account_module.receivable,
+      destination_value=self.portal.account_module.payable,
+      source_debit=10,
+    )
+    internal_invoice1.newContent(
+      source_value=self.portal.account_module.goods_sales,
+      destination_value=self.portal.account_module.goods_purchase,
+      source_credit=10,
+    )
+    internal_invoice1.stop()
+    self.tic()
+
+    internal_invoice2 = self.portal.accounting_module.newContent(
+      portal_type='Internal Invoice Transaction',
+      title='internal_invoice2',
+      source_section_value=self.section,
+      destination_section_value=self.main_section,
+      start_date=DateTime(2015, 1, 1),
+      causality_value=internal_invoice1,
+      created_by_builder=True,
+    )
+    internal_invoice2.start()
+    internal_invoice2.newContent(
+      source_value=self.portal.account_module.goods_sales,
+      destination_value=self.portal.account_module.payable,
+      source_debit=10,
+    )
+    internal_invoice2.newContent(
+      id='not_grouped',
+      source_value=self.portal.account_module.receivable,
+      destination_value=self.portal.account_module.goods_purchase,
+      source_debit=10,
+    )
+    internal_invoice2.stop()
+    self.tic()
+    self.assertFalse(internal_invoice1.not_grouped.getGroupingReference())
+    self.assertFalse(internal_invoice1.not_grouped.getGroupingReference())
+
 
 class TestAccountingAlarms(AccountingTestCase):
   def test_check_payable_receivable_account_grouped(self):

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_style/ERP5Document_getHateoas.py

@@ -1546,10 +1546,11 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
           'href': '%s' % view_action['url'],
           'name': view_action['id'],
           'icon': view_action['icon'],
-          'title': Base_translateString(
-              translateWorklistActionName(view_action['title'])
-              if 'worklist_id' in view_action
-              else view_action['title']),
+          'title': translateWorklistActionName(
+            view_action['title']
+            ) if 'worklist_id' in view_action else Base_translateString(
+              view_action['title']
+            ),
         })
 
         global_action_type = ("view", "workflow", "object_new_content_action",

bt5/erp5_hal_json_style/TestTemplateItem/portal_components/test.erp5.testHalJsonStyle.py

@@ -65,6 +65,7 @@ def simulate(script_id, params_string, code_string):
       try:
         result = f(self, *args, **kw)
       finally:
+        transaction.abort()
         if script_id in self.portal.portal_skins.custom.objectIds():
           self.portal.portal_skins.custom.manage_delObjects(script_id)
         transaction.commit()
@@ -159,18 +160,12 @@ def replace_request(new_request, context):
 
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 
-#####################################################
-# Base_getRequestHeader
-#####################################################
+
 class ERP5HALJSONStyleSkinsMixin(ERP5TypeTestCase):
   def afterSetUp(self):
     self.login()
     wipeFolder(self.portal.foo_module, commit=False)
 
-
-  def beforeTearDown(self):
-    transaction.abort()
-
   def generateNewId(self):
     return "%sö" % self.portal.portal_ids.generateNewId(
                                      id_group=('erp5_hal_json_style_test'))
@@ -184,6 +179,10 @@ class ERP5HALJSONStyleSkinsMixin(ERP5TypeTestCase):
     )
     return foo
 
+
+#####################################################
+# Base_getRequestHeader
+#####################################################
 class TestBase_getRequestHeader(ERP5HALJSONStyleSkinsMixin):
   @changeSkin('Hal')
   def test_getRequestHeader_REQUEST_disallowed(self):
@@ -2437,8 +2436,9 @@ class TestERP5Person_getHateoas_mode_search(ERP5HALJSONStyleSkinsMixin):
     self.tic()
 
   def beforeTearDown(self):
+    super(TestERP5Person_getHateoas_mode_search, self).beforeTearDown()
     self.portal.person_module.deleteContent(self.person.getId())
-
+    self.tic()
 
   @simulate('Base_getRequestUrl', '*args, **kwargs', 'return "http://example.org/bar"')
   @simulate('Base_getRequestHeader', '*args, **kwargs', 'return "application/hal+json"')
@@ -2843,6 +2843,16 @@ if translation_service is not None :\n\
     pass\n\
 return msg"
 
+  def afterSetUp(self):
+    super(TestERP5Document_getHateoas_translation, self).afterSetUp()
+    self.portal.Base_createUITestLanguages()
+    param_dict = [
+      { 'message': 'Title', 'translation': 'biaoti', 'language': 'wo'},
+      { 'message': 'Draft To Validate', 'translation': 'daiyanzhen', 'language': 'wo'},
+      { 'message': 'Foo', 'translation': 'Foo_zhongwen', 'language': 'wo'}]
+    for tmp in param_dict:
+      self.portal.Base_addUITestTranslation(message = tmp['message'], translation = tmp['translation'], language = tmp['language'])
+
   @simulate('Base_getRequestUrl', '*args, **kwargs',
       'return "http://example.org/bar"')
   @simulate('Base_getRequestHeader', '*args, **kwargs',
@@ -2852,13 +2862,6 @@ return msg"
 
   @changeSkin('Hal')
   def test_getHateoasBulk_default_view_translation(self):
-    self.portal.Base_createUITestLanguages()
-    param_dict = [
-      { 'message': 'Title', 'translation': 'biaoti', 'language': 'wo'},
-      { 'message': 'Draft To Validate', 'translation': 'daiyanzhen', 'language': 'wo'},
-      { 'message': 'Foo', 'translation': 'Foo_zhongwen', 'language': 'wo'}]
-    for tmp in param_dict:
-      self.portal.Base_addUITestTranslation(message = tmp['message'], translation = tmp['translation'], language = tmp['language'])
     document = self._makeDocument()
     fake_request = do_fake_request("POST")
 
@@ -2942,9 +2945,8 @@ return msg"
     # worklists with the actual count of document
     fake_request = do_fake_request("GET")
 
-    default_gettext = self.portal.Localizer.erp5_ui.gettext
     def gettext(message, **kw):
-      return default_gettext(message, **kw)
+      return kw.get('default', message)
 
     with mock.patch.object(self.portal.Localizer.erp5_ui.__class__, 'gettext', side_effect=gettext) as gettext_mock:
       self.portal.web_site_module.hateoas.ERP5Document_getHateoas(

bt5/erp5_stripe/bt/test_dependency_list

+erp5_full_text_mroonga_catalog
\ No newline at end of file

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BusinessTemplate_getPythonSourceCodeMessageList.py

@@ -81,16 +81,8 @@ def checkComponent(component_instance):
   code = component_instance.getTextContent()
   if six.PY2:
     code = unicode(code, 'utf8')
-  data = {'code': code}
-  try:
-    check_result_json = portal.ERP5Site_checkPythonSourceCodeAsJSON(data)
-  except Exception:
-    # pylint sometimes raises on the first attempt at importing modules, but
-    # may succeed on the second try (probably because of incomplete cleanup
-    # of partially imported moduled). We are not interested in pylint issues,
-    # we are interested in our code's issues, so give it one more try.
-    check_result_json = portal.ERP5Site_checkPythonSourceCodeAsJSON(data)
-  for annotation in json.loads(check_result_json)['annotations']:
+  for annotation in json.loads(portal.ERP5Site_checkPythonSourceCodeAsJSON(
+        {'code': code}))['annotations']:
     annotation['component_path'] = component_relative_url
     line_list.append(
       Message(

product/ERP5Type/patches/pylint.py

@@ -372,6 +372,25 @@ else:
         return m
     MANAGER.register_transform(Module, wendelin_transform, lambda node: node.name == 'wendelin')
 
+# prevent a crash with cryptography:
+#   File "develop-eggs/astroid-1.3.8+slapospatched001-py2.7.egg/astroid/raw_building.py", line 360, in _set_proxied
+#      return _CONST_PROXY[const.value.__class__]
+#  KeyError: <type 'CompiledFFI'>
+import cryptography.hazmat.bindings._openssl
+_register_module_extender_from_live_module(
+  'cryptography.hazmat.bindings._openssl',
+  cryptography.hazmat.bindings._openssl)
+
+
+try:
+  import xmlsec
+except ImportError:
+  pass
+else:
+  _register_module_extender_from_live_module('xmlsec', xmlsec)
+  _register_module_extender_from_live_module('xmlsec.tree', xmlsec.tree)
+  _register_module_extender_from_live_module('xmlsec.template', xmlsec.template)
+
 # Properly search for namespace packages: original astroid (as of 1.3.8) only
 # checks at top-level and it doesn't work for Shared.DC.ZRDB (defined in
 # Products.ZSQLMethods; Shared and Shared.DC being a namespace package defined

product/ERP5Type/tests/testDynamicClassGeneration.py

@@ -2285,12 +2285,39 @@ class FooBar(ValidationFailed):
   def __init__(self, *args, **kw):
     super(FooBar, self).__init__(*args, **kw)
 
+# Test for various pylint fixes
+
 # Transforms for Zope which should ideally be upstream'ed
 from AccessControl.PermissionRole import rolesForPermissionOn, PermissionRole, imPermissionRole, _what_not_even_god_should_do # pylint: disable=unused-import
 
 # Monkey patch of astroid 1.3.8: it raised 'no-name-in-module' because
 # Shared.DC was not considered a namespace package
 from Shared.DC.ZRDB.Results import Results # pylint: disable=unused-import
+
+import lxml.etree
+lxml.etree.Element('test')
+
+from BTrees.OOBTree import OOBTree
+OOBTree()
+
+from cryptography.hazmat.primitives.asymmetric import rsa
+rsa.generate_private_key(
+  public_exponent=65537,
+  key_size=2048,
+).public_key()
+
+def xmlsec_decrypt():
+  # from https://xmlsec.readthedocs.io/en/stable/examples.html#decrypt
+  import xmlsec
+  manager = xmlsec.KeysManager()
+  key = xmlsec.Key.from_file('rsakey.pem', xmlsec.constants.KeyDataFormatPem)
+  manager.add_key(key)
+  enc_ctx = xmlsec.EncryptionContext(manager)
+  root = lxml.etree.parse("enc1-res.xml").getroot()
+  enc_data = xmlsec.tree.find_child(root, "EncryptedData", xmlsec.constants.EncNs)
+  decrypted = enc_ctx.decrypt(enc_data)
+  print(lxml.etree.tostring(decrypted))
+
 """ % (dict(namespace=namespace,
             reference1=imported_reference1,
             module2=imported_module2,