web_renderjs_ui: use lxml to extract data-i18n messages
The previous regular expression based approach sometimes could not extract message properly. Using xml parser simplify code and fix several messages that were not extracted properly, like messages containing ", [] or {} This also fix some problems when looking for messages sources: - archived web pages were sometimes used instead of published ones - messages from gadgets implemented as page templates/OFS files were not extracted. A few more unit tests for the scripts involved in this process are added.
Showing
"""Returns the `text_content` that should be set on the translation data script for this RJS website. | """Returns the `text_content` that should be set on the translation data script for this RJS website. | ||
""" | """ | ||
import re | |||
import json | import json | ||
portal = context.getPortalObject() | portal = context.getPortalObject() | ||
... | @@ -11,24 +10,33 @@ Base_translateString = context.Base_translateString | ... | @@ -11,24 +10,33 @@ Base_translateString = context.Base_translateString |
# <span data-18n="The message">The message</span> | # <span data-18n="The message">The message</span> | ||
# or in comments, like this: | # or in comments, like this: | ||
# <!-- data-i18n="The message" --> | # <!-- data-i18n="The message" --> | ||
attribute_filter_re = re.compile(r"""(data-i18n)=["']?((?:.(?!["']?\s+(?:\S+)=|[>"']))+.)["']?""") | |||
translatable_message_set = set([]) | translatable_message_set = set([]) | ||
for web_page in portal.web_page_module.searchFolder(portal_type='Web Page', | |||
reference=context.Base_getTranslationSourceFileList(only_html=1)): | # Web pages can be in web page module ... | ||
data = attribute_filter_re.findall(web_page.getTextContent()) | web_page_reference_list = context.Base_getTranslationSourceFileList(only_html=1) | ||
for attribute in data: | not_found_in_web_page_reference_set = set([]) | ||
a = re.sub(r'[{|}]', "", attribute[1]) | for web_page_reference in web_page_reference_list: | ||
a = re.sub(r'\[.*?\]', "", a) | web_page = context.getDocumentValue(web_page_reference) | ||
if a: | if web_page is None: | ||
translatable_message_set.add(a) | not_found_in_web_page_reference_set.add(web_page_reference) | ||
else: | |||
for message in portal.ERP5Site_extractTranslationMessageListFromHTML(web_page.getTextContent()): | |||
translatable_message_set.add(message) | |||
# ... or in skin folders | |||
for web_page_reference in not_found_in_web_page_reference_set: | |||
if not '/' in web_page_reference: | |||
web_page = context.restrictedTraverse(web_page_reference, None) | |||
if web_page is not None and hasattr(web_page, 'manage_FTPget'): | |||
for message in portal.ERP5Site_extractTranslationMessageListFromHTML(web_page.manage_FTPget()): | |||
|
|||
translatable_message_set.add(message) | |||
tmp = {} | tmp = {} | ||
for language in context.getAvailableLanguageSet(): | for language in context.getAvailableLanguageSet(): | ||
tmp[language] = {} | tmp[language] = {} | ||
for word in translatable_message_set: | for word in translatable_message_set: | ||
tmp[language][word] = Base_translateString(word, lang = language) | tmp[language][word] = unicode(Base_translateString(word, lang = language), 'utf-8') | ||
return """/** | return u"""/** | ||
* This translation data is generated automatically and updated with upgrader in post-upgarde. | * This translation data is generated automatically and updated with upgrader in post-upgarde. | ||
* Do not edit manually, but use "Update Translation Data" action on web site to update from | * Do not edit manually, but use "Update Translation Data" action on web site to update from | ||
* Localizer and from data-i18n tags on web pages. | * Localizer and from data-i18n tags on web pages. | ||
... | @@ -41,7 +49,7 @@ return """/** | ... | @@ -41,7 +49,7 @@ return """/** |
// @ts-ignore | // @ts-ignore | ||
window.translation_data = %s; | window.translation_data = %s; | ||
}(window)); | }(window)); | ||
""" % ("\n ".join( | """ % (u"\n ".join( | ||
json.dumps( | json.dumps( | ||
tmp, | tmp, | ||
sort_keys=True, | sort_keys=True, | ||
... | ... |