Skip to content

erp5
erp5
Commit 43affc49 authored by Tatuya Kamada's avatar Tatuya Kamada
Browse files
Options

erp5_tabular_data_tool: Add portal_tabular (Tabular Data Tool) 

Tabular Data Tool is a Tool to handle tabular data (2-dimentional table data)
in erp5.
The tabular data type is currently a plain pandas DataFrame. It is intended to
use with ERP5 Reports to create spread sheets (ODS/Excel).

With Tabular Data Tool, we can do those things on reporting:

- transform dict-list, portal-catalog-result and document-list into a tabular data
- joining/grouping/sorting the tabular data
- using vectorizing function to optimise time consuming reports
parent 92062f69
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1134 additions and 0 deletions
bt5/erp5_tabular_data_tool/DocumentTemplateItem/portal_components/tool.erp5.TabularDataTool.py 0 → 100644
View file @ 43affc49
from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions
from Products.ERP5Type.Globals import InitializeClass
from Products.ERP5Type.Tool.BaseTool import BaseTool
import pandas as pd
MARKER = object()
DEFAULT_FILL_STRING = ''
DEFAULT_FILL_NUMBER = 0.0
DEFAULT_FILL_UID = -1
class TabularDataTool(BaseTool):
"""
Tabular Data Tool is a Tool to handle tabular data (2-dimentional table data) in erp5.
The tabular data type is currently a plain pandas DataFrame. It is intended to
use with ERP5 Reports to create spread sheets (ODS/Excel).
With Tabular Data Tool, we can do those things on reporting:
- transform dict-list, portal-catalog-result and document-list into a tabular data
- joining/grouping/sorting the tabular data
- using vectorizing function to optimise time consuming reports
"""
id = 'portal_tabular'
title = 'Tabular Data Tool'
meta_type = 'ERP5 Tabular Data Tool'
portal_type = 'Tabular Data Tool'
allowed_types = ()
# Declarative Security
security = ClassSecurityInfo()
# The empty DataFrame marker
def _createEmptyDataFrame(self):
return pd.DataFrame()
security.declareProtected(Permissions.AccessContentsInformation, 'fillna')
def fillna(self, tabular, fillna_on_string=MARKER, fillna_on_number=MARKER, fillna_on_uid=MARKER):
"""
Fill NA/NaN values on the given tabular (a dataframe).
Keyword arguments:
tabular -- the dataframe to fill empty values
fillna_on_string -- Fill given value for empty string
fillna_on_number -- Fill given value for empty numbers (int or float)
fillna_on_uid -- Fill given value for empty uid, which overrides fillna_on_numbers for uid
"""
if tabular is None:
return tabular
string_filler = DEFAULT_FILL_STRING if fillna_on_string is MARKER else fillna_on_string
number_filler = DEFAULT_FILL_NUMBER if fillna_on_number is MARKER else fillna_on_number
uid_filler = DEFAULT_FILL_UID if fillna_on_uid is MARKER else fillna_on_uid
fillna_setting_dict = {}
string_column_list = tabular.select_dtypes(include=[object]).columns.tolist()
string_none_column_list = []
if fillna_on_string is None:
# pandas.dataframe.fillna(None) raises error, so use .replace() instead
string_none_column_list = string_column_list
string_column_list = []
number_column_list = tabular.select_dtypes(include=[int,float]).columns.tolist()
uid_column_list = [column for column in tabular.columns.tolist() if column.endswith('_uid')]
number_column_list = list(set(number_column_list) - set(uid_column_list))
string_column_list = list(set(string_column_list) - set(uid_column_list))
fillna_setting_dict.update(dict.fromkeys(string_column_list, string_filler))
fillna_setting_dict.update(dict.fromkeys(number_column_list, number_filler))
fillna_setting_dict.update(dict.fromkeys(uid_column_list, uid_filler))
filled_tabular = tabular.fillna(value=fillna_setting_dict)
if string_none_column_list:
replace_dict = dict.fromkeys(string_none_column_list, {"":None})
filled_tabular = filled_tabular.replace(replace_dict)
return filled_tabular
security.declareProtected(Permissions.AccessContentsInformation, 'getTabular')
def getTabular(self, data_list,
fillna_on_string=MARKER,
fillna_on_number=MARKER,
fillna_on_uid=MARKER,
additional_property_name_list=MARKER):
"""
Create a tabular from dict list with filling values for convenience in ERP5.
Currently the tabular data type is pandas DataFrame.
Keyword arguments:
data_list -- A dict list or brain_list to convert it to dataframe
fillna_on_string -- Fill given value for empty string
fillna_on_numbers -- Fill given value for empty numbers (int or float)
fillna_on_uid -- Fill given value for empty uid, which overrides fillna_on_numbers for uid
additional_property_name_list -- Additional property name list of brain object
"""
if data_list is None or len(data_list) == 0:
return self._createEmptyDataFrame()
df = None
if isinstance(data_list[0], dict):
df = pd.DataFrame.from_dict(data_list)
# Expect Shared.ZRDB.Results
elif getattr(data_list, 'dictionaries', None) is not None:
if additional_property_name_list is not MARKER:
new_data_dict_list = []
for (data_, record_dict) in zip(data_list, data_list.dictionaries()):
property_dict = self._getPropertyDict(data_, additional_property_name_list, False, False)
property_dict.update(record_dict)
new_data_dict_list.append(property_dict)
df = self.getTabular(new_data_dict_list)
else:
df = self.getTabular(data_list.dictionaries())
if df is not None:
return self.fillna(df, fillna_on_string=fillna_on_string,
fillna_on_number=fillna_on_number,
fillna_on_uid=fillna_on_uid)
raise ValueError("The type is not supported, {}".format(data_list))
def _getPropertyDict(self, document, property_name_list, is_category_as_uid, add_category_title):
portal_type = document.getPortalType()
base_type = self.getPortalObject().portal_types[portal_type]
base_category_list = base_type.getInstanceBaseCategoryList()
if property_name_list is MARKER:
property_name_list = list(base_type.getInstancePropertySet())
else:
# If property_name_list is explicitly passed, respect the name list in category
base_category_list = list(set(base_category_list).intersection(set(property_name_list)))
property_dict = {}
for property_name in property_name_list:
property_dict[property_name] = document.getProperty(property_name)
for base_category_id in base_category_list:
category_name = "{}_uid".format(base_category_id) if is_category_as_uid else base_category_id
property_dict[category_name] = document.getProperty(category_name)
if add_category_title:
category_title = "{}_title".format(base_category_id)
property_dict[category_title] = document.getProperty(category_title)
return property_dict
security.declareProtected(Permissions.AccessContentsInformation, 'getFromDocumentList')
def getFromDocumentList(self, document_list,
property_name_list=MARKER,
fillna_on_string=MARKER,
fillna_on_number=MARKER,
fillna_on_uid=MARKER,
is_category_as_uid=True,
add_category_title=True):
"""
Create tabular data from document list
Keyword arguments:
property_name_list -- Explict to specify the property name list of document
If not specified all properties and categories of the portal type is applied
fillna_on_string -- Fill given value for empty string
fillna_on_numbers -- Fill given value for empty numbers (int or float)
fillna_on_uid -- Fill given value for empty uid, which overrides fill_on_numbers for uid
is_category_as_uid -- Store uid on {base_category}_uid, instead of store the url on {base_category}
add_category_title -- Add {category}_title columns into the tabular
Note: It takes O(N) time.
"""
if document_list is None or len(document_list) == 0:
return self._createEmptyDataFrame()
# raise Attribute error if given document does not have portal_type
portal_type_set = set([x.getPortalType() for x in document_list])
if len(portal_type_set) != 1:
raise ValueError('Given documents do not have the same portal type:{}'.format(portal_type_set))
dict_list = [self._getPropertyDict(x, property_name_list,
is_category_as_uid,
add_category_title) for x in document_list]
return self.getTabular(dict_list,
fillna_on_string=fillna_on_string,
fillna_on_number=fillna_on_number,
fillna_on_uid=fillna_on_uid)
security.declareProtected(Permissions.AccessContentsInformation, 'searchResults')
def searchResults(self, select_dict=None,
fillna_on_string=MARKER,
fillna_on_number=MARKER,
fillna_on_uid=MARKER, **kw):
"""
Create tabular data based on the portal_catalog.searchResult()
select_dict -- If select_dict is None, specify all the column ids of catalog table.
"""
if select_dict is None:
portal = self.getPortalObject()
catalog_id = portal.portal_catalog.getDefaultErp5CatalogId()
default_erp5_catalog = portal.portal_catalog[catalog_id]
column_id_set = set(default_erp5_catalog.getResultColumnIds())
sql_search_result_keys_set = set(default_erp5_catalog.getSqlSearchResultKeysList())
all_column_id_set = column_id_set - sql_search_result_keys_set
select_dict = dict([(column_id, None) for column_id in all_column_id_set if column_id.startswith('catalog.')])
search_result = portal.portal_catalog(select_dict=select_dict, **kw)
return self.getTabular(search_result,
fillna_on_string=fillna_on_string,
fillna_on_number=fillna_on_number,
fillna_on_uid=fillna_on_uid)
__call__ = getTabular
InitializeClass(TabularDataTool)
\ No newline at end of file
bt5/erp5_tabular_data_tool/DocumentTemplateItem/portal_components/tool.erp5.TabularDataTool.xml 0 → 100644
View file @ 43affc49
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Tool Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>TabularDataTool</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>tool.erp5.TabularDataTool</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Tool Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_tabular_data_tool/PortalTypeTemplateItem/portal_types/Tabular%20Data%20Tool.xml 0 → 100644
View file @ 43affc49
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Base Type" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>acquire_local_roles</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>content_icon</string> </key>
<value> <string>folder_icon.gif</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Tabular Data Tool is for handling tabular data (2-dimentional table data) .</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Tabular Data Tool</string> </value>
</item>
<item>
<key> <string>init_script</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>permission</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Base Type</string> </value>
</item>
<item>
<key> <string>type_class</string> </key>
<value> <string>TabularDataTool</string> </value>
</item>
<item>
<key> <string>type_interface</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>type_mixin</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_tabular_data_tool/ToolTemplateItem/portal_tabular.xml 0 → 100644
View file @ 43affc49
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Tabular Data Tool" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>portal_tabular</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Tabular Data Tool</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_tabular_data_tool/bt/dependency_list 0 → 100644
View file @ 43affc49
erp5_base
\ No newline at end of file
bt5/erp5_tabular_data_tool/bt/template_document_id_list 0 → 100644
View file @ 43affc49
tool.erp5.TabularDataTool
\ No newline at end of file
bt5/erp5_tabular_data_tool/bt/template_portal_type_id_list 0 → 100644
View file @ 43affc49
Tabular Data Tool
\ No newline at end of file
bt5/erp5_tabular_data_tool/bt/template_tool_id_list 0 → 100644
View file @ 43affc49
portal_tabular
\ No newline at end of file
bt5/erp5_tabular_data_tool/bt/title 0 → 100644
View file @ 43affc49
erp5_tabular_data_tool
\ No newline at end of file
bt5/erp5_tabular_data_tool_test/TestTemplateItem/portal_components/test.erp5.testTabularDataTool.xml 0 → 100644
View file @ 43affc49
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Test Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>testTabularDataTool</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>test.erp5.testTabularDataTool</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Test Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_tabular_data_tool_test/bt/dependency_list 0 → 100644
View file @ 43affc49
erp5_trade
erp5_full_text_mroonga_catalog
erp5_tabular_data_tool
\ No newline at end of file
bt5/erp5_tabular_data_tool_test/bt/template_test_id_list 0 → 100644
View file @ 43affc49
test.erp5.testTabularDataTool
\ No newline at end of file
bt5/erp5_tabular_data_tool_test/bt/title 0 → 100644
View file @ 43affc49
erp5_tabular_data_tool_test
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7