Commit 951bec58 authored by Jérome Perrin's avatar Jérome Perrin

WIP: Inventory api: interpolation + group by time sequence

jerome/erp5!6 rebased and in a big
commit so that we can start using that
SimulationTool: implement "linear" flow API

Conflicts:
	product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Resource_zGetInventoryList.xml

testInventoryAPI: silent pyflakes warnings

more interpolation implementation

inventory_list interpolation flow: support at_date & to_date

more and more interpolation API

interpolation

interpolation

group_by_time_sequence_list

testInventoryAPI: move inventory valuation methods in a dedicated test class

This should not be in test inventory list.

test: move Tracking API to a dedicated test module

inventory api: dirty way of disabling cache if keys such as node_category are used.

interpolation: security on SimulationTool_zGetInterpolationMethod
parent 82227348
...@@ -561,6 +561,8 @@ class SimulationTool(BaseTool): ...@@ -561,6 +561,8 @@ class SimulationTool(BaseTool):
omit_output=0, omit_output=0,
omit_asset_increase=0, omit_asset_increase=0,
omit_asset_decrease=0, omit_asset_decrease=0,
# interpolation_method
interpolation_method='default',
# group by # group by
group_by_node=0, group_by_node=0,
group_by_node_category=0, group_by_node_category=0,
...@@ -597,6 +599,7 @@ class SimulationTool(BaseTool): ...@@ -597,6 +599,7 @@ class SimulationTool(BaseTool):
group_by_function_category=0, group_by_function_category=0,
group_by_function_category_strict_membership=0, group_by_function_category_strict_membership=0,
group_by_date=0, group_by_date=0,
group_by_time_sequence_list=None,
# sort_on # sort_on
sort_on=None, sort_on=None,
group_by=None, group_by=None,
...@@ -650,6 +653,43 @@ class SimulationTool(BaseTool): ...@@ -650,6 +653,43 @@ class SimulationTool(BaseTool):
date_dict['range'] = 'ngt' date_dict['range'] = 'ngt'
if date_dict: if date_dict:
column_value_dict['date'] = date_dict column_value_dict['date'] = date_dict
if interpolation_method != 'default':
if not group_by_time_sequence_list:
if not (from_date and (to_date or at_date)):
raise ValueError("date_range is required to use interpolation_method")
# if we consider flow, we also select movement whose mirror date is
# in the from_date/to_date range and movement whose
# start_date/stop_date contains the report range.
# The selected range is wider, but the selected movements will have an
# "interpolation_ratio" applied to their quantity and prices.
if to_date:
column_value_dict['date'] = ComplexQuery(
Query(date=(from_date, to_date), range='minmax'),
Query(mirror_date=(from_date, to_date), range='minmax'),
ComplexQuery(
Query(mirror_date=from_date, range='min'),
Query(date=to_date, range='max'),
operator="AND"),
ComplexQuery(
Query(date=from_date, range='min'),
Query(mirror_date=to_date, range='max'),
operator="AND"),
operator="OR"
)
else:
column_value_dict['date'] = ComplexQuery(
Query(date=(from_date, at_date), range='minngt'),
Query(mirror_date=(from_date, at_date), range='minngt'),
ComplexQuery(
Query(mirror_date=from_date, range='min'),
Query(date=at_date, range='ngt'),
operator="AND"),
ComplexQuery(
Query(date=from_date, range='min'),
Query(mirror_date=at_date, range='ngt'),
operator="AND"),
operator="OR"
)
else: else:
column_value_dict['date'] = {'query': [to_date], 'range': 'ngt'} column_value_dict['date'] = {'query': [to_date], 'range': 'ngt'}
column_value_dict['mirror_date'] = {'query': [from_date], 'range': 'nlt'} column_value_dict['mirror_date'] = {'query': [from_date], 'range': 'nlt'}
...@@ -949,6 +989,8 @@ class SimulationTool(BaseTool): ...@@ -949,6 +989,8 @@ class SimulationTool(BaseTool):
new_kw['related_key_select_expression_list'] =\ new_kw['related_key_select_expression_list'] =\
related_key_select_expression_list related_key_select_expression_list
# XXX
sql_kw['group_by_time_sequence_list'] = group_by_time_sequence_list
return sql_kw, new_kw return sql_kw, new_kw
####################################################### #######################################################
...@@ -1033,6 +1075,17 @@ class SimulationTool(BaseTool): ...@@ -1033,6 +1075,17 @@ class SimulationTool(BaseTool):
output_simulation_state - only take rows with specified simulation_state output_simulation_state - only take rows with specified simulation_state
and quantity < 0 and quantity < 0
interpolation_method - Method to consider movements when calculating flows.
* (default): Consider the movement decreases 100% of source stock on
start date and increase 100% of the destination node stock on stop
date.
* linear: consider the movement decreases source stock and increase
destination stock linearly between start date and stop date.
* all_or_nothing: consider only movement who starts after the beginning of
the query period and finishes after the end of the query period.
* one_for_all: consider the movement fully as long as it is partially
contained in the query period.
ignore_variation - do not take into account variation in inventory ignore_variation - do not take into account variation in inventory
calculation (useless on getInventory, but useful on calculation (useless on getInventory, but useful on
getInventoryList) getInventoryList)
...@@ -1159,6 +1212,7 @@ class SimulationTool(BaseTool): ...@@ -1159,6 +1212,7 @@ class SimulationTool(BaseTool):
group_by_section_category=0, group_by_section_category=0,
group_by_section_category_strict_membership=0, group_by_section_category_strict_membership=0,
group_by_resource=None, group_by_resource=None,
group_by_time_sequence_list=(),
group_by=None, group_by=None,
**ignored): **ignored):
""" """
...@@ -1178,7 +1232,8 @@ class SimulationTool(BaseTool): ...@@ -1178,7 +1232,8 @@ class SimulationTool(BaseTool):
group_by_function or group_by_mirror_section or group_by_payment or \ group_by_function or group_by_mirror_section or group_by_payment or \
group_by_sub_variation or group_by_variation or \ group_by_sub_variation or group_by_variation or \
group_by_movement or group_by_date or group_by_section_category or\ group_by_movement or group_by_date or group_by_section_category or\
group_by_section_category_strict_membership: group_by_section_category_strict_membership or \
group_by_time_sequence_list:
if group_by_resource is None: if group_by_resource is None:
group_by_resource = 1 group_by_resource = 1
new_group_by_dict['group_by_resource'] = group_by_resource new_group_by_dict['group_by_resource'] = group_by_resource
...@@ -1195,6 +1250,7 @@ class SimulationTool(BaseTool): ...@@ -1195,6 +1250,7 @@ class SimulationTool(BaseTool):
omit_simulation=0, omit_simulation=0,
only_accountable=True, only_accountable=True,
default_stock_table='stock', default_stock_table='stock',
interpolation_method='default',
selection_domain=None, selection_report=None, selection_domain=None, selection_report=None,
statistic=0, inventory_list=1, statistic=0, inventory_list=1,
precision=None, connection_id=None, precision=None, connection_id=None,
...@@ -1273,6 +1329,7 @@ class SimulationTool(BaseTool): ...@@ -1273,6 +1329,7 @@ class SimulationTool(BaseTool):
'stock_table_id': default_stock_table, 'stock_table_id': default_stock_table,
'src__': src__, 'src__': src__,
'ignore_variation': ignore_variation, 'ignore_variation': ignore_variation,
'interpolation_method': interpolation_method,
'standardise': standardise, 'standardise': standardise,
'omit_simulation': omit_simulation, 'omit_simulation': omit_simulation,
'only_accountable': only_accountable, 'only_accountable': only_accountable,
...@@ -1288,7 +1345,9 @@ class SimulationTool(BaseTool): ...@@ -1288,7 +1345,9 @@ class SimulationTool(BaseTool):
# Get cached data # Get cached data
if getattr(self, "Resource_zGetInventoryCacheResult", None) is not None and \ if getattr(self, "Resource_zGetInventoryCacheResult", None) is not None and \
optimisation__ and (not kw.get('from_date')) and \ optimisation__ and (not kw.get('from_date')) and \
'transformed_resource' not in kw: 'transformed_resource' not in kw \
and "category" not in str(kw) \
and "group_by_time_sequence_list" not in kw:
# Here is the different kind of date # Here is the different kind of date
# from_date : >= # from_date : >=
# to_date : < # to_date : <
...@@ -1318,7 +1377,7 @@ class SimulationTool(BaseTool): ...@@ -1318,7 +1377,7 @@ class SimulationTool(BaseTool):
kw['from_date'] = cached_date kw['from_date'] = cached_date
else: else:
cached_result = [] cached_result = []
sql_kw, new_kw = self._generateKeywordDict(**kw) sql_kw, new_kw = self._generateKeywordDict(interpolation_method=interpolation_method, **kw)
# Copy kw content as _generateSQLKeywordDictFromKeywordDict # Copy kw content as _generateSQLKeywordDictFromKeywordDict
# remove some values from it # remove some values from it
try: try:
...@@ -1331,6 +1390,25 @@ class SimulationTool(BaseTool): ...@@ -1331,6 +1390,25 @@ class SimulationTool(BaseTool):
stock_sql_kw = self._generateSQLKeywordDictFromKeywordDict( stock_sql_kw = self._generateSQLKeywordDictFromKeywordDict(
table=default_stock_table, sql_kw=sql_kw, new_kw=new_kw_copy) table=default_stock_table, sql_kw=sql_kw, new_kw=new_kw_copy)
stock_sql_kw.update(base_inventory_kw) stock_sql_kw.update(base_inventory_kw)
# TODO: move in _generateSQLKeywordDictFromKeywordDict
if interpolation_method in ('linear', 'all_or_nothing', 'one_for_all'):
# XXX only DateTime instance are supported
from_date = kw.get('from_date')
if from_date:
from_date = from_date.toZone("UTC")
to_date = kw.get('to_date')
if to_date:
to_date = to_date.toZone("UTC")
at_date = kw.get('at_date')
if at_date:
at_date = at_date.toZone("UTC")
stock_sql_kw['interpolation_method_from_date'] = from_date
stock_sql_kw['interpolation_method_to_date'] = to_date
stock_sql_kw['interpolation_method_at_date'] = at_date
elif interpolation_method != 'default':
raise ValueError("Unsupported interpolation_method %r" % (interpolation_method,))
delta_result = self.Resource_zGetInventoryList( delta_result = self.Resource_zGetInventoryList(
**stock_sql_kw) **stock_sql_kw)
if src__: if src__:
......
<dtml-let interpolation_ratio="SimulationTool_zGetInterpolationMethod(
stock_table_id=stock_table_id,
interpolation_method=interpolation_method,
interpolation_method_from_date=interpolation_method_from_date,
interpolation_method_to_date=interpolation_method_to_date,
interpolation_method_at_date=interpolation_method_at_date,
group_by_time_sequence_list=group_by_time_sequence_list,
src__=1)">
SELECT SELECT
<dtml-if expr="precision is not None"> <dtml-if expr="precision is not None">
SUM(ROUND(<dtml-var stock_table_id>.quantity SUM(ROUND(
<dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>)) AS inventory, <dtml-var stock_table_id>.quantity
SUM(ROUND(<dtml-var stock_table_id>.quantity <dtml-if transformed_uid> * transformation.quantity</dtml-if>
<dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>)) AS total_quantity, * <dtml-var interpolation_ratio>, <dtml-var precision>)) AS inventory,
SUM(ROUND(
<dtml-var stock_table_id>.quantity
<dtml-if transformed_uid> * transformation.quantity</dtml-if>
* <dtml-var interpolation_ratio>, <dtml-var precision>)) AS total_quantity,
<dtml-if convert_quantity_result> <dtml-if convert_quantity_result>
SUM(ROUND(<dtml-var stock_table_id>.quantity * measure.quantity SUM(ROUND(<dtml-var stock_table_id>.quantity * measure.quantity
<dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if> <dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if>
<dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>)) * <dtml-var interpolation_ratio>, <dtml-var precision>))
AS converted_quantity, AS converted_quantity,
</dtml-if> </dtml-if>
IFNULL(SUM(ROUND(<dtml-var stock_table_id>.total_price, <dtml-var precision>)), 0) AS total_price
IFNULL(SUM(ROUND(
<dtml-var stock_table_id>.total_price * <dtml-var interpolation_ratio>, <dtml-var precision>)), 0) AS total_price
<dtml-else> <dtml-else>
SUM(<dtml-var stock_table_id>.quantity <dtml-if transformed_uid> * transformation.quantity</dtml-if>) AS inventory, SUM(<dtml-var stock_table_id>.quantity
SUM(<dtml-var stock_table_id>.quantity <dtml-if transformed_uid> * transformation.quantity</dtml-if>) AS total_quantity, <dtml-if transformed_uid> * transformation.quantity</dtml-if>
* <dtml-var interpolation_ratio>
) AS inventory,
SUM(<dtml-var stock_table_id>.quantity
<dtml-if transformed_uid> * transformation.quantity</dtml-if>
* <dtml-var interpolation_ratio>
) AS total_quantity,
<dtml-if convert_quantity_result> <dtml-if convert_quantity_result>
ROUND(SUM(<dtml-var stock_table_id>.quantity * measure.quantity ROUND(SUM(<dtml-var stock_table_id>.quantity * measure.quantity
<dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if> <dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if>
<dtml-if transformed_uid> * transformation.quantity</dtml-if>), 12) <dtml-if transformed_uid> * transformation.quantity</dtml-if> * <dtml-var interpolation_ratio>), 12)
AS converted_quantity, AS converted_quantity,
</dtml-if> </dtml-if>
IFNULL(SUM(<dtml-var stock_table_id>.total_price), 0) AS total_price IFNULL(SUM(<dtml-var stock_table_id>.total_price * <dtml-var interpolation_ratio>), 0) AS total_price
</dtml-if> </dtml-if>
<dtml-if inventory_list> <dtml-if inventory_list>
, ,
...@@ -56,6 +77,8 @@ SELECT ...@@ -56,6 +77,8 @@ SELECT
COUNT(DISTINCT <dtml-var stock_table_id>.uid) AS stock_uid, COUNT(DISTINCT <dtml-var stock_table_id>.uid) AS stock_uid,
MAX(<dtml-var stock_table_id>.date) AS date MAX(<dtml-var stock_table_id>.date) AS date
</dtml-if> </dtml-if>
<dtml-if group_by_time_sequence_list>, slot_index </dtml-if> <dtml-comment>XXX is this really needed? are empty slots returned ? </dtml-comment>
<dtml-if select_expression>, <dtml-var select_expression></dtml-if> <dtml-if select_expression>, <dtml-var select_expression></dtml-if>
FROM FROM
...@@ -69,6 +92,55 @@ FROM ...@@ -69,6 +92,55 @@ FROM
</dtml-if> </dtml-if>
</dtml-in> </dtml-in>
, <dtml-var stock_table_id> , <dtml-var stock_table_id>
<dtml-if group_by_time_sequence_list>
RIGHT JOIN
( <dtml-in prefix="time_slot" expr="_.list(_.enumerate(group_by_time_sequence_list))">
SELECT
<dtml-sqlvar expr="time_slot_key" type="int"> slot_index,
<dtml-sqlvar expr="time_slot_item.get('from_date')" type="datetime" optional> slot_from_date,
<dtml-sqlvar expr="time_slot_item.get('at_date')" type="datetime" optional> slot_at_date,
<dtml-sqlvar expr="time_slot_item.get('to_date')" type="datetime" optional> slot_to_date
<dtml-unless time_slot_end>UNION ALL</dtml-unless>
</dtml-in> ) slots
ON
<dtml-if group_by_time_sequence_list>
(
( slot_from_date is not null AND
( slot_at_date is not null AND
GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND
LEAST(`stock`.`date`, `stock`.`mirror_date`) <= slot_at_date
) OR (
(
slot_to_date is not null AND
GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND
LEAST(`stock`.`date`, `stock`.`mirror_date`) < slot_to_date
) OR (
GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND
slot_at_date is null AND slot_to_date is null
)
)
) OR (
slot_from_date is null AND (
( slot_at_date is not null AND
( LEAST(`stock`.`date`, `stock`.`mirror_date`) <= slot_at_date )
) OR LEAST(`stock`.`date`, `stock`.`mirror_date`) < slot_to_date
)
)
)
<dtml-else>
(
( slot_from_date is null OR stock.date >= slot_from_date )
AND ( slot_at_date is null OR stock.date <= slot_at_date )
AND ( slot_to_date is null OR stock.date < slot_to_date )
)
</dtml-if>
</dtml-if>
</dtml-if> </dtml-if>
<dtml-if quantity_unit_uid> <dtml-comment>XXX quantity unit conversion will not work when using implict_join=False</dtml-comment> <dtml-if quantity_unit_uid> <dtml-comment>XXX quantity unit conversion will not work when using implict_join=False</dtml-comment>
LEFT JOIN quantity_unit_conversion ON LEFT JOIN quantity_unit_conversion ON
...@@ -116,9 +188,16 @@ WHERE ...@@ -116,9 +188,16 @@ WHERE
<dtml-if group_by_expression> <dtml-if group_by_expression>
GROUP BY GROUP BY
<dtml-if transformed_uid>transformation.transformed_uid,</dtml-if> <dtml-if transformed_uid>transformation.transformed_uid,</dtml-if>
<dtml-if group_by_time_sequence_list>slot_index,</dtml-if>
<dtml-var group_by_expression> <dtml-var group_by_expression>
</dtml-if> </dtml-if>
<dtml-if order_by_expression> <dtml-if order_by_expression>
ORDER BY ORDER BY
<dtml-var order_by_expression> <dtml-var order_by_expression>
<dtml-else>
<dtml-if group_by_time_sequence_list>
ORDER BY slot_index
</dtml-if>
</dtml-if> </dtml-if>
</dtml-let>
...@@ -45,7 +45,12 @@ convert_quantity_result\r\n ...@@ -45,7 +45,12 @@ convert_quantity_result\r\n
quantity_unit_uid\r\n quantity_unit_uid\r\n
stock_table_id=stock\r\n stock_table_id=stock\r\n
transformed_uid\r\n transformed_uid\r\n
transformed_variation_text</string> </value> transformed_variation_text\r\n
group_by_time_sequence_list:list\r\n
interpolation_method\r\n
interpolation_method_from_date=not_applicable\r\n
interpolation_method_to_date=not_applicable\r\n
interpolation_method_at_date=not_applicable</string> </value>
</item> </item>
<item> <item>
<key> <string>cache_time_</string> </key> <key> <string>cache_time_</string> </key>
......
<dtml-if expr="interpolation_method == 'linear'">
<dtml-if group_by_time_sequence_list>
CASE
WHEN <dtml-var stock_table_id>.mirror_date = <dtml-var stock_table_id>.date THEN 1
ELSE (
UNIX_TIMESTAMP(
IFNULL(
LEAST(
IFNULL(slot_at_date, slot_to_date),
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
),
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
)
)
- UNIX_TIMESTAMP(
GREATEST(
IFNULL(
slot_from_date, TIMESTAMP(0)
),
LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
)
)
)
/ (
UNIX_TIMESTAMP(GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) -
UNIX_TIMESTAMP(LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) ) END
<dtml-else>
CASE
WHEN <dtml-var stock_table_id>.mirror_date = <dtml-var stock_table_id>.date THEN 1
ELSE (
UNIX_TIMESTAMP(LEAST(
<dtml-if interpolation_method_at_date>
<dtml-sqlvar interpolation_method_at_date type="datetime">
<dtml-else>
<dtml-sqlvar interpolation_method_to_date type="datetime">
</dtml-if>,
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date) ))
- UNIX_TIMESTAMP(GREATEST(<dtml-sqlvar interpolation_method_from_date type="datetime">,
LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date))))
/ ( UNIX_TIMESTAMP(GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) -
UNIX_TIMESTAMP(LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) ) END
</dtml-if>
<dtml-elif expr="interpolation_method == 'all_or_nothing'">
CASE
WHEN (
-- movement contained in time frame
LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
>= <dtml-sqlvar interpolation_method_from_date type="datetime"> AND
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
<dtml-if interpolation_method_at_date>
<= <dtml-sqlvar interpolation_method_at_date type="datetime">
<dtml-else>
< <dtml-sqlvar interpolation_method_to_date type="datetime">
</dtml-if>
) THEN 1
ELSE 0
END
<dtml-elif expr="interpolation_method == 'one_for_all'">
CASE
WHEN (
-- movement overlaps with time frame
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
<= <dtml-sqlvar interpolation_method_from_date type="datetime"> OR
LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
<dtml-if interpolation_method_at_date>
>= <dtml-sqlvar interpolation_method_at_date type="datetime">
<dtml-else>
> <dtml-sqlvar interpolation_method_to_date type="datetime">
</dtml-if>
) THEN 0
ELSE 1
END
<dtml-else>
1
</dtml-if>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Use_Database_Methods_Permission</string> </key>
<value>
<list>
<string>Member</string>
</list>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>stock_table_id\r\n
interpolation_method\r\n
interpolation_method_from_date\r\n
interpolation_method_to_date\r\n
interpolation_method_at_date\r\n
group_by_time_sequence_list:list</string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>cmf_activity_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SimulationTool_zGetInterpolationMethod</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment