Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
138
Merge Requests
138
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
erp5
Commits
4859cd0a
Commit
4859cd0a
authored
Mar 25, 2015
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
CMFActivity: more refactoring between SQLDict & SQLQueue
parent
bb2bfa6e
Changes
14
Show whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
101 additions
and
325 deletions
+101
-325
product/CMFActivity/Activity/Queue.py
product/CMFActivity/Activity/Queue.py
+3
-1
product/CMFActivity/Activity/SQLBase.py
product/CMFActivity/Activity/SQLBase.py
+83
-6
product/CMFActivity/Activity/SQLDict.py
product/CMFActivity/Activity/SQLDict.py
+0
-91
product/CMFActivity/Activity/SQLQueue.py
product/CMFActivity/Activity/SQLQueue.py
+0
-91
product/CMFActivity/skins/activity/SQLBase_dumpMessageList.zsql
...t/CMFActivity/skins/activity/SQLBase_dumpMessageList.zsql
+3
-2
product/CMFActivity/skins/activity/SQLBase_getPriority.zsql
product/CMFActivity/skins/activity/SQLBase_getPriority.zsql
+2
-2
product/CMFActivity/skins/activity/SQLBase_hasMessage.zsql
product/CMFActivity/skins/activity/SQLBase_hasMessage.zsql
+3
-2
product/CMFActivity/skins/activity/SQLBase_timeShift.zsql
product/CMFActivity/skins/activity/SQLBase_timeShift.zsql
+4
-5
product/CMFActivity/skins/activity/SQLBase_validateMessageList.zsql
...FActivity/skins/activity/SQLBase_validateMessageList.zsql
+3
-2
product/CMFActivity/skins/activity/SQLDict_timeShift.zsql
product/CMFActivity/skins/activity/SQLDict_timeShift.zsql
+0
-25
product/CMFActivity/skins/activity/SQLQueue_dumpMessageList.zsql
.../CMFActivity/skins/activity/SQLQueue_dumpMessageList.zsql
+0
-14
product/CMFActivity/skins/activity/SQLQueue_getPriority.zsql
product/CMFActivity/skins/activity/SQLQueue_getPriority.zsql
+0
-18
product/CMFActivity/skins/activity/SQLQueue_hasMessage.zsql
product/CMFActivity/skins/activity/SQLQueue_hasMessage.zsql
+0
-20
product/CMFActivity/skins/activity/SQLQueue_validateMessageList.zsql
...Activity/skins/activity/SQLQueue_validateMessageList.zsql
+0
-46
No files found.
product/CMFActivity/Activity/Queue.py
View file @
4859cd0a
...
...
@@ -227,7 +227,9 @@ class Queue(object):
return
0
def
countMessageWithTag
(
self
,
activity_tool
,
value
):
return
0
"""Return the number of messages which match the given tag.
"""
return
self
.
countMessage
(
activity_tool
,
tag
=
value
)
# Transaction Management
def
prepareQueueMessageList
(
self
,
activity_tool
,
message_list
):
...
...
product/CMFActivity/Activity/SQLBase.py
View file @
4859cd0a
...
...
@@ -130,12 +130,50 @@ class SQLBase(Queue):
processing
=
line
.
processing
)
for
line
in
result
]
def
_getPriority
(
self
,
activity_tool
,
method
,
default
):
result
=
method
()
if
not
result
:
return
default
def
countMessage
(
self
,
activity_tool
,
tag
=
None
,
path
=
None
,
method_id
=
None
,
message_uid
=
None
,
**
kw
):
"""Return the number of messages which match the given parameters.
"""
if
isinstance
(
tag
,
str
):
tag
=
[
tag
]
if
isinstance
(
path
,
str
):
path
=
[
path
]
if
isinstance
(
method_id
,
str
):
method_id
=
[
method_id
]
result
=
activity_tool
.
SQLBase_validateMessageList
(
table
=
self
.
sql_table
,
method_id
=
method_id
,
path
=
path
,
message_uid
=
message_uid
,
tag
=
tag
,
serialization_tag
=
None
,
count
=
1
)
return
result
[
0
].
uid_count
def
hasActivity
(
self
,
activity_tool
,
object
,
method_id
=
None
,
only_valid
=
None
,
active_process_uid
=
None
):
hasMessage
=
getattr
(
activity_tool
,
'SQLBase_hasMessage'
,
None
)
if
hasMessage
is
not
None
:
if
object
is
None
:
path
=
None
else
:
path
=
'/'
.
join
(
object
.
getPhysicalPath
())
result
=
hasMessage
(
table
=
self
.
sql_table
,
path
=
path
,
method_id
=
method_id
,
only_valid
=
only_valid
,
active_process_uid
=
active_process_uid
)
if
result
:
return
result
[
0
].
message_count
>
0
return
0
def
dumpMessageList
(
self
,
activity_tool
):
# Dump all messages in the table.
return
[
Message
.
load
(
line
.
message
,
uid
=
line
.
uid
,
line
=
line
)
for
line
in
activity_tool
.
SQLBase_dumpMessageList
(
table
=
self
.
sql_table
)]
def
getPriority
(
self
,
activity_tool
):
result
=
activity_tool
.
SQLBase_getPriority
(
table
=
self
.
sql_table
)
if
result
:
assert
len
(
result
)
==
1
,
len
(
result
)
return
result
[
0
][
'priority'
]
return
Queue
.
getPriority
(
self
,
activity_tool
)
def
_retryOnLockError
(
self
,
method
,
args
=
(),
kw
=
{}):
while
True
:
...
...
@@ -146,6 +184,36 @@ class SQLBase(Queue):
# a lock error into a conflict error.
LOG
(
'SQLBase'
,
INFO
,
'Got a lock error, retrying...'
)
# Validation private methods
def
_validate
(
self
,
activity_tool
,
method_id
=
None
,
message_uid
=
None
,
path
=
None
,
tag
=
None
,
serialization_tag
=
None
):
if
isinstance
(
method_id
,
str
):
method_id
=
[
method_id
]
if
isinstance
(
path
,
str
):
path
=
[
path
]
if
isinstance
(
tag
,
str
):
tag
=
[
tag
]
if
method_id
or
message_uid
or
path
or
tag
or
serialization_tag
:
result
=
activity_tool
.
SQLBase_validateMessageList
(
table
=
self
.
sql_table
,
method_id
=
method_id
,
message_uid
=
message_uid
,
path
=
path
,
tag
=
tag
,
count
=
False
,
serialization_tag
=
serialization_tag
)
message_list
=
[]
for
line
in
result
:
m
=
Message
.
load
(
line
.
message
,
line
=
line
,
uid
=
line
.
uid
,
date
=
line
.
date
,
processing_node
=
line
.
processing_node
)
if
not
hasattr
(
m
,
'order_validation_text'
):
# BBB
m
.
order_validation_text
=
self
.
getOrderValidationText
(
m
)
message_list
.
append
(
m
)
return
message_list
def
_validate_after_method_id
(
self
,
activity_tool
,
message
,
value
):
return
self
.
_validate
(
activity_tool
,
method_id
=
value
)
...
...
@@ -565,3 +633,12 @@ class SQLBase(Queue):
invoke
(
Message
.
load
(
line
.
message
,
uid
=
line
.
uid
,
line
=
line
))
if
uid_list
:
activity_tool
.
SQLBase_delMessage
(
table
=
self
.
sql_table
,
uid
=
uid_list
)
# Required for tests
def
timeShift
(
self
,
activity_tool
,
delay
,
processing_node
=
None
):
"""
To simulate time shift, we simply substract delay from
all dates in message(_queue) table
"""
activity_tool
.
SQLBase_timeShift
(
table
=
self
.
sql_table
,
delay
=
delay
,
processing_node
=
processing_node
)
product/CMFActivity/Activity/SQLDict.py
View file @
4859cd0a
...
...
@@ -191,29 +191,6 @@ class SQLDict(SQLBase):
return
None
,
original_uid
,
[
uid
]
return
load
def
hasActivity
(
self
,
activity_tool
,
object
,
method_id
=
None
,
only_valid
=
None
,
active_process_uid
=
None
):
hasMessage
=
getattr
(
activity_tool
,
'SQLDict_hasMessage'
,
None
)
if
hasMessage
is
not
None
:
if
object
is
None
:
my_object_path
=
None
else
:
my_object_path
=
'/'
.
join
(
object
.
getPhysicalPath
())
result
=
hasMessage
(
path
=
my_object_path
,
method_id
=
method_id
,
only_valid
=
only_valid
,
active_process_uid
=
active_process_uid
)
if
len
(
result
)
>
0
:
return
result
[
0
].
message_count
>
0
return
0
def
dumpMessageList
(
self
,
activity_tool
):
# Dump all messages in the table.
message_list
=
[]
dumpMessageList
=
getattr
(
activity_tool
,
'SQLDict_dumpMessageList'
,
None
)
if
dumpMessageList
is
not
None
:
result
=
dumpMessageList
()
for
line
in
result
:
m
=
Message
.
load
(
line
.
message
,
uid
=
line
.
uid
,
line
=
line
)
message_list
.
append
(
m
)
return
message_list
def
distribute
(
self
,
activity_tool
,
node_count
):
offset
=
0
assignMessage
=
getattr
(
activity_tool
,
'SQLBase_assignMessage'
,
None
)
...
...
@@ -289,72 +266,4 @@ class SQLDict(SQLBase):
return
offset
+=
READ_MESSAGE_LIMIT
# Validation private methods
def
_validate
(
self
,
activity_tool
,
method_id
=
None
,
message_uid
=
None
,
path
=
None
,
tag
=
None
,
serialization_tag
=
None
):
if
isinstance
(
method_id
,
str
):
method_id
=
[
method_id
]
if
isinstance
(
path
,
str
):
path
=
[
path
]
if
isinstance
(
tag
,
str
):
tag
=
[
tag
]
if
method_id
or
message_uid
or
path
or
tag
or
serialization_tag
:
validateMessageList
=
activity_tool
.
SQLDict_validateMessageList
result
=
validateMessageList
(
method_id
=
method_id
,
message_uid
=
message_uid
,
path
=
path
,
tag
=
tag
,
count
=
False
,
serialization_tag
=
serialization_tag
)
message_list
=
[]
for
line
in
result
:
m
=
Message
.
load
(
line
.
message
,
line
=
line
,
uid
=
line
.
uid
,
date
=
line
.
date
,
processing_node
=
line
.
processing_node
)
if
not
hasattr
(
m
,
'order_validation_text'
):
# BBB
m
.
order_validation_text
=
line
.
order_validation_text
message_list
.
append
(
m
)
return
message_list
else
:
return
[]
def
countMessage
(
self
,
activity_tool
,
tag
=
None
,
path
=
None
,
method_id
=
None
,
message_uid
=
None
,
**
kw
):
"""Return the number of messages which match the given parameters.
"""
if
isinstance
(
tag
,
str
):
tag
=
[
tag
]
if
isinstance
(
path
,
str
):
path
=
[
path
]
if
isinstance
(
method_id
,
str
):
method_id
=
[
method_id
]
result
=
activity_tool
.
SQLDict_validateMessageList
(
method_id
=
method_id
,
path
=
path
,
message_uid
=
message_uid
,
tag
=
tag
,
serialization_tag
=
None
,
count
=
1
)
return
result
[
0
].
uid_count
def
countMessageWithTag
(
self
,
activity_tool
,
value
):
"""Return the number of messages which match the given tag.
"""
return
self
.
countMessage
(
activity_tool
,
tag
=
value
)
# Required for tests (time shift)
def
timeShift
(
self
,
activity_tool
,
delay
,
processing_node
=
None
,
retry
=
None
):
"""
To simulate timeShift, we simply substract delay from
all dates in SQLDict message table
"""
activity_tool
.
SQLDict_timeShift
(
delay
=
delay
,
processing_node
=
processing_node
,
retry
=
retry
)
def
getPriority
(
self
,
activity_tool
):
method
=
activity_tool
.
SQLDict_getPriority
default
=
SQLBase
.
getPriority
(
self
,
activity_tool
)
return
self
.
_getPriority
(
activity_tool
,
method
,
default
)
registerActivity
(
SQLDict
)
product/CMFActivity/Activity/SQLQueue.py
View file @
4859cd0a
...
...
@@ -85,52 +85,6 @@ class SQLQueue(SQLBase):
processing_node_list
=
processing_node_list
,
serialization_tag_list
=
serialization_tag_list
)
def
hasActivity
(
self
,
activity_tool
,
object
,
method_id
=
None
,
only_valid
=
None
,
active_process_uid
=
None
):
hasMessage
=
getattr
(
activity_tool
,
'SQLQueue_hasMessage'
,
None
)
if
hasMessage
is
not
None
:
if
object
is
None
:
my_object_path
=
None
else
:
my_object_path
=
'/'
.
join
(
object
.
getPhysicalPath
())
result
=
hasMessage
(
path
=
my_object_path
,
method_id
=
method_id
,
only_valid
=
only_valid
,
active_process_uid
=
active_process_uid
)
if
len
(
result
)
>
0
:
return
result
[
0
].
message_count
>
0
return
0
def
countMessage
(
self
,
activity_tool
,
tag
=
None
,
path
=
None
,
method_id
=
None
,
message_uid
=
None
,
**
kw
):
"""Return the number of messages which match the given parameters.
"""
if
isinstance
(
tag
,
str
):
tag
=
[
tag
]
if
isinstance
(
path
,
str
):
path
=
[
path
]
if
isinstance
(
method_id
,
str
):
method_id
=
[
method_id
]
result
=
activity_tool
.
SQLQueue_validateMessageList
(
method_id
=
method_id
,
path
=
path
,
message_uid
=
message_uid
,
tag
=
tag
,
serialization_tag
=
None
,
count
=
1
)
return
result
[
0
].
uid_count
def
countMessageWithTag
(
self
,
activity_tool
,
value
):
"""Return the number of messages which match the given tag.
"""
return
self
.
countMessage
(
activity_tool
,
tag
=
value
)
def
dumpMessageList
(
self
,
activity_tool
):
# Dump all messages in the table.
message_list
=
[]
dumpMessageList
=
getattr
(
activity_tool
,
'SQLQueue_dumpMessageList'
,
None
)
if
dumpMessageList
is
not
None
:
result
=
dumpMessageList
()
for
line
in
result
:
m
=
Message
.
load
(
line
.
message
,
uid
=
line
.
uid
,
line
=
line
)
message_list
.
append
(
m
)
return
message_list
def
distribute
(
self
,
activity_tool
,
node_count
):
offset
=
0
assignMessage
=
getattr
(
activity_tool
,
'SQLBase_assignMessage'
,
None
)
...
...
@@ -182,49 +136,4 @@ class SQLQueue(SQLBase):
return
offset
+=
READ_MESSAGE_LIMIT
# Validation private methods
def
_validate
(
self
,
activity_tool
,
method_id
=
None
,
message_uid
=
None
,
path
=
None
,
tag
=
None
,
serialization_tag
=
None
):
if
isinstance
(
method_id
,
str
):
method_id
=
[
method_id
]
if
isinstance
(
path
,
str
):
path
=
[
path
]
if
isinstance
(
tag
,
str
):
tag
=
[
tag
]
if
method_id
or
message_uid
or
path
or
tag
or
serialization_tag
:
validateMessageList
=
activity_tool
.
SQLQueue_validateMessageList
result
=
validateMessageList
(
method_id
=
method_id
,
message_uid
=
message_uid
,
path
=
path
,
tag
=
tag
,
count
=
False
,
serialization_tag
=
serialization_tag
)
message_list
=
[]
for
line
in
result
:
m
=
Message
.
load
(
line
.
message
,
line
=
line
,
uid
=
line
.
uid
,
date
=
line
.
date
,
processing_node
=
line
.
processing_node
)
if
not
hasattr
(
m
,
'order_validation_text'
):
# BBB
m
.
order_validation_text
=
self
.
getOrderValidationText
(
m
)
message_list
.
append
(
m
)
return
message_list
else
:
return
[]
# Required for tests (time shift)
def
timeShift
(
self
,
activity_tool
,
delay
,
processing_node
=
None
):
"""
To simulate timeShift, we simply substract delay from
all dates in SQLQueue message table
"""
activity_tool
.
SQLQueue_timeShift
(
delay
=
delay
,
processing_node
=
processing_node
)
def
getPriority
(
self
,
activity_tool
):
method
=
activity_tool
.
SQLQueue_getPriority
default
=
SQLBase
.
getPriority
(
self
,
activity_tool
)
return
self
.
_getPriority
(
activity_tool
,
method
,
default
)
registerActivity
(
SQLQueue
)
product/CMFActivity/skins/activity/SQL
Dict
_dumpMessageList.zsql
→
product/CMFActivity/skins/activity/SQL
Base
_dumpMessageList.zsql
View file @
4859cd0a
...
...
@@ -7,8 +7,9 @@ cache_time:0
class_name:
class_file:
</dtml-comment>
<params></params>
<params>table
</params>
SELECT * FROM
message
<dtml-var table>
ORDER BY
uid
product/CMFActivity/skins/activity/SQL
Dict
_getPriority.zsql
→
product/CMFActivity/skins/activity/SQL
Base
_getPriority.zsql
View file @
4859cd0a
...
...
@@ -7,10 +7,10 @@ cache_time:0
class_name:
class_file:
</dtml-comment>
<params>
<params>
table
</params>
SELECT `priority` FROM
message
<dtml-var table>
WHERE
processing_node = 0
AND date <= UTC_TIMESTAMP()
...
...
product/CMFActivity/skins/activity/SQL
Dict
_hasMessage.zsql
→
product/CMFActivity/skins/activity/SQL
Base
_hasMessage.zsql
View file @
4859cd0a
...
...
@@ -7,12 +7,13 @@ cache_time:0
class_name:
class_file:
</dtml-comment>
<params>path
<params>table
path
method_id
active_process_uid
only_valid</params>
SELECT count(path) as message_count FROM
message
<dtml-var table>
WHERE 1 = 1
<dtml-if expr="path is not None">AND path = <dtml-sqlvar path type="string"> </dtml-if>
<dtml-if expr="method_id is not None">AND method_id = <dtml-sqlvar method_id type="string"></dtml-if>
...
...
product/CMFActivity/skins/activity/SQL
Queu
e_timeShift.zsql
→
product/CMFActivity/skins/activity/SQL
Bas
e_timeShift.zsql
View file @
4859cd0a
...
...
@@ -7,15 +7,14 @@ cache_time:0
class_name:
class_file:
</dtml-comment>
<params>delay
<params>table
delay
processing_node</params>
UPDATE
message_queue
<dtml-var table>
SET
date = DATE_SUB(date, INTERVAL <dtml-sqlvar delay type="int"> SECOND),
processing_date = DATE_SUB(processing_date, INTERVAL <dtml-sqlvar delay type="int"> SECOND)
WHERE
1 = 1
<dtml-if expr="processing_node is not None">
AND processing_node = <dtml-sqlvar
processing_node type="int">
WHERE <dtml-sqltest
processing_node type="int">
</dtml-if>
product/CMFActivity/skins/activity/SQL
Dict
_validateMessageList.zsql
→
product/CMFActivity/skins/activity/SQL
Base
_validateMessageList.zsql
View file @
4859cd0a
...
...
@@ -7,7 +7,8 @@ cache_time:0
class_name:
class_file:
</dtml-comment>
<params>method_id
<params>table
method_id
message_uid
path
tag
...
...
@@ -21,7 +22,7 @@ SELECT
*
</dtml-if>
FROM
message
<dtml-var table>
WHERE
processing_node > -10
<dtml-if expr="method_id is not None">
...
...
product/CMFActivity/skins/activity/SQLDict_timeShift.zsql
deleted
100644 → 0
View file @
bb2bfa6e
<dtml-comment>
title:
connection_id:cmf_activity_sql_connection
max_rows:1
max_cache:0
cache_time:0
class_name:
class_file:
</dtml-comment>
<params>delay
processing_node
retry</params>
UPDATE
message
SET
date = DATE_SUB(date, INTERVAL <dtml-sqlvar delay type="int"> SECOND),
processing_date = DATE_SUB(processing_date, INTERVAL <dtml-sqlvar delay type="int"> SECOND)
<dtml-if expr="retry is not None">
,retry = GREATEST(retry,<dtml-sqlvar retry type="int">) - <dtml-sqlvar retry type="int">
</dtml-if>
WHERE
1 = 1
<dtml-if expr="processing_node is not None">
AND processing_node = <dtml-sqlvar processing_node type="int">
</dtml-if>
product/CMFActivity/skins/activity/SQLQueue_dumpMessageList.zsql
deleted
100644 → 0
View file @
bb2bfa6e
<dtml-comment>
title:
connection_id:cmf_activity_sql_connection
max_rows:0
max_cache:0
cache_time:0
class_name:
class_file:
</dtml-comment>
<params></params>
SELECT * FROM
message_queue
ORDER BY
uid
product/CMFActivity/skins/activity/SQLQueue_getPriority.zsql
deleted
100644 → 0
View file @
bb2bfa6e
<dtml-comment>
title:
connection_id:cmf_activity_sql_connection
max_rows:0
max_cache:0
cache_time:0
class_name:
class_file:
</dtml-comment>
<params>
</params>
SELECT `priority` FROM
message_queue
WHERE
processing_node = 0
AND date <= UTC_TIMESTAMP()
ORDER BY priority
LIMIT 1
product/CMFActivity/skins/activity/SQLQueue_hasMessage.zsql
deleted
100644 → 0
View file @
bb2bfa6e
<dtml-comment>
title:
connection_id:cmf_activity_sql_connection
max_rows:1
max_cache:0
cache_time:0
class_name:
class_file:
</dtml-comment>
<params>path
method_id
active_process_uid
only_valid</params>
SELECT count(path) as message_count FROM
message_queue
WHERE 1 = 1
<dtml-if expr="path is not None">AND path = <dtml-sqlvar path type="string"> </dtml-if>
<dtml-if expr="method_id is not None"> AND method_id = <dtml-sqlvar method_id type="string"> </dtml-if>
<dtml-if expr="only_valid"> AND processing_node > -2 </dtml-if>
<dtml-if expr="active_process_uid is not None"> AND active_process_uid = <dtml-sqlvar active_process_uid type="int"> </dtml-if>
product/CMFActivity/skins/activity/SQLQueue_validateMessageList.zsql
deleted
100644 → 0
View file @
bb2bfa6e
<dtml-comment>
title:
connection_id:cmf_activity_sql_connection
max_rows:1000
max_cache:0
cache_time:0
class_name:
class_file:
</dtml-comment>
<params>method_id
message_uid
path
tag
count
serialization_tag
</params>
SELECT
<dtml-if expr="count">
COUNT(*) AS uid_count
<dtml-else>
*
</dtml-if>
FROM
message_queue
WHERE
processing_node > -10
<dtml-if expr="method_id is not None">
AND method_id IN (
<dtml-in method_id><dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
)
</dtml-if>
<dtml-if expr="message_uid is not None">AND uid = <dtml-sqlvar message_uid type="int"> </dtml-if>
<dtml-if expr="path is not None">
AND path IN (
<dtml-in path><dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
)
</dtml-if>
<dtml-if expr="tag is not None">
AND tag IN (
<dtml-in tag><dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
)
</dtml-if>
<dtml-if expr="serialization_tag is not None">
AND processing_node > -1
AND serialization_tag = <dtml-sqlvar serialization_tag type="string">
</dtml-if>
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment