Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
138
Merge Requests
138
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
erp5
Commits
ecefe4dd
Commit
ecefe4dd
authored
1 year ago
by
Jérome Perrin
Committed by
Arnaud Fontaine
8 months ago
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
big_file: py3
parent
d8d24248
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
36 additions
and
30 deletions
+36
-30
bt5/erp5_big_file/DocumentTemplateItem/portal_components/document.erp5.BigFile.py
...ntTemplateItem/portal_components/document.erp5.BigFile.py
+25
-21
bt5/erp5_big_file/ModuleComponentTemplateItem/portal_components/module.erp5.BTreeData.py
...ntTemplateItem/portal_components/module.erp5.BTreeData.py
+11
-9
No files found.
bt5/erp5_big_file/DocumentTemplateItem/portal_components/document.erp5.BigFile.py
View file @
ecefe4dd
...
...
@@ -14,7 +14,6 @@
#
##############################################################################
from
six.moves
import
cStringIO
as
StringIO
from
AccessControl
import
ClassSecurityInfo
from
Products.ERP5Type
import
Permissions
,
PropertySheet
from
Products.ERP5Type.Base
import
removeIContentishInterface
...
...
@@ -23,14 +22,17 @@ from erp5.component.module.BTreeData import BTreeData
from
ZPublisher.HTTPRequest
import
FileUpload
from
ZPublisher
import
HTTPRangeSupport
from
zope.datetime
import
rfc1123_date
from
mimetools
import
choose_boundary
from
Products.CMFCore.utils
import
_setCacheHeaders
,
_ViewEmulator
from
DateTime
import
DateTime
import
re
import
io
import
six
if
six
.
PY3
:
long
=
int
# pylint:disable=redefined-builtin
from
email.generator
import
_make_boundary
as
choose_boundary
else
:
from
mimetools
import
choose_boundary
# pylint:disable=import-error
class
BigFile
(
File
):
"""
...
...
@@ -43,10 +45,10 @@ class BigFile(File):
data property is either
- BTreeData instance, or
-
str
(*), or
-
bytes
(*), or
- None.
(*)
str has to be supported because
'' is a default value for `data` field
(*)
bytes has to be supported because b
'' is a default value for `data` field
from Data property sheet.
Even more - for
...
...
@@ -55,7 +57,7 @@ class BigFile(File):
b) desire to support automatic migration of File-based documents
from document_module to BigFiles
non-empty
str
for data also have to be supported.
non-empty
bytes
for data also have to be supported.
XXX(kirr) I'm not sure supporting non-empty str is a good idea (it
would be simpler if .data could be either BTreeData or "empty"),
...
...
@@ -64,6 +66,8 @@ class BigFile(File):
We discussed with Romain and settled on "None or str or BTreeData"
invariant for now.
notes: for python3 port "str" becomes "bytes", but kirr message was not modified.
"""
meta_type
=
'ERP5 Big File'
...
...
@@ -115,9 +119,9 @@ class BigFile(File):
# of memory.
n
=
1
<<
16
if
isinstance
(
file
,
str
):
if
isinstance
(
file
,
bytes
):
# Big string: cut it into smaller chunks
file
=
String
IO
(
file
)
file
=
io
.
Bytes
IO
(
file
)
if
isinstance
(
file
,
FileUpload
)
and
not
file
:
raise
ValueError
(
'File not specified'
)
...
...
@@ -130,9 +134,9 @@ class BigFile(File):
if
data
is
None
:
btree
=
BTreeData
()
elif
isinstance
(
data
,
str
):
elif
isinstance
(
data
,
bytes
):
# we'll want to append content to this file -
# - automatically convert
str
(empty or not) to BTreeData
# - automatically convert
bytes
(empty or not) to BTreeData
btree
=
BTreeData
()
btree
.
write
(
data
,
0
)
else
:
...
...
@@ -236,7 +240,7 @@ class BigFile(File):
RESPONSE
.
setStatus
(
206
)
# Partial content
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
if
isinstance
(
data
,
str
):
if
isinstance
(
data
,
bytes
):
RESPONSE
.
write
(
data
[
start
:
end
])
return
True
for
chunk
in
data
.
iterate
(
start
,
end
-
start
):
...
...
@@ -271,22 +275,22 @@ class BigFile(File):
RESPONSE
.
setStatus
(
206
)
# Partial content
for
start
,
end
in
ranges
:
RESPONSE
.
write
(
'
\
r
\
n
--%s
\
r
\
n
'
%
boundary
)
RESPONSE
.
write
(
'Content-Type: %s
\
r
\
n
'
%
self
.
content_type
)
RESPONSE
.
write
(
(
'
\
r
\
n
--%s
\
r
\
n
'
%
boundary
).
encode
()
)
RESPONSE
.
write
(
(
'Content-Type: %s
\
r
\
n
'
%
self
.
content_type
)
.
encode
())
RESPONSE
.
write
(
'Content-Range: bytes %d-%d/%d
\
r
\
n
\
r
\
n
'
%
(
start
,
end
-
1
,
self
.
getSize
()))
(
'Content-Range: bytes %d-%d/%d
\
r
\
n
\
r
\
n
'
%
(
start
,
end
-
1
,
self
.
getSize
()))
.
encode
())
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
if
isinstance
(
data
,
str
):
if
isinstance
(
data
,
bytes
):
RESPONSE
.
write
(
data
[
start
:
end
])
else
:
for
chunk
in
data
.
iterate
(
start
,
end
-
start
):
RESPONSE
.
write
(
chunk
)
RESPONSE
.
write
(
'
\
r
\
n
--%s--
\
r
\
n
'
%
boundary
)
RESPONSE
.
write
(
(
'
\
r
\
n
--%s--
\
r
\
n
'
%
boundary
).
encode
()
)
return
True
security
.
declareProtected
(
Permissions
.
View
,
'index_html'
)
...
...
@@ -296,7 +300,7 @@ class BigFile(File):
"""
if
self
.
_range_request_handler
(
REQUEST
,
RESPONSE
):
# we served a chunk of content in response to a range request.
return
''
return
b
''
web_cache_kw
=
kw
.
copy
()
if
format
is
not
_MARKER
:
...
...
@@ -327,13 +331,13 @@ class BigFile(File):
if
data
is
None
:
return
''
if
isinstance
(
data
,
str
):
return
b
''
if
isinstance
(
data
,
bytes
):
RESPONSE
.
setBase
(
None
)
return
data
for
chunk
in
data
.
iterate
():
RESPONSE
.
write
(
chunk
)
return
''
return
b
''
security
.
declareProtected
(
Permissions
.
ModifyPortalContent
,
'PUT'
)
def
PUT
(
self
,
REQUEST
,
RESPONSE
):
...
...
This diff is collapsed.
Click to expand it.
bt5/erp5_big_file/ModuleComponentTemplateItem/portal_components/module.erp5.BTreeData.py
View file @
ecefe4dd
...
...
@@ -3,6 +3,7 @@ from BTrees.LOBTree import LOBTree
from
persistent
import
Persistent
import
itertools
from
six.moves
import
range
import
six
# Maximum memory to allocate for sparse-induced padding.
MAX_PADDING_CHUNK
=
2
**
20
...
...
@@ -11,11 +12,13 @@ class PersistentString(Persistent):
def
__init__
(
self
,
value
):
self
.
value
=
value
def
__
str
__
(
self
):
def
__
bytes
__
(
self
):
return
self
.
value
if
six
.
PY2
:
__str__
=
__bytes__
# Save place when storing this data in zodb
__getstate__
=
__
str
__
__getstate__
=
__
bytes
__
__setstate__
=
__init__
negative_offset_error
=
ValueError
(
'Negative offset'
)
...
...
@@ -110,7 +113,7 @@ class BTreeData(Persistent):
chunk
=
tree
[
lower_key
]
chunk_end
=
lower_key
+
len
(
chunk
.
value
)
if
chunk_end
>
offset
or
(
len
(
chunk
.
value
)
<
self
.
_chunk_size
and
len
(
chunk
.
value
)
<
(
self
.
_chunk_size
or
0
)
and
chunk_end
==
offset
):
key
=
lower_key
...
...
@@ -137,7 +140,7 @@ class BTreeData(Persistent):
try
:
chunk
=
tree
[
key
]
except
KeyError
:
tree
[
key
]
=
chunk
=
PersistentString
(
''
)
tree
[
key
]
=
chunk
=
PersistentString
(
b
''
)
entry_size
=
len
(
chunk
.
value
)
if
entry_size
<
to_write_len
:
to_write_len
=
min
(
to_write_len
,
max_to_write_len
)
...
...
@@ -158,9 +161,9 @@ class BTreeData(Persistent):
size (int)
Number of bytes to read.
Returns
string
of read data.
Returns
bytes
of read data.
"""
return
''
.
join
(
self
.
iterate
(
offset
,
size
))
return
b
''
.
join
(
self
.
iterate
(
offset
,
size
))
def
iterate
(
self
,
offset
=
0
,
size
=
None
):
"""
...
...
@@ -243,7 +246,7 @@ class BTreeData(Persistent):
except
ValueError
:
break
del
tree
[
key
]
self
.
write
(
''
,
offset
)
self
.
write
(
b
''
,
offset
)
# XXX: Various batch_size values need to be benchmarked, and a saner
# default is likely to be applied.
...
...
@@ -314,12 +317,11 @@ class BTreeData(Persistent):
tree
[
key
]
=
next_chunk
if
__name__
==
'__main__'
:
def
check
(
tree
,
length
,
read_offset
,
read_length
,
data_
,
keys
=
None
):
print
(
list
(
tree
.
_tree
.
items
()))
tree_length
=
len
(
tree
)
tree_data
=
tree
.
read
(
read_offset
,
read_length
)
tree_iterator_data
=
''
.
join
(
tree
.
iterate
(
read_offset
,
read_length
))
tree_iterator_data
=
b
''
.
join
(
tree
.
iterate
(
read_offset
,
read_length
))
assert
tree_length
==
length
,
tree_length
assert
tree_data
==
data_
,
repr
(
tree_data
)
assert
tree_iterator_data
==
data_
,
repr
(
tree_iterator_data
)
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment