Commit 484a14e9 authored by Roque's avatar Roque

erp5_wendelin_data_lake_ingestion: script to get size of all data lake and by dataset

parent 07e4bdb7
import json
portal = context.getPortalObject()
portal_catalog = portal.portal_catalog
def getDatasetInfo(data_set):
size = 0
datastream_result_dict = json.loads(portal.ERP5Site_getDataStreamList(data_set.getReference()))
for stream_dict in datastream_result_dict['result']:
size += stream_dict['full-size']
return len(datastream_result_dict['result']), size
def format_size(num, suffix='b'):
for unit in ['','K','M','G','T','P','E','Z']:
if abs(num) < 1024.0:
return "%3.1f %s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f %s%s" % (num, 'Yi', suffix)
data_set_list = []
if data_set_reference:
try:
data_set = portal.data_set_module.get(data_set_reference)
if data_set is None or portal.ERP5Site_checkReferenceInvalidated(data_set):
return "Not found: there is no valid dataset for that reference"
data_set_list.append(data_set)
except Exception as e: # fails because unauthorized access
return "ERROR: " + str(e)
else:
data_set_list = portal_catalog(portal_type="Data Set", validation_state='validated OR published')
total_size = 0
for data_set in data_set_list:
print "Data set " + data_set.getReference()
nfiles, size = getDatasetInfo(data_set)
total_size += size
print " #files: " + str(nfiles)
print " Size: " + format_size(size)
print
if len(data_set_list) > 1:
print
print "TOTAL SIZE: " + format_size(total_size)
return printed
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>data_set_reference=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_checkIngestedData</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment