Commit 01e14257 authored by Vincent Pelletier's avatar Vincent Pelletier

ZMySQLDA: Optimise date & datetime parsing in results.

A pprofile run shows DateTime_or_None costing 3ms per value (including all
callees).
While this may seem low, it is common to receive hundreds of such values
in single query results (ex: web mode document & webpage lookups),
so the total time spent parting dates per query can become significantly
larger than the time to execute the query itself.
In the case of this measure setup, 504 dates were processed for a total of
1.5 seconds.
DateTime is slow to parse strings because it cannot rely on a pre-known
format. With MySQL/MariaDB, the format is know for those column types, so
implement specific parsing to get better performance.
On the same setup, pprofile run shows a parsing time of 123µs per value in
DATETIME_to_DateTime_or_None, or 65ms total.
DATE_to_DateTime_or_None should not deviate significantly from this.
parent 2dc4c674
...@@ -158,9 +158,39 @@ def _mysql_timestamp_converter(s): ...@@ -158,9 +158,39 @@ def _mysql_timestamp_converter(s):
s[8:10],s[10:12],s[12:14])) s[8:10],s[10:12],s[12:14]))
return DateTime("%04d-%02d-%02d %02d:%02d:%02d" % tuple(parts)) return DateTime("%04d-%02d-%02d %02d:%02d:%02d" % tuple(parts))
def DateTime_or_None(s): # DateTime(str) is slow. As the date format is part of the specifications,
try: return DateTime('%s UTC' % s) # parse it ourselves to save time.
except Exception: return None def DATETIME_to_DateTime_or_None(s):
try:
date, time = s.split(' ')
year, month, day = date.split('-')
hour, minte, second = time.split(':')
return DateTime(
int(year),
int(month),
int(day),
int(hour),
int(minute),
float(second),
'UTC',
)
except Exception:
return None
def DATE_to_DateTime_or_None(s):
try:
year, month, day = s.split('-')
return DateTime(
int(year),
int(month),
int(day),
0,
0,
0,
'UTC',
)
except Exception:
return None
def ord_or_None(s): def ord_or_None(s):
if s is not None: if s is not None:
...@@ -171,8 +201,8 @@ class DB(TM): ...@@ -171,8 +201,8 @@ class DB(TM):
conv=conversions.copy() conv=conversions.copy()
conv[FIELD_TYPE.LONG] = int conv[FIELD_TYPE.LONG] = int
conv[FIELD_TYPE.DATETIME] = DateTime_or_None conv[FIELD_TYPE.DATETIME] = DATETIME_to_DateTime_or_None
conv[FIELD_TYPE.DATE] = DateTime_or_None conv[FIELD_TYPE.DATE] = DATE_to_DateTime_or_None
conv[FIELD_TYPE.DECIMAL] = float conv[FIELD_TYPE.DECIMAL] = float
conv[FIELD_TYPE.BIT] = ord_or_None conv[FIELD_TYPE.BIT] = ord_or_None
del conv[FIELD_TYPE.TIME] del conv[FIELD_TYPE.TIME]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment